#. will be installed:
#. - helm and dokuwiki as a demo helm chart based application
#. - prometheus + grafana for cluster monitoring/stats
+#. And optionally, the following will be installed:
#. - cloudify + kubernetes plugin and a demo hello world (nginx) app installed
#. - OPNFV VES as an ONAP-compatible monitoring platform
+#. - Clearwater-docker as an example complex VNF
#.
#. Prerequisites:
#. - MAAS server as cluster admin for k8s master/worker nodes.
#. Usage: on the MAAS server
#. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models
#. $ bash ~/models/tools/kubernetes/demo_deploy.sh "<hosts>" <os> <key>
-#. <master> "<workers>" <pub-net> <priv-net> <ceph-mode> "<ceph-dev>" [<extras>]
+#. <master> "<workers>" <pub-net> <priv-net> <ceph-mode> "<ceph-dev>"
+#. <base|all> [<extras>]
#. <hosts>: space separated list of hostnames managed by MAAS
#. <os>: OS to deploy, one of "ubuntu" (Xenial) or "centos" (Centos 7)
#. <key>: name of private key for cluster node ssh (in current folder)
#. <ceph-mode>: "helm" or "baremetal"
#. <ceph-dev>: space-separated list of disks (e.g. sda, sdb) to use on each
#. worker, or folder (e.g. "/ceph")
+#. <base|all>: deploy k8s base services, or (for all) add Cloudify, VES, Clearwater
#. <extras>: optional name of script for extra setup functions as needed
#.
#. See tools/demo_deploy.sh in the OPNFV VES repo for additional environment
}
function run_master() {
+ trap 'fail' ERR
start=$((`date +%s`/60))
ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
$k8s_user@$k8s_master <<EOF
deploy_start=$((`date +%s`/60))
-extras=${10}
+deploy=${10}
+extras=${11}
if [[ "$4" != "$5" ]]; then
k8s_master_hostname=$(echo "$1" | cut -d ' ' -f 1)
eval `ssh-agent`
ssh-add $k8s_key
-scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_key \
- $k8s_user@$k8s_master:/home/$k8s_user/$k8s_key
+while ! scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_key \
+ $k8s_user@$k8s_master:/home/$k8s_user/$k8s_key ; do
+ echo; echo "$0 $(date): server is not yet ready for ssh; waiting 10 secs"
+ sleep 10
+done
scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
~/k8s_env.sh $k8s_user@$k8s_master:/home/$k8s_user/k8s_env.sh
echo; echo "$0 $(date): Skipping ceph (not yet working for AIO deployment)"
fi
-echo; echo "$0 $(date): Setting up cloudify..."
-scp -r -o StrictHostKeyChecking=no ~/models/tools/cloudify \
- $k8s_user@$k8s_master:/home/$k8s_user/.
-run_master "bash cloudify/k8s-cloudify.sh prereqs"
-run_master "bash cloudify/k8s-cloudify.sh setup"
-
-echo; echo "$0 $(date): Verifying kubernetes+helm+ceph+cloudify install..."
-run "bash $HOME/models/tools/cloudify/k8s-cloudify.sh demo start"
-
-echo; echo "$0 $(date): Setting up VES..."
-# not re-cloned if existing - allows patch testing locally
-if [[ ! -d ~/ves ]]; then
- echo; echo "$0 $(date): Cloning VES..."
- git clone https://gerrit.opnfv.org/gerrit/ves ~/ves
-fi
-# Can't pass quoted strings in commands
-start=$((`date +%s`/60))
-bash $HOME/ves/tools/demo_deploy.sh $k8s_user $k8s_master cloudify
-step_end "bash $HOME/ves/tools/demo_deploy.sh $k8s_user $k8s_master cloudify"
-
echo; echo "Setting up Prometheus..."
scp -r -o StrictHostKeyChecking=no ~/models/tools/prometheus/* \
$k8s_user@$k8s_master:/home/$k8s_user/.
run_master "bash prometheus-tools.sh setup prometheus helm"
-run_master "bash prometheus-tools.sh setup grafana helm $k8s_master:3000"
+run_master "bash prometheus-tools.sh setup grafana helm"
+
+if [[ "$deploy" == "all" ]]; then
+ echo; echo "$0 $(date): Setting up cloudify..."
+ scp -r -o StrictHostKeyChecking=no ~/models/tools/cloudify \
+ $k8s_user@$k8s_master:/home/$k8s_user/.
+ run_master "bash cloudify/k8s-cloudify.sh prereqs"
+ run_master "bash cloudify/k8s-cloudify.sh setup"
+
+ echo; echo "$0 $(date): Verifying kubernetes+helm+ceph+cloudify install..."
+ run "bash $HOME/models/tools/cloudify/k8s-cloudify.sh demo start"
+
+ echo; echo "$0 $(date): Setting up VES..."
+ # not re-cloned if existing - allows patch testing locally
+ if [[ ! -d ~/ves ]]; then
+ echo; echo "$0 $(date): Cloning VES..."
+ git clone https://gerrit.opnfv.org/gerrit/ves ~/ves
+ fi
+ # Can't pass quoted strings in commands
+ start=$((`date +%s`/60))
+ bash $HOME/ves/tools/demo_deploy.sh $k8s_user $k8s_master cloudify
+ step_end "bash $HOME/ves/tools/demo_deploy.sh $k8s_user $k8s_master cloudify"
+
+ echo; echo "Installing clearwater-docker..."
+ run "bash $HOME/models/tests/k8s-cloudify-clearwater.sh start $k8s_master blsaws latest"
-echo; echo "Installing clearwater-docker..."
-run "bash $HOME/models/tests/k8s-cloudify-clearwater.sh start $k8s_master blsaws latest"
+ echo; echo "Waiting 5 minutes for clearwater IMS to be fully ready..."
+ sleep 300
-echo; echo "Waiting 5 minutes for clearwater IMS to be fully ready..."
-sleep 300
+ echo; echo "Run clearwater-live-test..."
+ run "bash $HOME/models/tests/k8s-cloudify-clearwater.sh test $k8s_master"
+fi
-echo; echo "Run clearwater-live-test..."
-run "bash $HOME/models/tests/k8s-cloudify-clearwater.sh test $k8s_master"
-
echo; echo "$0 $(date): All done!"
deploy_end=$((`date +%s`/60))
runtime=$((deploy_end-deploy_start))
log "Deploy \"$1\" duration = $runtime minutes"
-source ~/ves/tools/ves_env.sh
-#echo "Prometheus UI is available at http://$k8s_master:30990"
-echo "InfluxDB API is available at http://$ves_influxdb_host:$ves_influxdb_port/query&db=veseventsdb&q=<string>"
-echo "Grafana dashboards are available at http://$ves_grafana_host:$ves_grafana_port (login as $ves_grafana_auth)"
-echo "Grafana API is available at http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/v1/query?query=<string>"
echo "Kubernetes API is available at https://$k8s_master:6443/api/v1/"
-echo "Cloudify API access example: curl -u admin:admin --header 'Tenant: default_tenant' http://$k8s_master/api/v3.1/status"
-port=$(bash ~/models/tools/cloudify/k8s-cloudify.sh nodePort nginx)
-echo "Cloudify-deployed demo app nginx is available at http://$k8s_master:$port"
+echo "Prometheus UI is available at http://$k8s_master:30990"
if [[ "$k8s_master" != "$k8s_workers" ]]; then
export NODE_PORT=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services dw-dokuwiki)
export NODE_IP=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
echo "Helm chart demo app dokuwiki is available at http://$NODE_IP:$NODE_PORT/"
fi
+
+if [[ "$deploy" == "all" ]]; then
+ source ~/ves/tools/ves_env.sh
+ echo "InfluxDB API is available at http://$ves_influxdb_host:$ves_influxdb_port/query&db=veseventsdb&q=<string>"
+ echo "Grafana dashboards are available at http://$ves_grafana_host:$ves_grafana_port (login as $ves_grafana_auth)"
+ echo "Grafana API is available at http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/v1/query?query=<string>"
+ echo "Cloudify API access example: curl -u admin:admin --header 'Tenant: default_tenant' http://$k8s_master/api/v3.1/status"
+ port=$(bash ~/models/tools/cloudify/k8s-cloudify.sh nodePort nginx)
+ echo "Cloudify-deployed demo app nginx is available at http://$k8s_master:$port"
+fi
+
function setup_helm() {
log "Setup helm"
# Install Helm
+ # per https://github.com/kubernetes/helm/blob/master/docs/install.md
cd ~
curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh
chmod 700 get_helm.sh
./get_helm.sh
+ log "Initialize helm"
helm init
- nohup helm serve > /dev/null 2>&1 &
- helm repo update
+# nohup helm serve > /dev/null 2>&1 &
+# log "Run helm repo update"
+# helm repo update
# TODO: Workaround for bug https://github.com/kubernetes/helm/issues/2224
# For testing use only!
kubectl create clusterrolebinding permissive-binding \
function wait_for_service() {
log "Waiting for service $1 to be available"
- pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }")
+ # TODO: fix 'head' workaround for more than one pod per service
+ pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }" | head -1)
log "Service $1 is at pod $pod"
ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod)
while [[ "$ready" != "true" ]]; do
#. Status: work in progress, incomplete
#
-trap 'fail' ERR
+# TODO: Debug why some commands below will trigger fail incorrectly
+# trap 'fail' ERR
-function fail() {
- log $1
- exit 1
-}
+# function fail() {
+# log $1
+# exit 1
+# }
function log() {
f=$(caller 0 | awk '{print $2}')
l=$(caller 0 | awk '{print $1}')
echo; echo "$f:$l ($(date)) $1"
- kubectl get pods --all-namespaces
+ if [[ "$kubectl_status" == "ready" ]]; then
+ kubectl get pods --all-namespaces
+ fi
}
function setup_prereqs() {
wait_dpkg; sudo apt-get update
wait_dpkg; sudo apt-get upgrade -y
- echo; echo "prereqs.sh: ($(date)) Install latest docker"
- wait_dpkg; sudo apt-get install -y docker.io
- # Alternate for 1.12.6
- #sudo apt-get install -y libltdl7
- #wget https://packages.docker.com/1.12/apt/repo/pool/main/d/docker-engine/docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb
- #sudo dpkg -i docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb
- sudo service docker restart
+
+ dce=$(dpkg -l | grep -c docker-ce)
+ if [[ $dce -eq 0 ]]; then
+ echo; echo "prereqs.sh: ($(date)) Install latest docker-ce"
+ # Per https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/
+ sudo apt-get remove -y docker docker-engine docker.io docker-ce
+ sudo apt-get update
+ sudo apt-get install -y \
+ apt-transport-https \
+ ca-certificates \
+ curl \
+ software-properties-common
+ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+ sudo add-apt-repository "deb [arch=amd64] \
+ https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+ sudo apt-get update
+ sudo apt-get install -y docker-ce
+ fi
+
echo; echo "prereqs.sh: ($(date)) Get k8s packages"
- export KUBE_VERSION=1.7.5
+ export KUBE_VERSION=1.10.0
# per https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/
# Install kubelet, kubeadm, kubectl per https://kubernetes.io/docs/setup/independent/install-kubeadm/
sudo apt-get update && sudo apt-get install -y apt-transport-https
log "Reset kubeadm in case pre-existing cluster"
sudo kubeadm reset
# Start cluster
- log "Start the cluster"
+ log "Workaround issue '/etc/kubernetes/manifests is not empty'"
mkdir ~/tmp
+ # workaround for [preflight] Some fatal errors occurred:
+ # /etc/kubernetes/manifests is not empty
+ sudo rm -rf /etc/kubernetes/manifests/*
+ log "Disable swap to workaround k8s incompatibility with swap"
+ # per https://github.com/kubernetes/kubeadm/issues/610
+ sudo swapoff -a
+ log "Start the cluster"
sudo kubeadm init --pod-network-cidr=192.168.0.0/16 >>~/tmp/kubeadm.out
cat ~/tmp/kubeadm.out
export k8s_joincmd=$(grep "kubeadm join" ~/tmp/kubeadm.out)
mkdir -p $HOME/.kube
sudo cp -f /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
+ export KUBECONFIG=$HOME/.kube/config
# Deploy pod network
log "Deploy calico as CNI"
# Updated to deploy Calico 2.6 per the create-cluster-kubeadm guide above
# Failure to wait for all calico pods to be running can cause the first worker
# to be incompletely setup. Symptom is that node_ports cannot be routed
# via that node (no response - incoming SYN packets are dropped).
- log "Wait for calico pods to be Running"
+ log "Wait for all calico pods to be Created"
# calico-etcd, calico-kube-controllers, calico-node
pods=$(kubectl get pods --namespace kube-system | grep -c calico)
while [[ $pods -lt 3 ]]; do
pods=$(kubectl get pods --namespace kube-system | grep -c calico)
done
- pods=$(kubectl get pods --all-namespaces | awk '/calico/ {print $2}')
+ log "Wait for all calico pods to be Running"
+ pods=$(kubectl get pods --namespace kube-system | awk '/calico/ {print $1}')
for pod in $pods; do
- status=$(kubectl get pods --all-namespaces | awk "/$pod/ {print \$4}")
+ status=$(kubectl get pods --namespace kube-system | awk "/$pod/ {print \$3}")
while [[ "$status" != "Running" ]]; do
log "$pod status is $status. Waiting 10 seconds"
sleep 10
- status=$(kubectl get pods --all-namespaces | awk "/$pod/ {print \$4}")
+ status=$(kubectl get pods --namespace kube-system | awk "/$pod/ {print \$3}")
done
log "$pod status is $status"
done
log "Wait for kubedns to be Running"
- kubedns=$(kubectl get pods --all-namespaces | awk '/kube-dns/ {print $4}')
+ kubedns=$(kubectl get pods --namespace kube-system | awk '/kube-dns/ {print $3}')
while [[ "$kubedns" != "Running" ]]; do
log "kube-dns status is $kubedns. Waiting 60 seconds"
sleep 60
- kubedns=$(kubectl get pods --all-namespaces | awk '/kube-dns/ {print $4}')
+ kubedns=$(kubectl get pods --namespace kube-system | awk '/kube-dns/ {print $3}')
done
log "kube-dns status is $kubedns"
#. helm: setup/clean via helm
#. agents: for docker-based setup, a quoted, space-separated list agent nodes
#. note: node running this script must have ssh-key enabled access to agents
-#. $ bash prometheus-tools.sh <setup|clean> grafana <docker|helm> [URI] [creds]
+#. $ bash prometheus-tools.sh <setup|clean> grafana <docker|helm> [server] [creds]
#. grafana: setup/clean grafana
#. docker: setup/clean via docker
#. helm: setup/clean via helm
-#. URI: optional URI of grafana server to use
+#. server: optional host:port of grafana server to use
#. creds: optional grafana credentials (default: admin:admin)
#
}
function setup_prometheus() {
+ trap 'fail' ERR
log "Setup prometheus"
log "Setup prerequisites"
if [[ "$dist" == "ubuntu" ]]; then
}
function setup_grafana() {
+ trap 'fail' ERR
host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}')
if [[ "$grafana" == "" ]]; then
if [[ "$how" == "docker" ]]; then
log "Setup grafana via Helm"
#TODO: add --set server.persistentVolume.storageClass=general
helm install --name gf stable/grafana \
- --set server.service.nodePort=30330 \
- --set server.service.type=NodePort \
- --set server.adminPassword=admin \
- --set server.persistentVolume.enabled=false
+ --set service.nodePort=30330 \
+ --set service.type=NodePort \
+ --set adminPassword=admin \
+ --set persistentVolume.enabled=false
fi
grafana=$host_ip:30330
fi
;;
*)
grep '#. ' $0
-esac
\ No newline at end of file
+esac
Code Review / models.git / commitdiff