echo "$f:$l ($(date)) $1"
}
+function make_ceph_setup() {
+ tee ~/ceph_setup.sh <<'EOG'
+#!/bin/bash
+# Basic server pre-reqs
+dist=$(grep --m 1 ID /etc/os-release | awk -F '=' '{print $2}' | sed 's/"//g')
+if [[ "$dist" == "ubuntu" ]]; then
+ sudo apt-get install -y ceph ceph-common
+else
+ # per http://docs.ceph.com/docs/master/install/get-packages/
+ sudo tee /etc/yum.repos.d/ceph.repo <<'EOF'
+[ceph]
+name=Ceph packages for $basearch
+baseurl=https://download.ceph.com/rpm-luminous/el7/x86_64
+enabled=1
+priority=2
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[ceph-noarch]
+name=Ceph noarch packages
+baseurl=https://download.ceph.com/rpm-luminous/el7/noarch
+enabled=1
+priority=2
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[ceph-source]
+name=Ceph source packages
+baseurl=https://download.ceph.com/rpm-luminous/el7/SRPMS
+enabled=0
+priority=2
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+EOF
+ # TODO: find out why package us unsigned and thus need --nogpgcheck
+ sudo rpm --import 'https://download.ceph.com/keys/release.asc'
+ sudo yum install --nogpgcheck -y ceph ceph-common
+fi
+EOG
+}
+
function setup_ceph() {
nodes=$1
private_net=$2
public_net=$3
dev=$4
-
- log "Install ceph prerequisites"
- sudo apt-get -y install ceph ceph-common
+ log "Install ceph and ceph-common"
+ make_ceph_setup
+ bash ~/ceph_setup.sh
+
# per https://github.com/att/netarbiter/tree/master/sds/ceph-docker/examples/helm
log "Clone netarbiter"
git clone https://github.com/att/netarbiter.git
- cd netarbiter/sds/ceph-docker/examples/helm
+
+ if [[ "$dist" != "ubuntu" ]]; then
+ log "Update ceph-helm chart to point to centos images"
+ sed -i -- 's~daemon: docker.io/knowpd~#daemon: docker.io/knowpd~' \
+ netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml
+ sed -i -- 's~#daemon: docker.io/ceph~daemon: docker.io/ceph~' \
+ netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml
+ sed -i -- 's~ceph_init: docker.io/knowpd~#ceph_init: docker.io/knowpd~' \
+ netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml
+ sed -i -- 's~#ceph_init: docker.io/kollakube~ceph_init: docker.io/kollakube~' \
+ netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml
+ fi
log "Prepare a ceph namespace in your K8s cluster"
+ cd netarbiter/sds/ceph-docker/examples/helm
./prep-ceph-ns.sh
log "Run ceph-mon, ceph-mgr, ceph-mon-check, and rbd-provisioner"
for node in $nodes; do
log "install ceph, setup resolv.conf, zap disk for $node"
- ssh -x -o StrictHostKeyChecking=no ubuntu@$node <<EOG
+ if [[ "$dist" == "ubuntu" ]]; then
+ ssh -x -o StrictHostKeyChecking=no $USER@$node \
+ sudo apt-get install -y ceph ceph-common
+ else
+ scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+ ~/ceph_setup.sh $USER@$node:/home/$USER/.
+ ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+ $USER@$node bash /home/$USER/ceph_setup.sh
+ fi
+
+ ssh -x -o StrictHostKeyChecking=no $USER@$node <<EOG
cat <<EOF | sudo tee /etc/resolv.conf
nameserver $kubedns
search ceph.svc.cluster.local svc.cluster.local cluster.local
options ndots:5
EOF
-sudo apt install -y ceph ceph-common
sudo ceph-disk zap /dev/$dev
EOG
log "Run ceph-osd at $node"
- name=$(ssh -x -o StrictHostKeyChecking=no ubuntu@$node hostname)
+ name=$(ssh -x -o StrictHostKeyChecking=no $USER@$node hostname)
+ # TODO: try sudo due to error
+ # command_check_call: Running command: /usr/bin/ceph-osd --cluster ceph --mkfs -i 0 --monmap /var/lib/ceph/tmp/mnt.JKiQbp/activate.monmap --osd-data /var/lib/ceph/tmp/mnt.JKiQbp --osd-uuid 23e72c93-e5b3-48ad-b919-ef59fe92b189 --setuser ceph --setgroup disk ... -1 bluestore(/var/lib/ceph/tmp/mnt.JKiQbp) _setup_block_symlink_or_file failed to open block file: (13) Permission denied
+ # TODO: leave out sudo... resulted in "./helm-install-ceph-osd.sh: line 40: helm: command not found"
+ ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+ $USER@$node sudo chmod 777 /var/lib/ceph/tmp
./helm-install-ceph-osd.sh $name /dev/$dev
done
for node in $nodes; do
- name=$(ssh -x -o StrictHostKeyChecking=no ubuntu@$node hostname)
+ name=$(ssh -x -o StrictHostKeyChecking=no $USER@$node hostname)
pod=$(kubectl get pods --namespace ceph | awk "/$name/{print \$1}")
- log "verify ceph-osd is Running at node $name"
+ while "$pod" == "" ; do
+ log "ceph-osd pod not yet created at node $name, waiting 10 seconds"
+ kubectl get pods --namespace ceph
+ sleep 10
+ pod=$(kubectl get pods --namespace ceph | awk "/$name/{print \$1}")
+ done
+
+ log "wait till ceph-osd pod $pod is Running at node $name"
status=$(kubectl get pods --namespace ceph $pod | awk "/$pod/ {print \$3}")
while [[ "x$status" != "xRunning" ]]; do
log "$pod status is $status. Waiting 10 seconds for it to be Running."
#. - OPNFV VES as an ONAP-compatible monitoring platform
#.
#. Prerequisites:
-#. - Ubuntu server for kubernetes cluster nodes (master and worker nodes)
-#. - MAAS server as cluster admin for kubernetes master/worker nodes
+#. - MAAS server as cluster admin for k8s master/worker nodes.
#. - Password-less ssh key provided for node setup
#. - hostname of kubernetes master setup in DNS or /etc/hosts
#. Usage: on the MAAS server
#. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models
-#. $ bash ~/models/tools/kubernetes/demo_deploy.sh <key> "<hosts>" <master>
-#. "<workers>" <pub-net> <priv-net> <ceph-mode> <ceph-dev> [<extras>]
-#. <key>: name of private key for cluster node ssh (in current folder)
+#. $ bash ~/models/tools/kubernetes/demo_deploy.sh "<hosts>" <os> <key>
+#. <master> "<workers>" <pub-net> <priv-net> <ceph-mode> <ceph-dev> [<extras>]
#. <hosts>: space separated list of hostnames managed by MAAS
+#. <os>: OS to deploy, one of "ubuntu" (Xenial) or "centos" (Centos 7)
+#. <key>: name of private key for cluster node ssh (in current folder)
#. <master>: IP of cluster master node
#. <workers>: space separated list of worker node IPs
#. <pub-net>: CID formatted public network
function run_master() {
start=$((`date +%s`/60))
ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
- ubuntu@$k8s_master <<EOF
+ $k8s_user@$k8s_master <<EOF
exec ssh-agent bash
ssh-add $k8s_key
$1
step_end "$1"
}
-extras=$9
+extras=${10}
+# Note MAAS deploys OS's with default user same as OS name
cat <<EOF >~/k8s_env.sh
-k8s_key=$1
-k8s_nodes="$2"
-k8s_master=$3
-k8s_workers="$4"
-k8s_priv_net=$5
-k8s_pub_net=$6
-k8s_ceph_mode=$7
-k8s_ceph_dev=$8
-export k8s_key
+k8s_nodes="$1"
+k8s_user=$2
+k8s_key=$3
+k8s_master=$4
+k8s_workers="$5"
+k8s_priv_net=$6
+k8s_pub_net=$7
+k8s_ceph_mode=$8
+k8s_ceph_dev=$9
export k8s_nodes
+export k8s_user
+export k8s_key
export k8s_master
export k8s_workers
export k8s_priv_net
source ~/k8s_env.sh
env | grep k8s_
-source ~/models/tools/maas/deploy.sh $k8s_key "$k8s_nodes" $extras
+source ~/models/tools/maas/deploy.sh $k8s_user $k8s_key "$k8s_nodes" $extras
eval `ssh-agent`
ssh-add $k8s_key
scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_key \
- ubuntu@$k8s_master:/home/ubuntu/$k8s_key
+ $k8s_user@$k8s_master:/home/$k8s_user/$k8s_key
scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ~/k8s_env.sh \
- ubuntu@$k8s_master:/home/ubuntu/.
+ $k8s_user@$k8s_master:/home/$k8s_user/.
echo; echo "$0 $(date): Setting up kubernetes master..."
scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
- ~/models/tools/kubernetes/* ubuntu@$k8s_master:/home/ubuntu/.
+ ~/models/tools/kubernetes/* $k8s_user@$k8s_master:/home/$k8s_user/.
run_master "bash k8s-cluster.sh master"
echo; echo "$0 $(date): Setting up kubernetes workers..."
echo; echo "Setting up Prometheus..."
scp -r -o StrictHostKeyChecking=no ~/models/tools/prometheus/* \
- ubuntu@$k8s_master:/home/ubuntu/.
+ $k8s_user@$k8s_master:/home/$k8s_user/.
run_master "bash prometheus-tools.sh all \"$k8s_workers\""
echo; echo "$0 $(date): Setting up cloudify..."
scp -r -o StrictHostKeyChecking=no ~/models/tools/cloudify \
- ubuntu@$k8s_master:/home/ubuntu/.
+ $k8s_user@$k8s_master:/home/$k8s_user/.
run_master "bash cloudify/k8s-cloudify.sh prereqs"
run_master "bash cloudify/k8s-cloudify.sh setup"
echo; echo "$0 $(date): Setting up VES"
# not re-cloned if existing - allows patch testing locally
if [[ ! -d ~/ves ]]; then
+ echo; echo "$0 $(date): Cloning VES"
git clone https://gerrit.opnfv.org/gerrit/ves ~/ves
fi
ves_influxdb_host=$k8s_master:8086
export ves_grafana_host
ves_grafana_auth=admin:admin
export ves_grafana_auth
-ves_kafka_hostname=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$k8s_master hostname)
+ves_kafka_hostname=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master hostname)
export ves_kafka_hostname
ves_loglevel=$ves_loglevel
export ves_loglevel
# Can't pass quoted strings in commands
start=$((`date +%s`/60))
-bash $HOME/ves/tools/demo_deploy.sh $k8s_key $k8s_master "$k8s_workers"
-step_end "bash $HOME/ves/tools/demo_deploy.sh $k8s_key $k8s_master \"$k8s_workers\""
+bash $HOME/ves/tools/demo_deploy.sh $k8s_key $k8s_user $k8s_master "$k8s_workers"
+step_end "bash $HOME/ves/tools/demo_deploy.sh $k8s_key $k8s_user $k8s_master \"$k8s_workers\""
echo; echo "$0 $(date): All done!"
-export NODE_PORT=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$k8s_master kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services dw-dokuwiki)
-export NODE_IP=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$k8s_master kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
+export NODE_PORT=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services dw-dokuwiki)
+export NODE_IP=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
echo "Helm chart demo app dokuwiki is available at http://$NODE_IP:$NODE_PORT/"
# TODO update Cloudify demo app to have public exposed service address
port=$( bash ~/models/tools/cloudify/k8s-cloudify.sh port nginx $k8s_master)
#
#. What this is: script to setup a kubernetes cluster with calico as sni
#. Prerequisites:
-#. - Ubuntu xenial server for master and worker nodes
+#. - Ubuntu Xenial or Centos 7 server for master and worker nodes
#. - key-based auth setup for ssh/scp between master and worker nodes
#. - 192.168.0.0/16 should not be used on your server network interface subnets
#. Usage:
f=$(caller 0 | awk '{print $2}')
l=$(caller 0 | awk '{print $1}')
echo; echo "$f:$l ($(date)) $1"
+ kubectl get pods --all-namespaces
}
function setup_prereqs() {
log "Create prerequisite setup script"
- cat <<'EOG' >/tmp/prereqs.sh
+ cat <<'EOG' >~/prereqs.sh
#!/bin/bash
# Basic server pre-reqs
-echo; echo "prereqs.sh: ($(date)) Basic prerequisites"
-sudo apt-get update
-sudo apt-get upgrade -y
+dist=$(grep --m 1 ID /etc/os-release | awk -F '=' '{print $2}' | sed 's/"//g')
if [[ $(grep -c $HOSTNAME /etc/hosts) -eq 0 ]]; then
echo; echo "prereqs.sh: ($(date)) Add $HOSTNAME to /etc/hosts"
- echo "$(ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME" \
+ # have to add "/sbin" to path of IP command for centos
+ echo "$(/sbin/ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME" \
| sudo tee -a /etc/hosts
fi
-echo; echo "prereqs.sh: ($(date)) Install latest docker"
-sudo apt-get install -y docker.io
-# Alternate for 1.12.6
-#sudo apt-get install -y libltdl7
-#wget https://packages.docker.com/1.12/apt/repo/pool/main/d/docker-engine/docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb
-#sudo dpkg -i docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb
-sudo service docker restart
-echo; echo "prereqs.sh: ($(date)) Get k8s packages"
-export KUBE_VERSION=1.7.5
-# per https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/
-# Install kubelet, kubeadm, kubectl per https://kubernetes.io/docs/setup/independent/install-kubeadm/
-sudo apt-get update && sudo apt-get install -y apt-transport-https
-curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
-cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
+if [[ "$dist" == "ubuntu" ]]; then
+ # Per https://kubernetes.io/docs/setup/independent/install-kubeadm/
+ echo; echo "prereqs.sh: ($(date)) Basic prerequisites"
+ sudo apt-get update
+ sudo apt-get upgrade -y
+ echo; echo "prereqs.sh: ($(date)) Install latest docker"
+ sudo apt-get install -y docker.io
+ # Alternate for 1.12.6
+ #sudo apt-get install -y libltdl7
+ #wget https://packages.docker.com/1.12/apt/repo/pool/main/d/docker-engine/docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb
+ #sudo dpkg -i docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb
+ sudo service docker restart
+ echo; echo "prereqs.sh: ($(date)) Get k8s packages"
+ export KUBE_VERSION=1.7.5
+ # per https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/
+ # Install kubelet, kubeadm, kubectl per https://kubernetes.io/docs/setup/independent/install-kubeadm/
+ sudo apt-get update && sudo apt-get install -y apt-transport-https
+ curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+ cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
deb http://apt.kubernetes.io/ kubernetes-xenial main
EOF
-sudo apt-get update
-echo; echo "prereqs.sh: ($(date)) Install kubectl, kubelet, kubeadm"
-sudo apt-get -y install --allow-downgrades kubectl=${KUBE_VERSION}-00 \
- kubelet=${KUBE_VERSION}-00 kubeadm=${KUBE_VERSION}-00
-echo; echo "prereqs.sh: ($(date)) Install jq for API output parsing"
-sudo apt-get -y install jq
-echo; echo "prereqs.sh: ($(date)) Set firewall rules"
-# Per https://kubernetes.io/docs/setup/independent/install-kubeadm/
-if [[ "$(sudo ufw status)" == "Status: active" ]]; then
- if [[ "$1" == "master" ]]; then
- sudo ufw allow 6443/tcp
- sudo ufw allow 2379:2380/tcp
- sudo ufw allow 10250/tcp
- sudo ufw allow 10251/tcp
- sudo ufw allow 10252/tcp
- sudo ufw allow 10255/tcp
- else
- sudo ufw allow 10250/tcp
- sudo ufw allow 10255/tcp
- sudo ufw allow 30000:32767/tcp
+ sudo apt-get update
+ echo; echo "prereqs.sh: ($(date)) Install kubectl, kubelet, kubeadm"
+ sudo apt-get -y install --allow-downgrades kubectl=${KUBE_VERSION}-00 \
+ kubelet=${KUBE_VERSION}-00 kubeadm=${KUBE_VERSION}-00
+ echo; echo "prereqs.sh: ($(date)) Install jq for API output parsing"
+ sudo apt-get install -y jq
+ if [[ "$(sudo ufw status)" == "Status: active" ]]; then
+ echo; echo "prereqs.sh: ($(date)) Set firewall rules"
+ if [[ "$1" == "master" ]]; then
+ sudo ufw allow 6443/tcp
+ sudo ufw allow 2379:2380/tcp
+ sudo ufw allow 10250/tcp
+ sudo ufw allow 10251/tcp
+ sudo ufw allow 10252/tcp
+ sudo ufw allow 10255/tcp
+ else
+ sudo ufw allow 10250/tcp
+ sudo ufw allow 10255/tcp
+ sudo ufw allow 30000:32767/tcp
+ fi
fi
+ # TODO: fix need for this workaround: disable firewall since the commands
+ # above do not appear to open the needed ports, even if ufw is inactive
+ # (symptom: nodeport requests fail unless sent from within the cluster or
+ # to the node IP where the pod is assigned) issue discovered ~11/16/17
+ sudo ufw disable
+else
+ echo; echo "prereqs.sh: ($(date)) Basic prerequisites"
+ sudo yum update -y
+ sudo yum install -y wget git
+ echo; echo "prereqs.sh: ($(date)) Install latest docker"
+ # per https://docs.docker.com/engine/installation/linux/docker-ce/centos/#install-from-a-package
+ sudo yum install -y docker
+ sudo systemctl enable docker
+ sudo systemctl start docker
+# wget https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-17.09.0.ce-1.el7.centos.x86_64.rpm
+# sudo yum install -y docker-ce-17.09.0.ce-1.el7.centos.x86_64.rpm
+# sudo systemctl start docker
+ echo; echo "prereqs.sh: ($(date)) Install kubectl, kubelet, kubeadm"
+ cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
+[kubernetes]
+name=Kubernetes
+baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
+EOF
+ sudo setenforce 0
+ sudo yum install -y kubelet kubeadm kubectl
+ sudo systemctl enable kubelet
+ sudo systemctl start kubelet
+ echo; echo "prereqs.sh: ($(date)) Install jq for API output parsing"
+ sudo yum install -y jq
+ echo; echo "prereqs.sh: ($(date)) Set firewall rules"
+ cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
+net.bridge.bridge-nf-call-ip6tables = 1
+net.bridge.bridge-nf-call-iptables = 1
+EOF
+ sudo sysctl --system
fi
-# TODO: fix need for this workaround: disable firewall since the commands
-# above do not appear to open the needed ports, even if ufw is inactive
-# (symptom: nodeport requests fail unless sent from within the cluster or
-# to the node IP where the pod is assigned) issue discovered ~11/16/17
-sudo ufw disable
EOG
}
setup_prereqs
# Install master
- bash /tmp/prereqs.sh master
+ bash ~/prereqs.sh master
# per https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/
# If the following command fails, run "kubeadm reset" before trying again
# --pod-network-cidr=192.168.0.0/16 is required for calico; this should not
# conflict with your server network interface subnets
+ log "Reset kubeadm in case pre-existing cluster"
+ sudo kubeadm reset
+ # Start cluster
+ log "Start the cluster"
sudo kubeadm init --pod-network-cidr=192.168.0.0/16 >>/tmp/kubeadm.out
cat /tmp/kubeadm.out
export k8s_joincmd=$(grep "kubeadm join" /tmp/kubeadm.out)
log "Cluster join command for manual use if needed: $k8s_joincmd"
-
- # Start cluster
- log "Start the cluster"
mkdir -p $HOME/.kube
sudo cp -f /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
# Deploy pod network
log "Deploy calico as CNI"
# Updated to deploy Calico 2.6 per the create-cluster-kubeadm guide above
- # sudo kubectl apply -f http://docs.projectcalico.org/v2.4/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml
- sudo kubectl apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml
+ # kubectl apply -f http://docs.projectcalico.org/v2.4/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml
+ kubectl apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml
+
+ # TODO: document process dependency
+ # Failure to wait for all calico pods to be running can cause the first worker
+ # to be incompletely setup. Symptom is that node_ports cannot be routed
+ # via that node (no response - incoming SYN packets are dropped).
+ log "Wait for calico pods to be Running"
+ # calico-etcd, calico-kube-controllers, calico-node
+ pods=$(kubectl get pods --namespace kube-system | grep -c calico)
+ while [[ $pods -lt 3 ]]; do
+ log "all calico pods are not yet created. Waiting 10 seconds"
+ sleep 10
+ pods=$(kubectl get pods --namespace kube-system | grep -c calico)
+ done
+
+ pods=$(kubectl get pods --all-namespaces | awk '/calico/ {print $2}')
+ for pod in $pods; do
+ status=$(kubectl get pods --all-namespaces | awk "/$pod/ {print \$4}")
+ while [[ "$status" != "Running" ]]; do
+ log "$pod status is $status. Waiting 10 seconds"
+ sleep 10
+ status=$(kubectl get pods --all-namespaces | awk "/$pod/ {print \$4}")
+ done
+ log "$pod status is $status"
+ done
log "Wait for kubedns to be Running"
- kubedns=$(kubectl get pods --all-namespaces | grep kube-dns | awk '{print $4}')
+ kubedns=$(kubectl get pods --all-namespaces | awk '/kube-dns/ {print $4}')
while [[ "$kubedns" != "Running" ]]; do
- log "kube-dns status is $kubedns. Waiting 60 seconds for it to be 'Running'"
+ log "kube-dns status is $kubedns. Waiting 60 seconds"
sleep 60
- kubedns=$(kubectl get pods --all-namespaces | grep kube-dns | awk '{print $4}')
+ kubedns=$(kubectl get pods --all-namespaces | awk '/kube-dns/ {print $4}')
done
log "kube-dns status is $kubedns"
}
export k8s_joincmd=$(grep "kubeadm join" /tmp/kubeadm.out)
log "Installing workers at $1 with joincmd: $k8s_joincmd"
+# TODO: kubeadm reset below is workaround for
+# Ubuntu: "[preflight] Some fatal errors occurred: /var/lib/kubelet is not empty"
+# per https://github.com/kubernetes/kubeadm/issues/1
+# Centos: "Failed to start ContainerManager failed to initialize top
+# level QOS containers: root container /kubepods doesn't exist"
+ tee start_worker.sh <<EOF
+sudo kubeadm reset
+sudo $k8s_joincmd
+EOF
+
for worker in $workers; do
- log "Install worker at $worker"
+ host=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $USER@$worker hostname)
+ log "Install worker at $worker hostname $host"
if ! scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
- /tmp/prereqs.sh ubuntu@$worker:/tmp/prereqs.sh ; then
+ ~/prereqs.sh $USER@$worker:/home/$USER/. ; then
fail "Failed copying setup files to $worker"
fi
+ ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+ $USER@$worker bash prereqs.sh
scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ~/k8s_env.sh \
- ubuntu@$worker:/home/ubuntu/.
+ $USER@$worker:/home/$USER/.
+ scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+ start_worker.sh $USER@$worker:/home/$USER/.
ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
- ubuntu@$worker <<EOF > /dev/null 2>&1 &
-bash /tmp/prereqs.sh worker
-# Workaround for "[preflight] Some fatal errors occurred: /var/lib/kubelet
-# is not empty" per https://github.com/kubernetes/kubeadm/issues/1
-sudo kubeadm reset
-sudo $k8s_joincmd
-EOF
+ $USER@$worker bash start_worker.sh
done
for worker in $workers; do
- host=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$worker hostname)
- log "checking node $host"
+ host=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $USER@$worker hostname)
+ log "checking that node $host is 'Ready'"
status=$(kubectl get nodes | awk "/$host/ {print \$2}")
while [[ "$status" != "Ready" ]]; do
- log "node $host is \"$status\", waiting 10 seconds for it to be 'Ready'."
+ log "node $host is \"$status\", waiting 10 seconds"
status=$(kubectl get nodes | awk "/$host/ {print \$2}")
+ ((tries++))
+ if [[ tries -gt 18 ]]; then
+ log "node $host is \"$status\" after 3 minutes; resetting kubeadm"
+ ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+ $USER@$worker bash start_worker.sh
+ tries=1
+ fi
sleep 10
done
log "node $host is 'Ready'."
done
+
+ log "***** kube proxy pods *****"
+ pods=$(kubectl get pods --all-namespaces | awk '/kube-proxy/ {print $2}')
+ for pod in $pods; do
+ echo; echo "**** $pod ****"
+ kubectl describe pods --namespace kube-system $pod
+ echo; echo "**** $pod logs ****"
+ kubectl logs --namespace kube-system $pod
+ done
+
log "Cluster is ready (all nodes in 'kubectl get nodes' show as 'Ready')."
}
fi
}
+dist=$(grep --m 1 ID /etc/os-release | awk -F '=' '{print $2}' | sed 's/"//g')
+
workers="$2"
privnet=$3
pubnet=$4
Code Review / models.git / commitdiff