From 3a3e735a4498cc05697842d7b329374fffda90cf Mon Sep 17 00:00:00 2001
From: Bryan Sullivan <bryan.sullivan@att.com>
Date: Wed, 18 Oct 2017 16:52:53 -0700
Subject: [PATCH] Tested updates to k8s-cluster and ceph-baremetal

JIRA: MODELS-23
Initial draft of ceph-helm.sh for testing

Change-Id: I39b2edebfe6e9b22de1addd69cc771845495ceb9
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>
---
 tools/kubernetes/ceph-baremetal.sh |  2 +-
 tools/kubernetes/ceph-helm.sh      | 54 ++++++++++++++++++++++++++++++++++++++
 tools/kubernetes/demo_deploy.sh    | 12 +++++----
 tools/kubernetes/k8s-cluster.sh    |  8 +++---
 4 files changed, 67 insertions(+), 9 deletions(-)

diff --git a/tools/kubernetes/ceph-baremetal.sh b/tools/kubernetes/ceph-baremetal.sh
index 55d2a7f..998d136 100644
--- a/tools/kubernetes/ceph-baremetal.sh
+++ b/tools/kubernetes/ceph-baremetal.sh
@@ -192,7 +192,7 @@ EOF
   kubectl describe pods
 }
 
-if [[ "$1" .ne "" ]]; then
+if [[ "$1" != "" ]]; then
   setup_ceph "$1" $2 $3 $4
 else
   grep '#. ' $0
diff --git a/tools/kubernetes/ceph-helm.sh b/tools/kubernetes/ceph-helm.sh
index 96a310c..038c438 100644
--- a/tools/kubernetes/ceph-helm.sh
+++ b/tools/kubernetes/ceph-helm.sh
@@ -30,3 +30,57 @@
 #.
 #. Status: work in progress, incomplete
 #
+
+function setup_ceph() {
+  nodes=$1
+  private_net=$2
+  public_net=$3
+  dev=$4
+  # per https://github.com/att/netarbiter/tree/master/sds/ceph-docker/examples/helm
+  echo "${FUNCNAME[0]}: Clone netarbiter"
+  git clone https://github.com/att/netarbiter.git
+  
+  echo "${FUNCNAME[0]}: Create a .kube/config secret so that a K8s job could run kubectl inside the container"
+  cd netarbiter/sds/ceph-docker/examples/helm
+  kubectl create namespace ceph
+  ./create-secret-kube-config.sh ceph
+  ./helm-install-ceph.sh my_ceph_chart $private_net $public_net
+
+  kubedns=$(kubectl get service -o json --namespace kube-system kube-dns | \
+    jq -r '.spec.clusterIP')
+  case "$dev" in
+    sda)
+    sdb)
+      for node in $nodes; do
+        echo "${FUNCNAME[0]}: setup resolv.conf for $node"
+        echo <<EOF | sudo tee -a /etc/resolv/.conf
+nameserver $kubedns
+search ceph.svc.cluster.local svc.cluster.local cluster.local 
+EOF
+        echo "${FUNCNAME[0]}: Zap disk $dev at $node"
+        ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+          ubuntu@$node ceph-disk zap $dev
+        echo "${FUNCNAME[0]}: Run ceph-osd at $node"
+        name=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+          ubuntu@$node hostname)
+        ./helm-install-ceph-osd.sh $name /dev/$dev
+      done
+      ;;
+      *)
+      ;;
+  esac
+
+  echo "${FUNCNAME[0]}: Activate Ceph for namespace 'default'"
+  ./activate-namespace.sh default
+
+  echo "${FUNCNAME[0]}: Relax access control rules"
+  kubectl replace -f relax-rbac-k8s1.7.yaml
+
+  # TODO: verification tests
+}
+
+if [[ "$1" != "" ]]; then
+  setup_ceph "$1" $2 $3 $4
+else
+  grep '#. ' $0
+fi
diff --git a/tools/kubernetes/demo_deploy.sh b/tools/kubernetes/demo_deploy.sh
index 6ac03b5..04dfacb 100644
--- a/tools/kubernetes/demo_deploy.sh
+++ b/tools/kubernetes/demo_deploy.sh
@@ -30,13 +30,14 @@
 #. Usage: on the MAAS server
 #. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models
 #. $ bash ~/models/tools/kubernetes/demo_deploy.sh <key> "<hosts>" <master>
-#.     "<workers>" <pub-net> <priv-net> [<extras>]
+#.     "<workers>" <pub-net> <priv-net> <ceph-mode> [<extras>]
 #. <key>: name of private key for cluster node ssh (in current folder)
 #. <hosts>: space separated list of hostnames managed by MAAS
 #. <master>: IP of cluster master node
 #. <workers>: space separated list of agent node IPs
 #. <pub-net>: CID formatted public network
 #. <priv-net>: CIDR formatted private network (may be same as pub-net)
+#. <ceph-mode>: "helm" or "baremetal"
 #. <extras>: optional name of script for extra setup functions as needed
 
 key=$1
@@ -45,20 +46,21 @@ master=$3
 workers="$4"
 priv_net=$5
 pub_net=$6
-extras=$7
+ceph_mode=$7
+extras=$8
 
-source ~/models/tools/maas/deploy.sh $1 "$2" $5
+source ~/models/tools/maas/deploy.sh $1 "$2" $8
 eval `ssh-agent`
 ssh-add $key
 if [[ "x$extras" != "x" ]]; then source $extras; fi
 scp -o StrictHostKeyChecking=no $key ubuntu@$master:/home/ubuntu/$key
 echo "Setting up kubernetes..."
-scp -o StrictHostKeyChecking=no ~/models/tools/kubernetes/k8s-cluster.sh \
+scp -r -o StrictHostKeyChecking=no ~/models/tools/kubernetes/* \
   ubuntu@$master:/home/ubuntu/.
 ssh -x -o StrictHostKeyChecking=no ubuntu@$master <<EOF
 exec ssh-agent bash
 ssh-add $key
-bash k8s-cluster.sh all "$workers" $priv_net $pub_net
+bash k8s-cluster.sh all "$workers" $priv_net $pub_net $ceph_mode
 EOF
 # TODO: Figure this out... Have to break the setup into two steps as something
 # causes the ssh session to end before the prometheus setup, if both scripts
diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh
index 015e217..fc40b65 100644
--- a/tools/kubernetes/k8s-cluster.sh
+++ b/tools/kubernetes/k8s-cluster.sh
@@ -69,6 +69,8 @@ sudo apt-get update
 # TODO: reverify if this is still an issue in the final working script
 sudo apt-get -y install ceph-common
 sudo apt-get -y install --allow-downgrades kubectl=${KUBE_VERSION}-00 kubelet=${KUBE_VERSION}-00 kubeadm=${KUBE_VERSION}-00
+# Needed for ceph setup etc
+sudo apt-get -y jq
 EOG
 }
 
@@ -247,10 +249,10 @@ function setup_helm() {
 }
 
 function setup_ceph() {
-  if [[ "$4" .eq "helm" ]]; then
-    source ./ceph-helm.sh "$1" $2 $3 $4
+  if [[ "$4" == "helm" ]]; then
+    source ./ceph-helm.sh "$1" $2 $3 $5
   else
-    source ./ceph-baremetal.sh "$1" $2 $3 $4
+    source ./ceph-baremetal.sh "$1" $2 $3 $5
   fi
 }
 
-- 
2.16.6