--- /dev/null
+#!/bin/bash
+# Copyright 2017 AT&T Intellectual Property, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#. What this is: script to setup a Ceph-based SDS (Software Defined Storage)
+#. service for a kubernetes cluster, directly on the master and worker nodes.
+#. Prerequisites:
+#. - Ubuntu xenial server for master and agent nodes
+#. - key-based auth setup for ssh/scp between master and agent nodes
+#. - 192.168.0.0/16 should not be used on your server network interface subnets
+#. Usage:
+# Intended to be called from k8s-cluster.sh in this folder. To run directly:
+#. $ bash ceph-baremetal.sh "<nodes>" <cluster-net> <public-net> [ceph_dev]
+#. nodes: space-separated list of ceph node IPs
+#. cluster-net: CIDR of ceph cluster network e.g. 10.0.0.1/24
+#. public-net: CIDR of public network
+#. ceph_dev: disk to use for ceph. ***MUST NOT BE USED FOR ANY OTHER PURPOSE***
+#. if not provided, ceph data will be stored on osd nodes in /ceph
+#.
+#. Status: work in progress, incomplete
+#
+
+function setup_ceph() {
+ node_ips=$1
+ cluster_net=$2
+ public_net=$3
+ ceph_dev=$4
+ echo "${FUNCNAME[0]}: Deploying ceph-mon on localhost $HOSTNAME"
+ echo "${FUNCNAME[0]}: Deploying ceph-osd on nodes $node_ips"
+ echo "${FUNCNAME[0]}: Setting cluster-network=$cluster_net and public-network=$public_net"
+ mon_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}')
+ all_nodes="$mon_ip $node_ips"
+ # Also caches the server fingerprints so ceph-deploy does not prompt the user
+ # Note this loop may be partially redundant with the ceph-deploy steps below
+ for node_ip in $all_nodes; do
+ echo "${FUNCNAME[0]}: Install ntp and ceph on $node_ip"
+ ssh -x -o StrictHostKeyChecking=no ubuntu@$node_ip <<EOF
+sudo timedatectl set-ntp no
+wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
+echo deb https://download.ceph.com/debian/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
+sudo apt update
+sudo apt-get install -y ntp ceph ceph-deploy
+EOF
+ done
+
+ # per http://docs.ceph.com/docs/master/start/quick-ceph-deploy/
+ # also https://upcommons.upc.edu/bitstream/handle/2117/101816/Degree_Thesis_Nabil_El_Alami.pdf#vote +1
+ echo "${FUNCNAME[0]}: Create ceph config folder ~/ceph-cluster"
+ mkdir ~/ceph-cluster
+ cd ~/ceph-cluster
+
+ echo "${FUNCNAME[0]}: Create new cluster with $HOSTNAME as initial ceph-mon node"
+ ceph-deploy new --cluster-network $cluster_net --public-network $public_net --no-ssh-copykey $HOSTNAME
+ # Update conf per recommendations of http://docs.ceph.com/docs/jewel/rados/configuration/filesystem-recommendations/
+ cat <<EOF >>ceph.conf
+osd max object name len = 256
+osd max object namespace len = 64
+EOF
+ cat ceph.conf
+
+ echo "${FUNCNAME[0]}: Deploy ceph packages on other nodes"
+ ceph-deploy install $mon_ip $node_ips
+
+ echo "${FUNCNAME[0]}: Deploy the initial monitor and gather the keys"
+ ceph-deploy mon create-initial
+
+ if [[ "x$ceph_dev" == "x" ]]; then
+ n=1
+ for node_ip in $node_ips; do
+ echo "${FUNCNAME[0]}: Prepare ceph OSD on node $node_ip"
+ echo "$node_ip ceph-osd$n" | sudo tee -a /etc/hosts
+ # Using ceph-osd$n here avoids need for manual acceptance of the new server hash
+ ssh -x -o StrictHostKeyChecking=no ubuntu@ceph-osd$n <<EOF
+echo "$node_ip ceph-osd$n" | sudo tee -a /etc/hosts
+sudo mkdir /ceph && sudo chown -R ceph:ceph /ceph
+EOF
+ ceph-deploy osd prepare ceph-osd$n:/ceph
+ ceph-deploy osd activate ceph-osd$n:/ceph
+ ((n++))
+ done
+ else
+ echo "${FUNCNAME[0]}: Deploy OSDs"
+ for node_ip in $node_ips; do
+ echo "${FUNCNAME[0]}: Create ceph osd on $node_ip using $ceph_dev"
+ ceph-deploy osd create $node_ip:$ceph_dev
+ done
+ fi
+
+ echo "${FUNCNAME[0]}: Copy the config file and admin key to the admin node and OSD nodes"
+ ceph-deploy admin $mon_ip $node_ips
+
+ echo "${FUNCNAME[0]}: Check the cluster health"
+ sudo ceph health
+ sudo ceph -s
+
+ # per https://crondev.com/kubernetes-persistent-storage-ceph/ and https://github.com/kubernetes/kubernetes/issues/38923
+ # rbd is not included in default kube-controller-manager... use attcomdev version
+ sudo sed -i -- 's~gcr.io/google_containers/kube-controller-manager-amd64:.*~quay.io/attcomdev/kube-controller-manager:v1.7.3~' /etc/kubernetes/manifests/kube-controller-manager.yaml
+ if [[ $(sudo grep -c attcomdev/kube-controller-manager /etc/kubernetes/manifests/kube-controller-manager.yaml) == 0 ]]; then
+ echo "${FUNCNAME[0]}: Problem patching /etc/kubernetes/manifests/kube-controller-manager.yaml... script update needed"
+ exit 1
+ fi
+ mgr=$(kubectl get pods --all-namespaces | grep kube-controller-manager | awk '{print $4}')
+ while [[ "$mgr" != "Running" ]]; do
+ echo "${FUNCNAME[0]}: kube-controller-manager status is $mgr. Waiting 60 seconds for it to be 'Running'"
+ sleep 60
+ mgr=$(kubectl get pods --all-namespaces | grep kube-controller-manager | awk '{print $4}')
+ done
+ echo "${FUNCNAME[0]}: kube-controller-manager status is $mgr"
+
+ echo "${FUNCNAME[0]}: Create Ceph admin secret"
+ admin_key=$(sudo ceph auth get-key client.admin)
+ kubectl create secret generic ceph-secret-admin --from-literal=key="$admin_key" --namespace=kube-system --type=kubernetes.io/rbd
+
+ echo "${FUNCNAME[0]}: Create rdb storageClass 'slow'"
+ cat <<EOF >/tmp/ceph-sc.yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+ name: slow
+provisioner: kubernetes.io/rbd
+parameters:
+ monitors: $mon_ip:6789
+ adminId: admin
+ adminSecretName: ceph-secret-admin
+ adminSecretNamespace: "kube-system"
+ pool: kube
+ userId: kube
+ userSecretName: ceph-secret-user
+EOF
+ # TODO: find out where in the above ~/.kube folders became owned by root
+ sudo chown -R ubuntu:ubuntu ~/.kube/*
+ kubectl create -f /tmp/ceph-sc.yaml
+
+ echo "${FUNCNAME[0]}: Create storage pool 'kube'"
+ # https://github.com/kubernetes/examples/blob/master/staging/persistent-volume-provisioning/README.md method
+ sudo ceph osd pool create kube 32 32
+
+ echo "${FUNCNAME[0]}: Authorize client 'kube' access to pool 'kube'"
+ sudo ceph auth get-or-create client.kube mon 'allow r' osd 'allow rwx pool=kube'
+
+ echo "${FUNCNAME[0]}: Create ceph-secret-user secret in namespace 'default'"
+ kube_key=$(sudo ceph auth get-key client.kube)
+ kubectl create secret generic ceph-secret-user --from-literal=key="$kube_key" --namespace=default --type=kubernetes.io/rbd
+ # A similar secret must be created in other namespaces that intend to access the ceph pool
+
+ # Per https://github.com/kubernetes/examples/blob/master/staging/persistent-volume-provisioning/README.md
+
+ echo "${FUNCNAME[0]}: Create andtest a persistentVolumeClaim"
+ cat <<EOF >/tmp/ceph-pvc.yaml
+{
+ "kind": "PersistentVolumeClaim",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "claim1",
+ "annotations": {
+ "volume.beta.kubernetes.io/storage-class": "slow"
+ }
+ },
+ "spec": {
+ "accessModes": [
+ "ReadWriteOnce"
+ ],
+ "resources": {
+ "requests": {
+ "storage": "3Gi"
+ }
+ }
+ }
+}
+EOF
+ kubectl create -f /tmp/ceph-pvc.yaml
+ while [[ "x$(kubectl get pvc -o jsonpath='{.status.phase}' claim1)" != "xBound" ]]; do
+ echo "${FUNCNAME[0]}: Waiting for pvc claim1 to be 'Bound'"
+ kubectl describe pvc
+ sleep 10
+ done
+ echo "${FUNCNAME[0]}: pvc claim1 successfully bound to $(kubectl get pvc -o jsonpath='{.spec.volumeName}' claim1)"
+ kubectl get pvc
+ kubectl delete pvc claim1
+ kubectl describe pods
+}
+
+if [[ "$1" .ne "" ]]; then
+ setup_ceph "$1" $2 $3 $4
+else
+ grep '#. ' $0
+fi
--- /dev/null
+#!/bin/bash
+# Copyright 2017 AT&T Intellectual Property, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#. What this is: script to setup a Ceph-based SDS (Software Defined Storage)
+#. service for a kubernetes cluster, using Helm as deployment tool.
+#. Prerequisites:
+#. - Ubuntu xenial server for master and agent nodes
+#. - key-based auth setup for ssh/scp between master and agent nodes
+#. - 192.168.0.0/16 should not be used on your server network interface subnets
+#. Usage:
+# Intended to be called from k8s-cluster.sh in this folder. To run directly:
+#. $ bash ceph-helm.sh "<nodes>" <cluster-net> <public-net> [ceph_dev]
+#. nodes: space-separated list of ceph node IPs
+#. cluster-net: CIDR of ceph cluster network e.g. 10.0.0.1/24
+#. public-net: CIDR of public network
+#. ceph_dev: disk to use for ceph. ***MUST NOT BE USED FOR ANY OTHER PURPOSE***
+#. if not provided, ceph data will be stored on osd nodes in /ceph
+#.
+#. Status: work in progress, incomplete
+#
#. $ bash k8s-cluster.sh master
#. $ bash k8s-cluster.sh agents "<nodes>"
#. nodes: space-separated list of ceph node IPs
-#. $ bash k8s-cluster.sh ceph "<nodes>" <cluster-net> <public-net> [ceph_dev]
+#. $ bash k8s-cluster.sh ceph "<nodes>" <cluster-net> <public-net> <ceph-mode> [ceph_dev]
#. nodes: space-separated list of ceph node IPs
#. cluster-net: CIDR of ceph cluster network e.g. 10.0.0.1/24
#. public-net: CIDR of public network
+#. ceph-mode: "helm" or "baremetal"
#. ceph_dev: disk to use for ceph. ***MUST NOT BE USED FOR ANY OTHER PURPOSE***
#. if not provided, ceph data will be stored on osd nodes in /ceph
#. $ bash k8s-cluster.sh helm
#. Setup helm as app kubernetes orchestration tool
#. $ bash k8s-cluster.sh demo
#. Install helm charts for mediawiki and dokuwiki
-#. $ bash k8s-cluster.sh all "<nodes>" <cluster-net> <public-net> [ceph_dev]
+#. $ bash k8s-cluster.sh all "<nodes>" <cluster-net> <public-net> <ceph-mode> [ceph_dev]
#. Runs all the steps above
#.
#. Status: work in progress, incomplete
echo "${FUNCNAME[0]}: Cluster is ready when all nodes in the output of 'kubectl get nodes' show as 'Ready'."
}
-function setup_ceph() {
- node_ips=$1
- cluster_net=$2
- public_net=$3
- ceph_dev=$4
- echo "${FUNCNAME[0]}: Deploying ceph-mon on localhost $HOSTNAME"
- echo "${FUNCNAME[0]}: Deploying ceph-osd on nodes $node_ips"
- echo "${FUNCNAME[0]}: Setting cluster-network=$cluster_net and public-network=$public_net"
- mon_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}')
- all_nodes="$mon_ip $node_ips"
- # Also caches the server fingerprints so ceph-deploy does not prompt the user
- # Note this loop may be partially redundant with the ceph-deploy steps below
- for node_ip in $all_nodes; do
- echo "${FUNCNAME[0]}: Install ntp and ceph on $node_ip"
- ssh -x -o StrictHostKeyChecking=no ubuntu@$node_ip <<EOF
-sudo timedatectl set-ntp no
-wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
-echo deb https://download.ceph.com/debian/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
-sudo apt update
-sudo apt-get install -y ntp ceph ceph-deploy
-EOF
- done
-
- # per http://docs.ceph.com/docs/master/start/quick-ceph-deploy/
- # also https://upcommons.upc.edu/bitstream/handle/2117/101816/Degree_Thesis_Nabil_El_Alami.pdf#vote +1
- echo "${FUNCNAME[0]}: Create ceph config folder ~/ceph-cluster"
- mkdir ~/ceph-cluster
- cd ~/ceph-cluster
-
- echo "${FUNCNAME[0]}: Create new cluster with $HOSTNAME as initial ceph-mon node"
- ceph-deploy new --cluster-network $cluster_net --public-network $public_net --no-ssh-copykey $HOSTNAME
- # Update conf per recommendations of http://docs.ceph.com/docs/jewel/rados/configuration/filesystem-recommendations/
- cat <<EOF >>ceph.conf
-osd max object name len = 256
-osd max object namespace len = 64
-EOF
- cat ceph.conf
-
- echo "${FUNCNAME[0]}: Deploy ceph packages on other nodes"
- ceph-deploy install $mon_ip $node_ips
-
- echo "${FUNCNAME[0]}: Deploy the initial monitor and gather the keys"
- ceph-deploy mon create-initial
-
- if [[ "x$ceph_dev" == "x" ]]; then
- n=1
- for node_ip in $node_ips; do
- echo "${FUNCNAME[0]}: Prepare ceph OSD on node $node_ip"
- echo "$node_ip ceph-osd$n" | sudo tee -a /etc/hosts
- # Using ceph-osd$n here avoids need for manual acceptance of the new server hash
- ssh -x -o StrictHostKeyChecking=no ubuntu@ceph-osd$n <<EOF
-echo "$node_ip ceph-osd$n" | sudo tee -a /etc/hosts
-sudo mkdir /ceph && sudo chown -R ceph:ceph /ceph
-EOF
- ceph-deploy osd prepare ceph-osd$n:/ceph
- ceph-deploy osd activate ceph-osd$n:/ceph
- ((n++))
- done
- else
- echo "${FUNCNAME[0]}: Deploy OSDs"
- for node_ip in $node_ips; do
- echo "${FUNCNAME[0]}: Create ceph osd on $node_ip using $ceph_dev"
- ceph-deploy osd create $node_ip:$ceph_dev
- done
- fi
-
- echo "${FUNCNAME[0]}: Copy the config file and admin key to the admin node and OSD nodes"
- ceph-deploy admin $mon_ip $node_ips
-
- echo "${FUNCNAME[0]}: Check the cluster health"
- sudo ceph health
- sudo ceph -s
-
- # per https://crondev.com/kubernetes-persistent-storage-ceph/ and https://github.com/kubernetes/kubernetes/issues/38923
- # rbd is not included in default kube-controller-manager... use attcomdev version
- sudo sed -i -- 's~gcr.io/google_containers/kube-controller-manager-amd64:.*~quay.io/attcomdev/kube-controller-manager:v1.7.3~' /etc/kubernetes/manifests/kube-controller-manager.yaml
- if [[ $(sudo grep -c attcomdev/kube-controller-manager /etc/kubernetes/manifests/kube-controller-manager.yaml) == 0 ]]; then
- echo "${FUNCNAME[0]}: Problem patching /etc/kubernetes/manifests/kube-controller-manager.yaml... script update needed"
- exit 1
- fi
- mgr=$(kubectl get pods --all-namespaces | grep kube-controller-manager | awk '{print $4}')
- while [[ "$mgr" != "Running" ]]; do
- echo "${FUNCNAME[0]}: kube-controller-manager status is $mgr. Waiting 60 seconds for it to be 'Running'"
- sleep 60
- mgr=$(kubectl get pods --all-namespaces | grep kube-controller-manager | awk '{print $4}')
- done
- echo "${FUNCNAME[0]}: kube-controller-manager status is $mgr"
-
- echo "${FUNCNAME[0]}: Create Ceph admin secret"
- admin_key=$(sudo ceph auth get-key client.admin)
- kubectl create secret generic ceph-secret-admin --from-literal=key="$admin_key" --namespace=kube-system --type=kubernetes.io/rbd
-
- echo "${FUNCNAME[0]}: Create rdb storageClass 'slow'"
- cat <<EOF >/tmp/ceph-sc.yaml
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
- name: slow
-provisioner: kubernetes.io/rbd
-parameters:
- monitors: $mon_ip:6789
- adminId: admin
- adminSecretName: ceph-secret-admin
- adminSecretNamespace: "kube-system"
- pool: kube
- userId: kube
- userSecretName: ceph-secret-user
-EOF
- # TODO: find out where in the above ~/.kube folders became owned by root
- sudo chown -R ubuntu:ubuntu ~/.kube/*
- kubectl create -f /tmp/ceph-sc.yaml
-
- echo "${FUNCNAME[0]}: Create storage pool 'kube'"
- # https://github.com/kubernetes/examples/blob/master/staging/persistent-volume-provisioning/README.md method
- sudo ceph osd pool create kube 32 32
-
- echo "${FUNCNAME[0]}: Authorize client 'kube' access to pool 'kube'"
- sudo ceph auth get-or-create client.kube mon 'allow r' osd 'allow rwx pool=kube'
-
- echo "${FUNCNAME[0]}: Create ceph-secret-user secret in namespace 'default'"
- kube_key=$(sudo ceph auth get-key client.kube)
- kubectl create secret generic ceph-secret-user --from-literal=key="$kube_key" --namespace=default --type=kubernetes.io/rbd
- # A similar secret must be created in other namespaces that intend to access the ceph pool
-
- # Per https://github.com/kubernetes/examples/blob/master/staging/persistent-volume-provisioning/README.md
-
- echo "${FUNCNAME[0]}: Create andtest a persistentVolumeClaim"
- cat <<EOF >/tmp/ceph-pvc.yaml
-{
- "kind": "PersistentVolumeClaim",
- "apiVersion": "v1",
- "metadata": {
- "name": "claim1",
- "annotations": {
- "volume.beta.kubernetes.io/storage-class": "slow"
- }
- },
- "spec": {
- "accessModes": [
- "ReadWriteOnce"
- ],
- "resources": {
- "requests": {
- "storage": "3Gi"
- }
- }
- }
-}
-EOF
- kubectl create -f /tmp/ceph-pvc.yaml
- while [[ "x$(kubectl get pvc -o jsonpath='{.status.phase}' claim1)" != "xBound" ]]; do
- echo "${FUNCNAME[0]}: Waiting for pvc claim1 to be 'Bound'"
- kubectl describe pvc
- sleep 10
- done
- echo "${FUNCNAME[0]}: pvc claim1 successfully bound to $(kubectl get pvc -o jsonpath='{.spec.volumeName}' claim1)"
- kubectl get pvc
- kubectl delete pvc claim1
- kubectl describe pods
-}
-
function wait_for_service() {
echo "${FUNCNAME[0]}: Waiting for service $1 to be available"
pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }")
# e.g. helm install stable/dokuwiki
}
+function setup_ceph() {
+ if [[ "$4" .eq "helm" ]]; then
+ source ./ceph-helm.sh "$1" $2 $3 $4
+ else
+ source ./ceph-baremetal.sh "$1" $2 $3 $4
+ fi
+}
+
export WORK_DIR=$(pwd)
case "$1" in
master)
setup_k8s_agents "$2"
;;
ceph)
- setup_ceph "$2" $3 $4 $5
+ setup_ceph "$2" $3 $4 $5 $6
;;
helm)
setup_helm
all)
setup_k8s_master
setup_k8s_agents "$2"
- setup_ceph "$2" $3 $4 $5
+ setup_ceph "$2" $3 $4 $5 $6
setup_helm
demo_chart dokuwiki
;;
Code Review / models.git / commitdiff