--- /dev/null
+#!/bin/bash
+#
+# Copyright 2018 Tieto
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script for automated deployment of ONAP on top of OPNFV Fuel/MCP installation
+# In the future both OOM and heat install methods should be supported.
+# At the beginning OOM will be used for simplification.
+
+# TODO:
+# Configure ONAP to be able to control underlying OpenStack
+
+# Configuration to be passed to ci/deploy-onap.sh
+export SSH_USER="ubuntu"
+export SSH_IDENTITY="/root/.ssh/onap_key"
+
+# Use default values if compute configuration was not set by FUEL installer
+AUTO_INSTALL_DIR=${AUTO_INSTALL_DIR:-"."}
+AUTO_IMAGE_DIR="${AUTO_INSTALL_DIR}/images"
+CMP_COUNT=${CMP_COUNT:-2} # number of compute nodes
+CMP_MIN_MEM=${CMP_MIN_MEM:-64000} # MB RAM of the weakest compute node
+CMP_MIN_CPUS=${CMP_MIN_CPUS:-36} # CPU count of the weakest compute node
+# size of storage for instances
+CMP_STORAGE_TOTAL=${CMP_STORAGE_TOTAL:-$((80*$CMP_COUNT))}
+VM_COUNT=${VM_COUNT:-2} # number of VMs available for k8s cluster
+
+#
+# Functions
+#
+# function minimum accepts two numbers and prints smaller one
+function minimum(){
+ echo $(($1<$2?$1:$2))
+}
+
+# function remove_openstack_setup removes OS configuration performed by this
+# script; So previously created configuration and deployed VMs will be
+# removed before new ONAP deployment will be started.
+function remove_openstack_setup(){
+ # flavor is created 1st but removed last, so...
+ if ( ! openstack flavor list | grep 'onap.large' &> /dev/null ) ; then
+ #...no flavor means nothing to be removed
+ return
+ fi
+ echo -e "\nRemoving ONAP specific OpenStack configuration"
+ for a in $(openstack server list --name onap_vm -f value -c ID) ; do
+ openstack server delete $a
+ done
+ RULES=$(openstack security group rule list onap_security_group -f value -c ID)
+ for a in $RULES; do
+ openstack security group rule delete $a
+ done
+ openstack security group delete onap_security_group
+ for a in $(openstack floating ip list -f value -c ID) ; do
+ openstack floating ip delete $a
+ done
+ PORTS=$(openstack port list --network onap_private_network -f value -c ID)
+ for a in $PORTS ; do
+ openstack router remove port onap_router $a
+ done
+ PORTS=$(openstack port list --network onap_private_network -f value -c ID)
+ for a in $PORTS ; do
+ openstack port delete $a
+ done
+ openstack router delete onap_router
+ openstack subnet delete onap_private_subnet
+ openstack network delete onap_private_network
+ openstack image delete xenial
+ rm -rf $AUTO_IMAGE_DIR
+ openstack keypair delete onap_key
+ rm $SSH_IDENTITY
+ openstack flavor delete onap.large
+ echo
+}
+
+#
+# Script Main
+#
+
+# remove OpenStack configuration if it exists
+remove_openstack_setup
+
+echo -e "\nOpenStack configuration\n"
+
+# Calculate VM resources, so that flavor can be created
+echo "Configuration of compute node:"
+echo "Number of computes: CMP_COUNT=$CMP_COUNT"
+echo "Minimal RAM: CMP_MIN_MEM=$CMP_MIN_MEM"
+echo "Minimal CPUs count: CMP_MIN_CPUS=$CMP_MIN_CPUS"
+echo "Storage for instances: CMP_STORAGE_TOTAL=$CMP_STORAGE_TOTAL"
+echo "Number of VMs: VM_COUNT=$VM_COUNT"
+# Calculate VM parameters; there will be up to 1 VM per Compute node
+# to maximize resources available for VMs
+PER=90 # % of compute resources will be consumed by VMs
+VM_DISK_MAX=100 # GB - max VM disk size
+VM_MEM_MAX=81920 # MB - max VM RAM size
+VM_CPUS_MAX=56 # max count of VM CPUs
+VM_MEM=$(minimum $(($CMP_MIN_MEM*$CMP_COUNT*$PER/100/$VM_COUNT)) $VM_MEM_MAX)
+VM_CPUS=$(minimum $(($CMP_MIN_CPUS*$CMP_COUNT*$PER/100/$VM_COUNT)) $VM_CPUS_MAX)
+VM_DISK=$(minimum $(($CMP_STORAGE_TOTAL*$PER/100/$VM_COUNT)) $VM_DISK_MAX)
+
+echo -e "\nFlavor configuration:"
+echo "CPUs : $VM_CPUS"
+echo "RAM [MB] : $VM_MEM"
+echo "DISK [GB] : $VM_DISK"
+
+# Create onap flavor
+openstack flavor create --ram $VM_MEM --vcpus $VM_CPUS --disk $VM_DISK \
+ onap.large
+
+# Generate a keypair and store private key
+openstack keypair create onap_key > $SSH_IDENTITY
+chmod 600 $SSH_IDENTITY
+
+# Download and import VM image(s)
+mkdir $AUTO_IMAGE_DIR
+wget -P $AUTO_IMAGE_DIR https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
+openstack image create --disk-format qcow2 --container-format bare --public \
+ --file $AUTO_IMAGE_DIR/xenial-server-cloudimg-amd64-disk1.img xenial
+
+# Modify quotas (add 10% to required VM resources)
+openstack quota set --ram $(($VM_MEM*$VM_COUNT*110/100)) admin
+openstack quota set --cores $(($VM_CPUS*$VM_COUNT*110/100)) admin
+
+# Configure networking with DNS for access to the internet
+openstack network create onap_private_network --provider-network-type vxlan
+openstack subnet create onap_private_subnet --network onap_private_network \
+ --subnet-range 192.168.33.0/24 --ip-version 4 --dhcp --dns-nameserver "8.8.8.8"
+openstack router create onap_router
+openstack router add subnet onap_router onap_private_subnet
+openstack router set onap_router --external-gateway floating_net
+
+# Allow selected ports and protocols
+openstack security group create onap_security_group
+openstack security group rule create --protocol icmp onap_security_group
+openstack security group rule create --proto tcp \
+ --dst-port 22:22 onap_security_group
+openstack security group rule create --proto tcp \
+ --dst-port 8080:8080 onap_security_group # rancher
+openstack security group rule create --proto tcp \
+ --dst-port 8078:8078 onap_security_group # horizon
+openstack security group rule create --proto tcp \
+ --dst-port 8879:8879 onap_security_group # helm
+openstack security group rule create --proto tcp \
+ --dst-port 80:80 onap_security_group
+openstack security group rule create --proto tcp \
+ --dst-port 443:443 onap_security_group
+
+# Allow communication between k8s cluster nodes
+PUBLIC_NET=`openstack subnet list --name floating_subnet -f value -c Subnet`
+openstack security group rule create --remote-ip $PUBLIC_NET --proto tcp \
+ --dst-port 1:65535 onap_security_group
+openstack security group rule create --remote-ip $PUBLIC_NET --proto udp \
+ --dst-port 1:65535 onap_security_group
+
+# Create VMs and assign floating IPs to them
+VM_ITER=1
+while [ $VM_ITER -le $VM_COUNT ] ; do
+ openstack floating ip create floating_net
+ VM_NAME[$VM_ITER]="onap_vm${VM_ITER}"
+ VM_IP[$VM_ITER]=$(openstack floating ip list -c "Floating IP Address" \
+ -c "Port" -f value | grep None | cut -f1 -d " " | head -n1)
+ openstack server create --flavor onap.large --image xenial \
+ --nic net-id=onap_private_network --security-group onap_security_group \
+ --key-name onap_key ${VM_NAME[$VM_ITER]}
+ sleep 5 # wait for VM init before floating IP can be assigned
+ openstack server add floating ip ${VM_NAME[$VM_ITER]} ${VM_IP[$VM_ITER]}
+ VM_ITER=$(($VM_ITER+1))
+done
+
+openstack server list
+
+echo "Waiting for VMs to start up for 5 minutes at $(date)"
+sleep 5m
+
+openstack server list
+
+# Start ONAP installation
+DATE_START=$(date)
+echo -e "\nONAP Installation Started at $DATE_START\n"
+$AUTO_INSTALL_DIR/ci/deploy-onap.sh ${VM_IP[@]}
+echo -e "\nONAP Installation Started at $DATE_START"
+echo -e "ONAP Installation Finished at $(date)\n"
# NOTE: Following must be assured for all MASTER and SLAVE servers before
# onap-deploy.sh execution:
# 1) ssh access without a password
-# 2) an "opnfv" user account with password-less sudo access must be
-# available
+# 2) an user account with password-less sudo access must be
+# available - default user is "opnfv"
#
# Configuration
MASTER=$1
SERVERS=$*
+shift
+SLAVES=$*
-BRANCH='master'
+BRANCH='beijing'
ENVIRON='onap'
+SSH_USER=${SSH_USER:-"opnfv"}
+SSH_OPTIONS='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+# by defalult install full ONAP installation
+ONAP_COMPONENT_DISABLE=${ONAP_COMPONENT_DISABLE:-""}
+# example of minimal ONAP installation
+#ONAP_COMPONENT_DISABLE="clamp cli consul dcaegen2 esr log msb multicloud nbi oof policy uui vfc vnfsdk"
+
+# use identity file from the environment SSH_IDENTITY
+if [ -n "$SSH_IDENTITY" ] ; then
+ SSH_OPTIONS="-i $SSH_IDENTITY $SSH_OPTIONS"
+fi
+
#
# Installation
#
for MACHINE in $SERVERS;
do
-ssh opnfv@"$MACHINE" "bash -s" <<DOCKERINSTALL &
- sudo su
- apt-get update
+ssh $SSH_OPTIONS $SSH_USER@"$MACHINE" "bash -s" <<DOCKERINSTALL &
+ sudo -i
+ sysctl -w vm.max_map_count=262144
+ apt-get update -y
curl https://releases.rancher.com/install-docker/$DOCKER_VERSION.sh | sh
mkdir -p /etc/systemd/system/docker.service.d/
echo "INSTALLING RANCHER ON MASTER"
echo "$MASTER"
-ssh opnfv@"$MASTER" "bash -s" <<RANCHERINSTALL &
-sudo su
-apt install jq -y
+ssh $SSH_OPTIONS $SSH_USER@"$MASTER" "bash -s" <<RANCHERINSTALL
+sudo -i
+echo "INSTALL STARTS"
+apt-get install -y jq make htop
echo "Waiting for 30 seconds at \$(date)"
sleep 30
docker login -u docker -p docker nexus3.onap.org:10001
-echo "INSTALL STARTS"
-apt-get install make -y
-
docker run -d --restart=unless-stopped -p 8080:8080\
--name rancher_server rancher/server:v$RANCHER_VERSION
curl -LO https://storage.googleapis.com/kubernetes-release/\
service nfs-kernel-server restart
echo "Waiting 10 minutes for Rancher to setup at \$(date)"
-sleep 600
+sleep 10m
echo "Installing RANCHER CLI, KUBERNETES ENV on RANCHER"
wget https://github.com/rancher/cli/releases/download/v${RANCHER_CLI_VER}-rc2\
/rancher-linux-amd64-v${RANCHER_CLI_VER}-rc2.tar.gz
./rancher env create -t kubernetes $ENVIRON > kube_env_id.json
PROJECT_ID=\$(<kube_env_id.json)
echo "env id: \$PROJECT_ID"
+
+echo "Waiting for ${ENVIRON} creation - 1 min at \$(date)"
+sleep 1m
+
export RANCHER_HOST_URL=http://${MASTER}:8080/v1/projects/\$PROJECT_ID
echo "you should see an additional kubernetes environment"
./rancher env ls
"http://$MASTER:8080/v1/projects/\$PROJECT_ID/registrationtokens"\`
echo "REG_URL_RESPONSE: \$REG_URL_RESPONSE"
echo "Waiting for the server to finish url configuration - 1 min at \$(date)"
-sleep 60
+sleep 1m
# see registrationUrl in
REGISTRATION_TOKENS=\`curl http://$MASTER:8080/v2-beta/registrationtokens\`
echo "REGISTRATION_TOKENS: \$REGISTRATION_TOKENS"
\$REGISTRATION_DOCKER\
\$RANCHER_URL/v1/scripts/\$REGISTRATION_TOKEN"\
> /tmp/rancher_register_host
-chown opnfv /tmp/rancher_register_host
+chown $SSH_USER /tmp/rancher_register_host
RANCHERINSTALL
-wait
echo "REGISTER TOKEN"
-HOSTREGTOKEN=$(ssh opnfv@"$MASTER" cat /tmp/rancher_register_host)
+HOSTREGTOKEN=$(ssh $SSH_OPTIONS $SSH_USER@"$MASTER" cat /tmp/rancher_register_host)
echo "$HOSTREGTOKEN"
echo "REGISTERING HOSTS WITH RANCHER ENVIRONMENT '$ENVIRON'"
for MACHINE in $SERVERS;
do
-ssh opnfv@"$MACHINE" "bash -s" <<REGISTERHOST &
- sudo su
+ssh $SSH_OPTIONS $SSH_USER@"$MACHINE" "bash -s" <<REGISTERHOST &
+ sudo -i
$HOSTREGTOKEN
sleep 5
echo "Host $MACHINE waiting for host registration 5 min at \$(date)"
- sleep 300
+ sleep 5m
REGISTERHOST
done
wait
+echo "CONFIGURING NFS ON SLAVES"
+echo "$SLAVES"
+
+for SLAVE in $SLAVES;
+do
+ssh $SSH_OPTIONS $SSH_USER@"$SLAVE" "bash -s" <<CONFIGURENFS &
+ sudo -i
+ apt-get install nfs-common -y
+ mkdir /dockerdata-nfs
+ chmod 777 /dockerdata-nfs
+ echo "$MASTER:/dockerdata-nfs /dockerdata-nfs nfs auto 0 0" >> /etc/fstab
+ mount -a
+ mount | grep dockerdata-nfs
+CONFIGURENFS
+done
+wait
+
echo "DEPLOYING OOM ON RANCHER WITH MASTER"
echo "$MASTER"
-ssh opnfv@"$MASTER" "bash -s" <<OOMDEPLOY &
-sudo su
-sysctl -w vm.max_map_count=262144
+ssh $SSH_OPTIONS $SSH_USER@"$MASTER" "bash -s" <<OOMDEPLOY
+sudo -i
rm -rf oom
echo "pulling new oom"
git clone -b $BRANCH http://gerrit.onap.org/r/oom
./prepull_docker.sh
echo "starting onap pods"
cd oom/kubernetes/
+
+# Disable ONAP components
+if [ -n "$ONAP_COMPONENT_DISABLE" ] ; then
+ echo -n "Disable following ONAP components:"
+ for COMPONENT in $ONAP_COMPONENT_DISABLE; do
+ echo -n " \$COMPONENT"
+ sed -i '/^'\${COMPONENT}':$/!b;n;s/enabled: *true/enabled: false/' onap/values.yaml
+ done
+ echo
+fi
+
helm init --upgrade
-helm serve &
+# run helm server on the background and detached from current shell
+nohup helm serve 0<&- &>/dev/null &
echo "Waiting for helm setup for 5 min at \$(date)"
-sleep 300
+sleep 5m
helm version
helm repo add local http://127.0.0.1:8879
helm repo list
cd ../../
echo "Waiting for all pods to be up for 15-80 min at \$(date)"
-FAILED_PODS_LIMIT=0
-MAX_WAIT_PERIODS=480 # 120 MIN
+echo "Ignore failure of sdnc-ansible-server, see SDNC-443"
+TMP_POD_LIST='/tmp/onap_pod_list.txt'
+function get_onap_pods() {
+ kubectl get pods --namespace $ENVIRON > \$TMP_POD_LIST
+ return \$(cat \$TMP_POD_LIST | wc -l)
+}
+FAILED_PODS_LIMIT=1 # maximal number of falied ONAP PODs
+ALL_PODS_LIMIT=20 # minimum ONAP PODs to be up & running
+MAX_WAIT_PERIODS=500 # over 2 hours
COUNTER=0
-PENDING_PODS=0
-while [ \$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l) \
--gt \$FAILED_PODS_LIMIT ]; do
- PENDING=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l)
- PENDING_PODS=\$PENDING
+get_onap_pods
+ALL_PODS=\$?
+PENDING=\$(grep -E '0/|1/2' \$TMP_POD_LIST | wc -l)
+while [ \$PENDING -gt \$FAILED_PODS_LIMIT -o \$ALL_PODS -lt \$ALL_PODS_LIMIT ]; do
+ # print header every 20th lines
+ if [ \$COUNTER -eq \$((\$COUNTER/20*20)) ] ; then
+ printf "%-3s %-29s %-3s/%s\n" "Nr." "Datetime of check" "Err" "Total PODs"
+ fi
+ COUNTER=\$((\$COUNTER+1))
+ printf "%3s %-29s %3s/%-3s\n" \$COUNTER "\$(date)" \$PENDING \$ALL_PODS
sleep 15
- LIST_PENDING=\$(kubectl get pods --all-namespaces -o wide | grep -E '0/|1/2' )
- echo "\${LIST_PENDING}"
- echo "\${PENDING} pending > \${FAILED_PODS_LIMIT} at the \${COUNTER}th"\
- " 15 sec interval out of \${MAX_WAIT_PERIODS}"
- echo ""
- COUNTER=\$((\$COUNTER + 1 ))
if [ "\$MAX_WAIT_PERIODS" -eq \$COUNTER ]; then
FAILED_PODS_LIMIT=800
+ ALL_PODS_LIMIT=0
fi
+ get_onap_pods
+ ALL_PODS=\$?
+ PENDING=\$(grep -E '0/|1/2' \$TMP_POD_LIST | wc -l)
done
echo "Report on non-running containers"
-PENDING=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2')
-PENDING_COUNT=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l)
-PENDING_COUNT_AAI=\$(kubectl get pods -n $ENVIRON | grep aai- \
-| grep -E '0/|1/2' | wc -l)
-
-echo "Check filebeat 2/2 count for ELK stack logging consumption"
-FILEBEAT=\$(kubectl get pods --all-namespaces -a | grep 2/)
-echo "\${FILEBEAT}"
+get_onap_pods
+grep -E '0/|1/2' \$TMP_POD_LIST
+echo
+
echo "sleep 5 min - to allow rest frameworks to finish at \$(date)"
-sleep 300
-echo "List of ONAP Modules"
-LIST_ALL=\$(kubectl get pods --all-namespaces -a --show-all )
-echo "\${LIST_ALL}"
+sleep 5m
echo "run healthcheck 2 times to warm caches and frameworks"\
"so rest endpoints report properly - see OOM-447"
# OOM-484 - robot scripts moved
cd oom/kubernetes/robot
-echo "run healthcheck prep 1"
+echo -e "\nrun healthcheck prep 1"
# OOM-722 adds namespace parameter
-if [ "$BRANCH" == "amsterdam" ]; then
- ./ete-k8s.sh health > ~/health1.out
-else
- ./ete-k8s.sh $ENVIRON health > ~/health1.out
-fi
+./ete-k8s.sh $ENVIRON health > ~/health1.out
echo "sleep 5 min at \$(date)"
-sleep 300
+sleep 5m
+
echo "run healthcheck prep 2"
-if [ "$BRANCH" == "amsterdam" ]; then
- ./ete-k8s.sh health > ~/health2.out
-else
- ./ete-k8s.sh $ENVIRON health > ~/health2.out
-fi
+./ete-k8s.sh $ENVIRON health > ~/health2.out
+
echo "run healthcheck for real - wait a further 5 min at \$(date)"
-sleep 300
-if [ "$BRANCH" == "amsterdam" ]; then
- ./ete-k8s.sh health
-else
- ./ete-k8s.sh $ENVIRON health
-fi
+sleep 5m
+./ete-k8s.sh $ENVIRON health
OOMDEPLOY
-wait
-echo "Finished install, ruturned from Master"
+
+echo "Finished install, ruturned from Master at $(date)"
exit 0
Code Review / auto.git / commitdiff