+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 NEC Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import argparse
-import json
-import os
-
-import novaclient.client as novaclient
-
-
-nova_api_version = '2.11'
-
-def enable_compute_host(hostname):
- self.nova = novaclient.Client(self.nova_api_version,
- os.environ['OS_USERNAME'],
- os.environ['OS_PASSWORD'],
- os.environ['OS_TENANT_NAME'],
- os.environ['OS_AUTH_URL'],
- connection_pool=True)
- opts = {'all_tenants': True, 'host': hostname}
- for server in self.nova.servers.list(detailed=False, search_opts=opts):
- self.nova.servers.reset_state(server, 'active')
- self.nova.services.force_down(hostname, 'nova-compute', False)
-
-
-def get_args():
- parser = argparse.ArgumentParser(description='Doctor Test Cleaner')
- parser.add_argument('hostname', metavar='HOSTNAME', type=str, nargs='?',
- help='a hostname to be re-enable')
- return parser.parse_args()
-
-
-def main():
- args = get_args()
- enable_compute_host(args.hostname)
-
-
-if __name__ == '__main__':
- main()
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 NEC Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import argparse
-from flask import Flask
-from flask import request
-import json
-import logger as doctor_log
-import time
-
-LOG = doctor_log.Logger('doctor_consumer').getLogger()
-
-
-app = Flask(__name__)
-
-
-@app.route('/failure', methods=['POST'])
-def event_posted():
- LOG.info('doctor consumer notified at %s' % time.time())
- LOG.info('received data = %s' % request.data)
- d = json.loads(request.data)
- return "OK"
-
-
-def get_args():
- parser = argparse.ArgumentParser(description='Doctor Sample Consumer')
- parser.add_argument('port', metavar='PORT', type=int, nargs='?',
- help='the port for consumer')
- return parser.parse_args()
-
-
-def main():
- args = get_args()
- app.run(host="0.0.0.0", port=args.port)
-
-
-if __name__ == '__main__':
- main()
+++ /dev/null
-#!/bin/bash
-
-# Test if the named environment variable is set and not zero length
-# is_set env-var
-function is_set {
- local var=\$"$1"
- eval "[[ -n \"$var\" ]]"
-}
-
-# Prints backtrace info
-# filename:lineno:function
-# backtrace level
-function backtrace {
- local level=$1
- local deep
- deep=$((${#BASH_SOURCE[@]} - 1))
- echo "[Call Trace]"
- while [ $level -le $deep ]; do
- echo "${BASH_SOURCE[$deep]}:${BASH_LINENO[$deep-1]}:${FUNCNAME[$deep-1]}"
- deep=$((deep - 1))
- done
-}
-
-# Prints line number and "message" in error format
-# err $LINENO "message"
-function err {
- local exitcode=$?
- local xtrace
- xtrace=$(set +o | grep xtrace)
- set +o xtrace
- local msg="[ERROR] ${BASH_SOURCE[2]}:$1 $2"
- echo $msg 1>&2;
- if [[ -n ${LOGDIR} ]]; then
- echo $msg >> "${LOGDIR}/error.log"
- fi
- $xtrace
- return $exitcode
-}
-
-# Prints line number and "message" then exits
-# die $LINENO "message"
-function die {
- local exitcode=$?
- local xtrace
- xtrace=$(set +o | grep xtrace)
- set +o xtrace
- local line=$1; shift
- if [ $exitcode == 0 ]; then
- exitcode=1
- fi
- backtrace 2
- err $line "$*"
- # Give buffers a second to flush
- sleep 1
- $xtrace
- exit $exitcode
-}
-
-# Checks an environment variable is not set or has length 0 OR if the
-# exit code is non-zero and prints "message" and exits
-# NOTE: env-var is the variable name without a '$'
-# die_if_not_set $LINENO env-var "message"
-function die_if_not_set {
- local exitcode=$?
- local xtrace
- xtrace=$(set +o | grep xtrace)
- set +o xtrace
- local line=$1; shift
- local evar=$1; shift
- if ! is_set $evar || [ $exitcode != 0 ]; then
- die $line "$*"
- fi
- $xtrace
-}
-
-# Check the function is defined
-# die_if_not_defined $LINENO function-name "message"
-function die_if_not_defined {
- local xtrace
- xtrace=$(set +o | grep xtrace)
- set +o xtrace
- local line=$1; shift
- local func_name=$1; shift
- if ! declare -f "$func_name" > /dev/null; then
- die $line "$*"
- fi
- $xtrace
-}
-
-# Wait until the condition is met.
-# wait_until condition timeout interval
-function wait_until {
- local condition="$1"
- local timeout=$2
- local interval=$3
-
- while eval ${condition}
- do
- sleep ${interval}
- timeout=$((${timeout} - ${interval}))
- if [[ ${timeout} < 0 ]]; then
- err $LINENO "timed out ($condition)..."
- return 1
- fi
- done
-}
-
-# Print IP address of the first vNIC owned by specified VM via virsh
-# get_first_vnic_ip vm_name
-function get_first_vnic_ip {
- local vm_name=$1
-
- _vnic_mac=$(sudo virsh domiflist $vm_name | \
- sed -n -e 's/^.*\([0-9a-f]\{2\}\(:[0-9a-f]\{2\}\)\{5\}\).*$/\1/p' | \
- head -1)
- die_if_not_set $LINENO _vnic_mac
- _vnic_ip=$(arp -e | grep $_vnic_mac | awk '{print $1}')
- die_if_not_set $LINENO _vnic_ip
- echo $_vnic_ip
-}
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 NEC Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import argparse
-import collections
-from flask import Flask
-from flask import request
-import json
-import logger as doctor_log
-import threading
-import time
-
-from keystoneauth1 import session
-import novaclient.client as novaclient
-
-import doctor_tests.identity_auth
-
-LOG = doctor_log.Logger('doctor_inspector').getLogger()
-
-
-class ThreadedResetState(threading.Thread):
-
- def __init__(self, nova, state, server):
- threading.Thread.__init__(self)
- self.nova = nova
- self.state = state
- self.server = server
-
- def run(self):
- self.nova.servers.reset_state(self.server, self.state)
- LOG.info('doctor mark vm(%s) error at %s' % (self.server, time.time()))
-
-
-class DoctorInspectorSample(object):
-
- NOVA_API_VERSION = '2.34'
- NUMBER_OF_CLIENTS = 50
- # TODO(tojuvone): This could be enhanced in future with dynamic
- # reuse of self.novaclients when all threads in use and
- # self.NUMBER_OF_CLIENTS based on amount of cores or overriden by input
- # argument
-
- def __init__(self):
- self.servers = collections.defaultdict(list)
- self.novaclients = list()
- auth=identity_auth.get_identity_auth()
- sess=session.Session(auth=auth)
- # Pool of novaclients for redundant usage
- for i in range(self.NUMBER_OF_CLIENTS):
- self.novaclients.append(
- novaclient.Client(self.NOVA_API_VERSION, session=sess))
- # Normally we use this client for non redundant API calls
- self.nova=self.novaclients[0]
- self.nova.servers.list(detailed=False)
- self.init_servers_list()
-
- def init_servers_list(self):
- opts = {'all_tenants': True}
- servers=self.nova.servers.list(search_opts=opts)
- self.servers.clear()
- for server in servers:
- try:
- host=server.__dict__.get('OS-EXT-SRV-ATTR:host')
- self.servers[host].append(server)
- LOG.debug('get hostname=%s from server=%s' % (host, server))
- except Exception as e:
- LOG.error('can not get hostname from server=%s' % server)
-
- def disable_compute_host(self, hostname):
- threads = []
- if len(self.servers[hostname]) > self.NUMBER_OF_CLIENTS:
- # TODO(tojuvone): This could be enhanced in future with dynamic
- # reuse of self.novaclients when all threads in use
- LOG.error('%d servers in %s. Can handle only %d'%(
- self.servers[hostname], hostname, self.NUMBER_OF_CLIENTS))
- for nova, server in zip(self.novaclients, self.servers[hostname]):
- t = ThreadedResetState(nova, "error", server)
- t.start()
- threads.append(t)
- for t in threads:
- t.join()
- self.nova.services.force_down(hostname, 'nova-compute', True)
- LOG.info('doctor mark host(%s) down at %s' % (hostname, time.time()))
-
-
-app = Flask(__name__)
-inspector = DoctorInspectorSample()
-
-
-@app.route('/events', methods=['POST'])
-def event_posted():
- LOG.info('event posted at %s' % time.time())
- LOG.info('inspector = %s' % inspector)
- LOG.info('received data = %s' % request.data)
- d = json.loads(request.data)
- for event in d:
- hostname = event['details']['hostname']
- event_type = event['type']
- if event_type == 'compute.host.down':
- inspector.disable_compute_host(hostname)
- return "OK"
-
-
-def get_args():
- parser = argparse.ArgumentParser(description='Doctor Sample Inspector')
- parser.add_argument('port', metavar='PORT', type=int, nargs='?',
- help='a port for inspector')
- return parser.parse_args()
-
-
-def main():
- args = get_args()
- app.run(host='0.0.0.0', port=args.port)
-
-if __name__ == '__main__':
- main()
+++ /dev/null
-#!/bin/bash
-
-INSPECTOR_TYPE=${INSPECTOR_TYPE:-sample}
-
-function is_inspector_supported {
- local inspector="$1"
- [[ -f $TOP_DIR/lib/inspectors/$inspector ]]
-}
-
-function is_inspector {
- local inspector="$1"
- [[ $inspector == $INSPECTOR_TYPE ]]
-}
-
-function start_inspector {
- start_inspector_$INSPECTOR_TYPE
-}
-
-function stop_inspector {
- stop_inspector_$INSPECTOR_TYPE
-}
-
-function cleanup_inspector {
- cleanup_inspector_$INSPECTOR_TYPE
-}
-
-
-if ! is_inspector_supported $INSPECTOR_TYPE; then
- die $LINENO "INSPECTOR_TYPE=$INSPECTOR_TYPE is not supported."
-fi
-
-source $TOP_DIR/lib/inspectors/$INSPECTOR_TYPE
+++ /dev/null
-#!/bin/bash
-
-function _congress_add_rule {
- name=$1
- policy=$2
- rule=$3
-
- if ! openstack congress policy rule list $policy | grep -q -e "// Name: $name$" ; then
- openstack congress policy rule create --name $name $policy "$rule"
- fi
-}
-
-function _congress_del_rule {
- name=$1
- policy=$2
-
- if openstack congress policy rule list $policy | grep -q -e "^// Name: $name$" ; then
- openstack congress policy rule delete $policy $name
- fi
-}
-
-function _congress_add_rules {
- _congress_add_rule host_down classification \
- 'host_down(host) :-
- doctor:events(hostname=host, type="compute.host.down", status="down")'
-
- _congress_add_rule active_instance_in_host classification \
- 'active_instance_in_host(vmid, host) :-
- nova:servers(id=vmid, host_name=host, status="ACTIVE")'
-
- _congress_add_rule host_force_down classification \
- 'execute[nova:services.force_down(host, "nova-compute", "True")] :-
- host_down(host)'
-
- _congress_add_rule error_vm_states classification \
- 'execute[nova:servers.reset_state(vmid, "error")] :-
- host_down(host),
- active_instance_in_host(vmid, host)'
-}
-
-function start_inspector_congress {
- nova_api_min_version="2.11"
- nova_api_version=$(openstack congress datasource list | \
- grep nova | grep -Po "(?<='api_version': ')[^']*")
- [[ -z $nova_api_version ]] && nova_api_version="2.0"
- if [[ "$nova_api_version" < "$nova_api_min_version" ]]; then
- echo "ERROR: Congress Nova datasource API version < $nova_api_min_version ($nova_api_version)"
- exit 1
- fi
- openstack congress driver list | grep -q " doctor "
- openstack congress datasource list | grep -q " doctor " || {
- openstack congress datasource create doctor doctor
- }
- _congress_add_rules
-
-}
-
-function stop_inspector_congress {
- _congress_del_rule host_force_down classification
- _congress_del_rule error_vm_states classification
- _congress_del_rule active_instance_in_host classification
- _congress_del_rule host_down classification
-
-}
-
-function cleanup_inspector_congress {
- # Noop
- return
-}
+++ /dev/null
-#!/bin/bash
-
-INSPECTOR_PORT=${INSPECTOR_PORT:-12345}
-
-function start_inspector_sample {
- pgrep -f "python inspector.py" && return 0
- python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1 &
-}
-
-function stop_inspector_sample {
- pgrep -f "python inspector.py" || return 0
- kill $(pgrep -f "python inspector.py")
-}
-
-function cleanup_inspector_sample {
- # Noop
- return
-}
+++ /dev/null
-#!/bin/bash
-
-INSTALLER_TYPE=${INSTALLER_TYPE:-local}
-ssh_opts="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
-
-function is_installer_supported {
- local installer="$1"
- [[ -f $TOP_DIR/lib/installers/$installer ]]
-}
-
-function is_installer {
- local installer="$1"
- [[ $installer == $INSTALLER_TYPE ]]
-}
-
-function validate_installer_lib {
- local xtrace
- xtrace=$(set +o | grep xtrace)
- set +o xtrace
- for p in COMPUTE_USER ssh_opts_cpu
- do
- die_if_not_set $LINENO $p \
- "Parameter $p for $INSTALLER_TYPE is missing."
- done
- for f in setup_installer get_compute_ip_from_hostname cleanup_installer
- do
- die_if_not_defined $LINENO $f \
- "Mandatory function ${f}() for $INSTALLER_TYPE is missing."
- done
- $xtrace
-}
-
-if ! is_installer_supported $INSTALLER_TYPE; then
- die $LINENO "INSTALLER_TYPE=$INSTALLER_TYPE is not supported."
-fi
-
-source $TOP_DIR/lib/installers/$INSTALLER_TYPE
-
-validate_installer_lib
+++ /dev/null
-#!/bin/bash
-
-COMPUTE_USER=${COMPUTE_USER:-heat-admin}
-ssh_opts_cpu="$ssh_opts -i instack_key"
-
-function get_installer_ip {
- is_set INSTALLER_IP && return
- INSTALLER_IP=$(get_first_vnic_ip undercloud)
-}
-
-function installer_get_ssh_keys {
- sudo scp $ssh_opts "root@$INSTALLER_IP:/home/stack/.ssh/id_rsa" instack_key
- sudo chown $(whoami):$(whoami) instack_key
- chmod 400 instack_key
-}
-
-function get_controller_ips {
- is_set CONTROLLER_IPS && return
- get_installer_ip
- CONTROLLER_IPS=$(sudo ssh $ssh_opts $INSTALLER_IP \
- "source stackrc
- nova list | grep ' overcloud-controller-[0-9] ' | \
- sed -e 's/^.*ctlplane=//' -e 's/ *|\$//'")
- die_if_not_set $LINENO CONTROLLER_IPS "No controller IPs"
-}
-
-function installer_apply_patches {
- # TODO(r-mibu): fix the followings in upstream (apex)
- for node in $CONTROLLER_IPS;do
- echo "check controller configuration for doctor ($node)"
- ssh $ssh_opts_cpu "heat-admin@$node" '
- set -x
- date
- echo "### apply patches (installer=apex)"
-
- ep_conf=/etc/ceilometer/event_pipeline.yaml
- ep_entry="- notifier://?topic=alarm.all"
- if sudo grep -e "$ep_entry" $ep_conf; then
- echo "NOTE: ceilometer is configured as we needed"
- else
- echo "modify the ceilometer config"
- sudo sed -i -e "$ a \ \ \ \ \ \ \ \ \ \ $ep_entry # added by doctor script" $ep_conf
- sudo systemctl restart openstack-ceilometer-notification.service
- fi
-
- co_conf=/etc/congress/congress.conf
- co_conf_bak=/etc/congress/congress.conf.bak
- co_entry="congress.datasources.doctor_driver.DoctorDriver"
- if sudo grep -e "^drivers.*$co_entry" $co_conf; then
- echo "NOTE: congress is configured as we needed"
- else
- echo "modify the congress config"
- sudo cp $co_conf $co_conf_bak
- sudo sed -i -e "/^drivers/s/$/,$co_entry/" $co_conf
- sudo systemctl restart openstack-congress-server.service
- fi
- ' > installer_apply_patches_$node.log 2>&1
- done
-}
-
-function setup_installer {
- get_installer_ip
- installer_get_ssh_keys
- get_controller_ips
- installer_apply_patches
-
- # NOTE: while executing command as doctor user,
- # 'OS_PROJECT_ID' env parameter make openstack clients confused.
- unset OS_PROJECT_ID
-}
-
-function get_compute_ip_from_hostname {
- local compute_host=$1
-
- compute_host_in_undercloud=${compute_host%%.*}
- COMPUTE_IP=$(sudo ssh $ssh_opts $INSTALLER_IP \
- "source stackrc;
- nova show $compute_host_in_undercloud | \
- awk '/ ctlplane network /{print \$5}'")
- die_if_not_set $LINENO COMPUTE_IP "Could get IP address of $compute_host."
-}
-
-function installer_revert_patches {
- # TODO(r-mibu): fix the followings in upstream (apex)
- get_controller_ips
- for node in $CONTROLLER_IPS;do
- echo "restore controller configuration if touched ($node)"
- ssh $ssh_opts_cpu "heat-admin@$node" '
- set -x
- echo "### revert patches (installer=apex)"
- date
-
- co_conf=/etc/congress/congress.conf
- co_conf_bak=/etc/congress/congress.conf.bak
- if [ -e $co_conf_bak ]; then
- echo "restore the congress config"
- sudo mv $co_conf_bak $co_conf
- sudo systemctl restart openstack-congress-server.service
- fi
-
- ep_conf=/etc/ceilometer/event_pipeline.yaml
- if sudo grep -q -e "# added by doctor script" $ep_conf; then
- echo "modify the ceilometer config"
- sudo sed -ie "/# added by doctor script/d" $ep_conf
- sudo systemctl restart openstack-ceilometer-notification.service
- fi
- ' >> installer_apply_patches_$node.log 2>&1
- done
-}
-
-function cleanup_installer {
- installer_revert_patches
- return
-}
+++ /dev/null
-#!/bin/bash
-
-if [[ "congress " == "$INSPECTOR_TYPE" ]]; then
- die $LINENO "fuel does not support congress yet..."
-fi
-
-COMPUTE_USER=${COMPUTE_USER:-root}
-ssh_opts_cpu="$ssh_opts -i instack_key"
-
-function get_installer_ip {
- is_set INSTALLER_IP && return
- INSTALLER_IP=$(get_first_vnic_ip fuel-master)
-}
-
-function get_controller_ips {
- is_set CONTROLLER_IPS && return
- CONTROLLER_IPS=$(ssh $ssh_opts_cpu root@$INSTALLER_IP \
- "fuel node | grep controller | cut -d '|' -f 5|xargs")
- die_if_not_set $LINENO CONTROLLER_IPS "No controller IPs"
-}
-
-function installer_get_ssh_keys {
- if [[ -e instack_key ]]; then
- echo "test existing instack_key..."
- ssh $ssh_opts_cpu root@${INSTALLER_IP} "hostname" && return
- fi
- echo "getting instack_key from fuel node..."
- sshpass -p r00tme scp $ssh_opts root@${INSTALLER_IP}:.ssh/id_rsa instack_key
- sudo chown $(whoami):$(whoami) instack_key
- chmod 400 instack_key
-}
-
-function installer_apply_patches {
- # TODO(r-mibu): fix the followings in upstream (fuel)
- for node in $CONTROLLER_IPS;do
- echo "check controller configuration for doctor ($node)"
- ssh $ssh_opts_cpu "root@$node" '
- set -x
- date
- echo "### apply patches (installer=fuel)"
-
- ep_conf=/etc/ceilometer/event_pipeline.yaml
- entry="- notifier://?topic=alarm.all"
- if ! grep -q -e "$entry" $ep_conf; then
- echo "modify the ceilometer config"
- echo " $entry # added by doctor script" >> $ep_conf
- service ceilometer-agent-notification restart
- fi
-
- # TODO(r-mibu): enable this section once congress 4.0.0 is available
- if false; then
- co_conf=/etc/congress/congress.conf
- entry="congress.datasources.doctor_driver.DoctorDriver"
- if ! grep -q -e "^drivers.*$entry" $co_conf; then
- echo "modify the congress config"
- sed -i -e "/^drivers/s/$/,$entry # added by doctor script/" \
- $co_conf
- service congress-server restart
- fi
-
- rule="-m multiport -p tcp --dports 1789"
- rule+=" -m comment --comment doctor-congress"
- rule+=" -j ACCEPT"
- if ! iptables -C INPUT $rule; then
- iptables -I INPUT $rule
- fi
-
- ha_conf=/etc/haproxy/conf.d/180-congress.cfg
- if [[ ! -e $ha_conf ]]; then
- sed -e "1i# generated by doctor script" \
- -e "s/9696/1789/" \
- -e "s/neutron/congress/" \
- /etc/haproxy/conf.d/085-neutron.cfg > $ha_conf
- ip netns exec haproxy /usr/lib/ocf/resource.d/fuel/ns_haproxy restart
- fi
- fi
-
- np_conf=/etc/nova/policy.json
- if [ -e $np_conf ]; then
- entry="os_compute_api:servers:show:host_status"
- new="rule:admin_or_owner"
- np_backup="${np_conf}-doctor-saved"
- if grep -q "${entry}.*${new}" $np_conf; then
- echo "Not modifying nova policy"
- elif grep -q "${entry}" $np_conf; then
- echo "modify nova policy"
- cp $np_conf $np_backup
- oldline=$(grep "$entry" $np_conf)
- newline=$(echo "$oldline" | sed "s/rule.*\"/$new\"/")
- sed -i "s/$oldline/$newline/" $np_conf
- service nova-api restart
- else
- echo "add nova policy"
- cp $np_conf $np_backup
- sed -i "/{/a \ \"${entry}\": \"$new\"" $np_conf
- service nova-api restart
- fi
- else
- # policy.json does not exist in Ocata.
- echo "$np_conf does not exist. Creating new one."
- echo -e "{\n \"context_is_admin\": \"role:admin\"," > $np_conf
- echo -e " \"owner\" : \"user_id:%(user_id)s\"," >> $np_conf
- echo -e " \"admin_or_owner\": \"rule:context_is_admin or rule:owner\"," >> $np_conf
- echo -e " \"os_compute_api:servers:show:host_status\": \"rule:admin_or_owner\" \n}" >> $np_conf
- np_rm="${np_conf}-doctor-rm"
- cp $np_conf $np_rm
- service nova-api restart
- fi
- ' > installer_apply_patches_$node.log 2>&1
- done
-}
-
-function setup_installer {
- get_installer_ip
- installer_get_ssh_keys
- get_controller_ips
- installer_apply_patches
- #Might take a moment for nova-api to restart
- sleep 20
- if ! openstack flavor show $VM_FLAVOR ; then
- openstack flavor create --ram 512 --disk 1 $VM_FLAVOR \
- && touch created_doctor_flavor
- fi
-}
-
-function get_compute_ip_from_hostname {
- local compute_host=$1
-
- compute_host_in_undercloud=${compute_host%%.*}
- node_id=$(echo $compute_host_in_undercloud | cut -d "-" -f 2)
- COMPUTE_IP=$(sshpass -p r00tme ssh 2>/dev/null $ssh_opts root@${INSTALLER_IP} \
- "fuel node|awk -F '|' -v id=$node_id '{if (\$1 == id) print \$5}' |xargs")
- die_if_not_set $LINENO COMPUTE_IP "Could get IP address of $compute_host."
-}
-
-function installer_revert_patches {
- # TODO(r-mibu): fix the followings in upstream (fuel)
- get_controller_ips
- for node in $CONTROLLER_IPS;do
- echo "restore controller configuration if touched ($node)"
- ssh $ssh_opts_cpu "root@$node" '
- set -x
- echo "### revert patches (installer=fuel)"
- date
-
- # TODO(r-mibu): enable this section once congress 4.0.0 is available
- if false; then
- ha_conf=/etc/haproxy/conf.d/180-congress.cfg
- if grep -q "# generated by doctor script" $ha_conf; then
- rm -f $ha_conf
- ip netns exec haproxy /usr/lib/ocf/resource.d/fuel/ns_haproxy restart
- fi
-
- rule="-m multiport -p tcp --dports 1789"
- rule+=" -m comment --comment doctor-congress"
- rule+=" -j ACCEPT"
- if iptables -C INPUT $rule; then
- iptables -D INPUT $rule
- fi
-
- co_conf=/etc/congress/congress.conf
- entry="congress.datasources.doctor_driver.DoctorDriver"
- if grep -q -e "^drivers.*$entry # added by doctor script" $co_conf; then
- echo "modify the congress config"
- sed -i -e "/^drivers/s/^\(.*\),$entry/\1/" $co_conf
- service congress-server restart
- fi
- fi
-
- ep_conf=/etc/ceilometer/event_pipeline.yaml
- if grep -q "# added by doctor script" $ep_conf; then
- sed -ie "/# added by doctor script/d" $ep_conf
- service ceilometer-agent-notification restart
- fi
-
- np_conf=/etc/nova/policy.json
- np_backup="${np_conf}-doctor-saved"
- np_rm="${np_conf}-doctor-rm"
- if [ -e $np_backup ]; then
- cp -f $np_backup $np_conf
- rm $np_backup
- service nova-api restart
- elif [ -e $np_rm ]; then
- rm $np_conf
- rm $np_rm
- service nova-api restart
- fi
- ' >> installer_apply_patches_$node.log 2>&1
- done
-}
-
-function cleanup_installer {
- if [[ -e created_doctor_flavor ]] && openstack flavor show $VM_FLAVOR ; then
- openstack flavor delete $VM_FLAVOR && rm -f created_doctor_flavor
- fi
- installer_revert_patches
-}
+++ /dev/null
-#!/bin/bash
-
-COMPUTE_USER=${COMPUTE_USER:-$(whoami)}
-ssh_opts_cpu="$ssh_opts"
-
-function installer_get_ssh_keys {
- echo "INSTALLER_TYPE set to 'local'. Assuming SSH keys already exchanged with $COMPUTE_HOST"
- return
-}
-
-function installer_apply_patches {
- set -x
- date
- echo "### apply patches (installer=local)"
- np_conf=/etc/nova/policy.json
- if [ -e $np_conf ]; then
- entry="os_compute_api:servers:show:host_status"
- new="rule:admin_or_owner"
- np_backup="${np_conf}-doctor-saved"
- if grep -q "${entry}.*${new}" $np_conf; then
- echo "Not modifying nova policy"
- elif grep -q "${entry}" $np_conf; then
- echo "modify nova policy"
- cp $np_conf $np_backup
- oldline=$(grep "$entry" $np_conf)
- newline=$(echo "$oldline" | sed "s/rule.*\"/$new\"/")
- sed -i "s/$oldline/$newline/" $np_conf
- # TODO(umar): Update to systemd when screen is no more used for devstack
- screen -S stack -p n-api -X stuff "^C^M^[[A^M" # restart n-api service
- else
- echo "add nova policy"
- cp $np_conf $np_backup
- sed -i "/{/a \ \"${entry}\": \"$new\"" $np_conf
- screen -S stack -p n-api -X stuff "^C^M^[[A^M"
- fi
- else
- # policy.json does not exist in Ocata.
- echo "$np_conf does not exist. Creating a new one"
- echo -e '{\n "context_is_admin": "role:admin",' > $np_conf
- echo -e ' "owner" : "user_id:%(user_id)s",' >> $np_conf
- echo -e ' "admin_or_owner": "rule:context_is_admin or rule:owner",' >> $np_conf
- echo -e ' "os_compute_api:servers:show:host_status": "rule:admin_or_owner"\n}' >> $np_conf
- np_rm="${np_conf}-doctor-rm"
- cp $np_conf $np_rm
- screen -S stack -p n-api -X stuff "^C^M^[[A^M"
- fi
-
- return
-}
-
-function setup_installer {
- installer_get_ssh_keys
- installer_apply_patches
-}
-
-function get_compute_ip_from_hostname {
- local compute_host=$1
-
- if is_set COMPUTE_IP; then
- echo "Using pre-configured COMPUTE_IP=$COMPUTE_IP ..."
- return
- fi
- COMPUTE_IP=$(getent hosts "$compute_host" | awk '{ print $1 }')
- die_if_not_set $LINENO COMPUTE_IP \
- "Could not resolve $compute_host. Either manually set COMPUTE_IP or enable DNS resolution."
-}
-
-function cleanup_installer {
- set -x
- echo "### revert patches (installer=local)"
- date
-
- np_conf=/etc/nova/policy.json
- np_backup="${np_conf}-doctor-saved"
- np_rm="${np_conf}-doctor-rm"
- if [ -e $np_backup ]; then
- cp -f $np_backup $np_conf
- rm $np_backup
- screen -S stack -p n-api -X stuff "^C^M^[[A^M"
- elif [ -e $np_rm ]; then
- rm $np_conf
- rm $np_rm
- screen -S stack -p n-api -X stuff "^C^M^[[A^M"
- fi
-
- return
-}
+++ /dev/null
-#!/bin/bash
-
-MONITOR_TYPE=${MONITOR_TYPE:-sample}
-
-function is_monitor_supported {
- local monitor="$1"
- [[ -f $TOP_DIR/lib/monitors/$monitor/$monitor ]]
-}
-
-function is_monitor {
- local monitor="$1"
- [[ $monitor == $MONITOR_TYPE ]]
-}
-
-function start_monitor {
- start_monitor_$MONITOR_TYPE
-}
-
-function stop_monitor {
- stop_monitor_$MONITOR_TYPE
-}
-
-function cleanup_monitor {
- cleanup_monitor_$MONITOR_TYPE
-}
-
-if ! is_monitor_supported $MONITOR_TYPE; then
- die $LINENO "MONITOR_TYPE=$MONITOR_TYPE is not supported."
-fi
-
-source $TOP_DIR/lib/monitors/$MONITOR_TYPE/$MONITOR_TYPE
+++ /dev/null
-#!/bin/bash
-
-function start_monitor_collectd {
- ## CONTROL_IP is the IP of primary interface of control node i.e.
- ## eth0, eno1. It is used by collectd monitor to communicate with
- ## sample inspector.
- ## @TODO (umar) see if mgmt IP of control is a better option. Also
- ## primary interface may not be the right option
- CONTROL_IP="$(ip a | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | sed -n 1p)"
- #CONTROL_IP=192.168.98.6
-
- echo "
-Hostname \"$COMPUTE_HOST\"
-FQDNLookup false
-Interval 1
-MaxReadInterval 2
-
-<LoadPlugin python>
- Globals true
-</LoadPlugin>
-LoadPlugin ovs_events
-LoadPlugin logfile
-
-<Plugin logfile>
- File \"/var/log/collectd.log\"
- Timestamp true
- LogLevel \"info\"
-</Plugin>
-
-<Plugin python>
- ModulePath \"/home/$COMPUTE_USER\"
- LogTraces true
- Interactive false
- Import \"collectd_plugin\"
- <Module \"collectd_plugin\">
- control_ip \"$CONTROL_IP\"
- compute_ip \"$COMPUTE_IP\"
- compute_host \"$COMPUTE_HOST\"
- compute_user \"$COMPUTE_USER\"
- inspector_type \"$INSPECTOR_TYPE\"
- os_auth_url \"$OS_AUTH_URL\"
- os_username \"$OS_USERNAME\"
- os_password \"$OS_PASSWORD\"
- os_project_name \"$OS_PROJECT_NAME\"
- os_user_domain_name \"$OS_USER_DOMAIN_NAME\"
- os_user_domain_id \"$OS_USER_DOMAIN_ID\"
- os_project_domain_name \"$OS_PROJECT_DOMAIN_NAME\"
- os_project_domain_id \"$OS_PROJECT_DOMAIN_ID\"
- </Module>
-</Plugin>
-
-<Plugin ovs_events>
- Port 6640
- Socket \"/var/run/openvswitch/db.sock\"
- Interfaces \"@INTERFACE_NAME@\"
- SendNotification true
- DispatchValues false
-</Plugin>
-
-" > $TOP_DIR/lib/monitors/collectd.conf
-
- scp $ssh_opts_cpu $TOP_DIR/lib/monitors/collectd.conf $COMPUTE_USER@$COMPUTE_IP:
- ## @TODO (umar) Always assuming that the interface is assigned an IP if
- ## interface name is not provided. See if there is a better approach
- ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" "
- if [ -n \"$INTERFACE_NAME\" ]; then
- dev=$INTERFACE_NAME
- else
- dev=\$(sudo ip a | awk '/ $COMPUTE_IP\//{print \$NF}')
- fi
- sed -i -e \"s/@INTERFACE_NAME@/\$dev/\" collectd.conf
- collectd_conf=/opt/collectd/etc/collectd.conf
- if [ -e \$collectd_conf ]; then
- sudo cp \$collectd_conf \${collectd_conf}-doctor-saved
- else
- sudo touch \${collectd_conf}-doctor-created
- fi
- sudo mv collectd.conf /opt/collectd/etc/collectd.conf"
-
- scp $ssh_opts_cpu $TOP_DIR/lib/monitors/collectd/collectd_plugin.py $COMPUTE_USER@$COMPUTE_IP:collectd_plugin.py
- ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" "sudo pkill collectd
- sudo /opt/collectd/sbin/collectd"
-}
-
-function stop_monitor_collectd {
- ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'sudo pkill collectd'
-}
-
-function cleanup_monitor_collectd {
- ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" "
- collectd_conf=/opt/collectd/etc/collectd.conf
- if [ -e \"\${collectd_conf}-doctor-created\" ]; then
- sudo rm \"\${collectd_conf}-doctor-created\"
- sudo rm \$collectd_conf
- elif [ -e \"\${collectd_conf}-doctor-saved\" ]; then
- sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf
- sudo rm \"\${collectd_conf}-doctor-saved\"
- fi"
-
- rm $TOP_DIR/lib/monitors/collectd.conf
-}
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 NEC Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import collectd
-import sys
-from netifaces import interfaces, ifaddresses, AF_INET
-from datetime import datetime
-import json
-import requests
-import time
-from requests.exceptions import ConnectionError
-
-from keystoneauth1 import loading
-from keystoneauth1 import session
-from congressclient.v1 import client
-
-
-def write_debug(str_write, write_type, compute_user):
- file_name = ('/home/%s/monitor.log' % compute_user)
- file_tmp = open(file_name, write_type)
- file_tmp.write( "%s" % str_write)
- file_tmp.close()
-
-
-class DoctorMonitorCollectd(object):
- def __init__(self):
- self.control_ip = ''
- self.compute_user = ''
- self.compute_ip = ''
- self.host_name = ''
- self.inspector_type = ''
- self.inspector_url = ''
- self.os_auth_url = ''
- self.os_username = ''
- self.os_password = ''
- self.os_project_name = ''
- self.os_user_domain_name = ''
- self.os_user_domain_id = ''
- self.os_project_domain_name = ''
- self.os_project_domain_id = ''
- self.sess = ''
- self.auth = ''
- self.inspector_notified = 0
- self.start_notifications = 0
- self.monitor_type = 'sample'
-
- def config_func(self, config):
- for node in config.children:
- key = node.key.lower()
- val = node.values[0]
-
- if key == 'compute_host':
- self.host_name = val
- elif key == 'control_ip':
- self.control_ip = val
- elif key == 'compute_ip':
- self.compute_ip = val
- elif key == 'compute_user':
- self.compute_user = val
- elif key == 'inspector_type':
- self.inspector_type = val
- elif key == 'os_auth_url':
- self.os_auth_url = val
- elif key == 'os_username':
- self.os_username = val
- elif key == 'os_password':
- self.os_password = val
- elif key == 'os_project_name':
- self.os_project_name = val
- elif key == 'os_user_domain_name':
- self.os_user_domain_name = val
- elif key == 'os_user_domain_id':
- self.os_user_domain_id = val
- elif key == 'os_project_domain_name':
- self.os_project_domain_name = val
- elif key == 'os_project_domain_id':
- self.os_project_domain_id = val
- else:
- collectd.info('Unknown config key "%s"' % key)
-
- def init_collectd(self):
- write_debug("Compute node collectd monitor start at %s\n\n" % datetime.now().isoformat(), "w", self.compute_user)
-
- if self.inspector_type == 'sample':
- self.inspector_url = ('http://%s:12345/events' % self.control_ip)
- elif self.inspector_type == 'congress':
- loader = loading.get_plugin_loader('password')
- self.auth = loader.load_from_options(auth_url=self.os_auth_url,
- username=self.os_username,
- password=self.os_password,
- project_name=self.os_project_name,
- user_domain_name=self.os_user_domain_name,
- user_domain_id=self.os_user_domain_id,
- project_domain_name=self.os_project_domain_name,
- project_domain_id=self.os_project_domain_id)
- self.sess=session.Session(auth=self.auth)
- congress = client.Client(session=self.sess, service_type='policy')
- ds = congress.list_datasources()['results']
- doctor_ds = next((item for item in ds if item['driver'] == 'doctor'),
- None)
-
- congress_endpoint = congress.httpclient.get_endpoint(auth=self.auth)
- self.inspector_url = ('%s/v1/data-sources/%s/tables/events/rows' %
- (congress_endpoint, doctor_ds['id']))
- else:
- sys.exit()
- self.start_notifications = 1
-
-
- def notify_inspector(self):
- event_type = "compute.host.down"
- payload = [
- {
- 'id': ("monitor_%s_id1" % self.monitor_type),
- 'time': datetime.now().isoformat(),
- 'type': event_type,
- 'details': {
- 'hostname': self.host_name,
- 'status': 'down',
- 'monitor': ("monitor_%s" % self.monitor_type),
- 'monitor_event_id': ("monitor_%s_event1" % self.monitor_type)
- },
- },
- ]
- data = json.dumps(payload)
- self.inspector_notified = 1
-
- if self.inspector_type == 'sample':
- headers = {'content-type': 'application/json'}
- try:
- requests.post(self.inspector_url, data=data, headers=headers)
- except ConnectionError as err:
- print err
- elif self.inspector_type == 'congress':
- # TODO(umar) enhance for token expiry case
- headers = {
- 'Content-Type': 'application/json',
- 'Accept': 'application/json',
- 'X-Auth-Token': self.sess.get_token()
- }
- requests.put(self.inspector_url, data=data, headers=headers)
-
-
- def handle_notif(self, notification, data=None):
- if (notification.severity == collectd.NOTIF_FAILURE or
- notification.severity == collectd.NOTIF_WARNING):
- if (self.start_notifications == 1 and self.inspector_notified == 0):
- write_debug("Received down notification: doctor monitor detected at %s\n" % time.time(), "a", self.compute_user)
- self.notify_inspector()
-
- elif notification.severity == collectd.NOTIF_OKAY:
- collectd.info("Interface status: UP again %s\n" % time.time())
- else:
- collectd.info("Unknown notification severity %s\n" % notification.severity)
-
-
-monitor = DoctorMonitorCollectd()
-
-collectd.register_config(monitor.config_func)
-collectd.register_init(monitor.init_collectd)
-collectd.register_notification(monitor.handle_notif)
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 NEC Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import argparse
-from datetime import datetime
-import json
-import logger as doctor_log
-import requests
-import socket
-import time
-
-from keystoneauth1 import session
-from congressclient.v1 import client
-
-import identity_auth
-
-# NOTE: icmp message with all zero data (checksum = 0xf7ff)
-# see https://tools.ietf.org/html/rfc792
-ICMP_ECHO_MESSAGE = '\x08\x00\xf7\xff\x00\x00\x00\x00'
-
-SUPPORTED_INSPECTOR_TYPES = ['sample', 'congress']
-
-LOG = doctor_log.Logger('doctor_monitor').getLogger()
-
-
-class DoctorMonitorSample(object):
-
- interval = 0.1 # second
- timeout = 0.1 # second
- event_type = "compute.host.down"
-
- def __init__(self, args):
- if args.inspector_type not in SUPPORTED_INSPECTOR_TYPES:
- raise Exception("Inspector type '%s' not supported", args.inspector_type)
-
- self.hostname = args.hostname
- self.inspector_type = args.inspector_type
- self.ip_addr = args.ip or socket.gethostbyname(self.hostname)
-
- if self.inspector_type == 'sample':
- self.inspector_url = 'http://127.0.0.1:12345/events'
- elif self.inspector_type == 'congress':
- auth=identity_auth.get_identity_auth()
- self.session=session.Session(auth=auth)
- congress = client.Client(session=self.session, service_type='policy')
- ds = congress.list_datasources()['results']
- doctor_ds = next((item for item in ds if item['driver'] == 'doctor'),
- None)
-
- congress_endpoint = congress.httpclient.get_endpoint(auth=auth)
- self.inspector_url = ('%s/v1/data-sources/%s/tables/events/rows' %
- (congress_endpoint, doctor_ds['id']))
-
- def start_loop(self):
- LOG.debug("start ping to host %(h)s (ip=%(i)s)" % {'h': self.hostname,
- 'i': self.ip_addr})
- sock = socket.socket(socket.AF_INET, socket.SOCK_RAW,
- socket.IPPROTO_ICMP)
- sock.settimeout(self.timeout)
- while True:
- try:
- sock.sendto(ICMP_ECHO_MESSAGE, (self.ip_addr, 0))
- data = sock.recv(4096)
- except socket.timeout:
- LOG.info("doctor monitor detected at %s" % time.time())
- self.report_error()
- LOG.info("ping timeout, quit monitoring...")
- return
- time.sleep(self.interval)
-
- def report_error(self):
- payload = [
- {
- 'id': 'monitor_sample_id1',
- 'time': datetime.now().isoformat(),
- 'type': self.event_type,
- 'details': {
- 'hostname': self.hostname,
- 'status': 'down',
- 'monitor': 'monitor_sample',
- 'monitor_event_id': 'monitor_sample_event1'
- },
- },
- ]
- data = json.dumps(payload)
-
- if self.inspector_type == 'sample':
- headers = {'content-type': 'application/json'}
- requests.post(self.inspector_url, data=data, headers=headers)
- elif self.inspector_type == 'congress':
- headers = {
- 'Content-Type': 'application/json',
- 'Accept': 'application/json',
- 'X-Auth-Token':self.session.get_token(),
- }
- requests.put(self.inspector_url, data=data, headers=headers)
-
-
-def get_args():
- parser = argparse.ArgumentParser(description='Doctor Sample Monitor')
- parser.add_argument('hostname', metavar='HOSTNAME', type=str, nargs='?',
- help='a hostname to monitor connectivity')
- parser.add_argument('ip', metavar='IP', type=str, nargs='?',
- help='an IP address to monitor connectivity')
- parser.add_argument('inspector_type', metavar='INSPECTOR_TYPE', type=str, nargs='?',
- help='inspector to report',
- default='sample')
- return parser.parse_args()
-
-
-def main():
- args = get_args()
- monitor = DoctorMonitorSample(args)
- monitor.start_loop()
-
-
-if __name__ == '__main__':
- main()
+++ /dev/null
-#!/bin/bash
-
-function start_monitor_sample {
- cp $TOP_DIR/lib/monitors/sample/monitor.py $TOP_DIR/monitor.py
- pgrep -f "python monitor.py" && return 0
- sudo -E python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" "$INSPECTOR_TYPE" \
- > monitor.log 2>&1 &
-}
-
-function stop_monitor_sample {
- pgrep -f "python monitor.py" || return 0
- sudo kill $(pgrep -f "python monitor.py")
-}
-
-function cleanup_monitor_sample {
- rm monitor.py
- return
-}
+++ /dev/null
-#!/bin/bash -e
-##############################################################################
-# Copyright (c) 2016 NEC Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-# Configuration
-
-[[ "${CI_DEBUG:-true}" == [Tt]rue ]] && set -x
-
-IMAGE_URL=https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img
-#if an existing image name is provided in the enviroment, use that one
-IMAGE_NAME=${IMAGE_NAME:-cirros}
-IMAGE_FILE="${IMAGE_NAME}.img"
-IMAGE_FORMAT=qcow2
-VM_BASENAME=doctor_vm
-VM_FLAVOR=m1.tiny
-#if VM_COUNT set, use that instead
-VM_COUNT=${VM_COUNT:-1}
-NET_NAME=doctor_net
-NET_CIDR=192.168.168.0/24
-ALARM_BASENAME=doctor_alarm
-CONSUMER_PORT=12346
-DOCTOR_USER=doctor
-DOCTOR_PW=doctor
-DOCTOR_PROJECT=doctor
-DOCTOR_ROLE=_member_
-PROFILER_TYPE=${PROFILER_TYPE:-poc}
-PYTHON_ENABLE=${PYTHON_ENABLE:-false}
-
-TOP_DIR=$(cd $(dirname "$0") && pwd)
-
-as_doctor_user="--os-username $DOCTOR_USER --os-password $DOCTOR_PW
- --os-project-name $DOCTOR_PROJECT --os-tenant-name $DOCTOR_PROJECT"
-# NOTE: ceilometer command still requires '--os-tenant-name'.
-#ceilometer="ceilometer ${as_doctor_user/--os-project-name/--os-tenant-name}"
-ceilometer="ceilometer $as_doctor_user"
-as_admin_user="--os-username admin --os-project-name $DOCTOR_PROJECT
- --os-tenant-name $DOCTOR_PROJECT"
-
-upper_constraints="https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/ocata"
-pip_install="pip install -c${upper_constraints}"
-
-# Functions
-
-get_compute_host_info() {
- # get computer host info which first VM boot in as admin user
- COMPUTE_HOST=$(openstack $as_admin_user server show ${VM_BASENAME}1 |
- grep "OS-EXT-SRV-ATTR:host " | awk '{ print $4 }')
- compute_host_in_undercloud=${COMPUTE_HOST%%.*}
- die_if_not_set $LINENO COMPUTE_HOST "Failed to get compute hostname"
-
- get_compute_ip_from_hostname $COMPUTE_HOST
-
- echo "COMPUTE_HOST=$COMPUTE_HOST"
- echo "COMPUTE_IP=$COMPUTE_IP"
-
- # verify connectivity to target compute host
- ping -c 1 "$COMPUTE_IP"
- if [[ $? -ne 0 ]] ; then
- die $LINENO "Can not ping to computer host"
- fi
-
- # verify ssh to target compute host
- ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'exit'
- if [[ $? -ne 0 ]] ; then
- die $LINENO "Can not ssh to computer host"
- fi
-}
-
-# TODO(r-mibu): update this function to support consumer instance
-# and migrate this function into installer lib
-get_consumer_ip___to_be_removed() {
- local get_consumer_command="ip route get $COMPUTE_IP | awk '/ src /{print \$NF}'"
- if is_installer apex; then
- CONSUMER_IP=$(sudo ssh $ssh_opts root@$INSTALLER_IP \
- "$get_consumer_command")
- elif is_installer fuel; then
- CONSUMER_IP=$(sudo sshpass -p r00tme ssh $ssh_opts root@${INSTALLER_IP} \
- "$get_consumer_command")
- elif is_installer local; then
- CONSUMER_IP=`$get_consumer_command`
- fi
- echo "CONSUMER_IP=$CONSUMER_IP"
-
- die_if_not_set $LINENO CONSUMER_IP "Could not get CONSUMER_IP."
-}
-
-download_image() {
- #if a different name was provided for the image in the enviroment there's no need to download the image
- use_existing_image=false
- openstack image list | grep -q " $IMAGE_NAME " && use_existing_image=true
-
- if [[ "$use_existing_image" == false ]] ; then
- [ -e "$IMAGE_FILE" ] && return 0
- wget "$IMAGE_URL" -o "$IMAGE_FILE"
- fi
-}
-
-register_image() {
- openstack image list | grep -q " $IMAGE_NAME " && return 0
- openstack image create "$IMAGE_NAME" \
- --public \
- --disk-format "$IMAGE_FORMAT" \
- --container-format bare \
- --file "$IMAGE_FILE"
-}
-
-create_test_user() {
- openstack project list | grep -q " $DOCTOR_PROJECT " || {
- openstack project create --description "Doctor Project" \
- "$DOCTOR_PROJECT"
- }
- openstack user list | grep -q " $DOCTOR_USER " || {
- openstack user create "$DOCTOR_USER" --password "$DOCTOR_PW" \
- --project "$DOCTOR_PROJECT"
- }
- openstack role show "$DOCTOR_ROLE" | grep -q " $DOCTOR_ROLE " || {
- openstack role create "$DOCTOR_ROLE"
- }
- openstack role assignment list --user "$DOCTOR_USER" \
- --project "$DOCTOR_PROJECT" --names | grep -q " $DOCTOR_ROLE " || {
- openstack role add "$DOCTOR_ROLE" --user "$DOCTOR_USER" \
- --project "$DOCTOR_PROJECT"
- }
- openstack role assignment list --user admin --project "$DOCTOR_PROJECT" \
- --names | grep -q " admin " || {
- openstack role add admin --user admin --project "$DOCTOR_PROJECT"
- }
- # tojuvone: openstack quota show is broken and have to use nova
- # https://bugs.launchpad.net/manila/+bug/1652118
- # Note! while it is encouraged to use openstack client it has proven
- # quite buggy.
- # QUOTA=$(openstack quota show $DOCTOR_PROJECT)
- DOCTOR_QUOTA=$(nova quota-show --tenant $DOCTOR_PROJECT)
- # We make sure that quota allows number of instances and cores
- OLD_INSTANCE_QUOTA=$(echo "${DOCTOR_QUOTA}" | grep " instances " | \
- awk '{print $4}')
- if [ $OLD_INSTANCE_QUOTA -lt $VM_COUNT ]; then
- openstack quota set --instances $VM_COUNT \
- $DOCTOR_USER
- fi
- OLD_CORES_QUOTA=$(echo "${DOCTOR_QUOTA}" | grep " cores " | \
- awk '{print $4}')
- if [ $OLD_CORES_QUOTA -lt $VM_COUNT ]; then
- openstack quota set --cores $VM_COUNT \
- $DOCTOR_USER
- fi
-}
-
-remove_test_user() {
- openstack project list | grep -q " $DOCTOR_PROJECT " && {
- openstack role assignment list --user admin \
- --project "$DOCTOR_PROJECT" --names | grep -q " admin " && {
- openstack role remove admin --user admin --project "$DOCTOR_PROJECT"
- }
- openstack user list | grep -q " $DOCTOR_USER " && {
- openstack role assignment list --user "$DOCTOR_USER" \
- --project "$DOCTOR_PROJECT" --names | grep -q " $DOCTOR_ROLE " && {
- openstack role remove "$DOCTOR_ROLE" --user "$DOCTOR_USER" \
- --project "$DOCTOR_PROJECT"
- }
- openstack user delete "$DOCTOR_USER"
- }
- openstack project delete "$DOCTOR_PROJECT"
- }
-}
-
-boot_vm() {
- # test VM done with test user, so can test non-admin
-
- if ! openstack $as_doctor_user network show $NET_NAME; then
- openstack $as_doctor_user network create $NET_NAME
- fi
- if ! openstack $as_doctor_user subnet show $NET_NAME; then
- openstack $as_doctor_user subnet create $NET_NAME \
- --network $NET_NAME --subnet-range $NET_CIDR --no-dhcp
- fi
- net_id=$(openstack $as_doctor_user network show $NET_NAME -f value -c id)
-
- servers=$(openstack $as_doctor_user server list)
- for i in `seq $VM_COUNT`; do
- echo "${servers}" | grep -q " $VM_BASENAME$i " && continue
- openstack $as_doctor_user server create --flavor "$VM_FLAVOR" \
- --image "$IMAGE_NAME" --nic net-id=$net_id "$VM_BASENAME$i"
- done
- sleep 1
-}
-
-create_alarm() {
- # get vm_id as test user
- alarm_list=$($ceilometer alarm-list)
- vms=$(openstack $as_doctor_user server list)
- for i in `seq $VM_COUNT`; do
- echo "${alarm_list}" | grep -q " $ALARM_BASENAME$i " || {
- vm_id=$(echo "${vms}" | grep " $VM_BASENAME$i " | awk '{print $2}')
- # TODO(r-mibu): change notification endpoint from localhost to the
- # consumer. IP address (functest container).
- $ceilometer alarm-event-create \
- --name "$ALARM_BASENAME$i" \
- --alarm-action "http://localhost:$CONSUMER_PORT/failure" \
- --description "VM failure" \
- --enabled True \
- --repeat-actions False \
- --severity "moderate" \
- --event-type compute.instance.update \
- -q "traits.state=string::error; \
- traits.instance_id=string::$vm_id"
- }
- done
-}
-
-start_consumer() {
- pgrep -f "python consumer.py" && return 0
- python consumer.py "$CONSUMER_PORT" > consumer.log 2>&1 &
-
- # NOTE(r-mibu): create tunnel to the controller nodes, so that we can
- # avoid some network problems dpends on infra and installers.
- # This tunnel will be terminated by stop_consumer() or after 10 mins passed.
- if ! is_installer local; then
- for ip in $CONTROLLER_IPS
- do
- forward_rule="-R $CONSUMER_PORT:localhost:$CONSUMER_PORT"
- tunnel_command="sudo ssh $ssh_opts_cpu $COMPUTE_USER@$ip $forward_rule sleep 600"
- $tunnel_command > "ssh_tunnel.${ip}.log" 2>&1 < /dev/null &
- done
- fi
-}
-
-stop_consumer() {
- pgrep -f "python consumer.py" || return 0
- kill $(pgrep -f "python consumer.py")
-
- # NOTE(r-mibu): terminate tunnels to the controller nodes
- if ! is_installer local; then
- for ip in $CONTROLLER_IPS
- do
- forward_rule="-R $CONSUMER_PORT:localhost:$CONSUMER_PORT"
- tunnel_command="sudo ssh $ssh_opts_cpu $COMPUTE_USER@$ip $forward_rule sleep 600"
- kill $(pgrep -f "$tunnel_command")
- done
- fi
-}
-
-wait_for_vm_launch() {
- echo "waiting for vm launch..."
-
- count=0
- while [[ ${count} -lt 60 ]]
- do
- active_count=0
- vms=$(openstack $as_doctor_user server list)
- for i in `seq $VM_COUNT`; do
- state=$(echo "${vms}" | grep " $VM_BASENAME$i " | awk '{print $6}')
- if [[ "$state" == "ACTIVE" ]]; then
- active_count=$(($active_count+1))
- elif [[ "$state" == "ERROR" ]]; then
- die $LINENO "vm state $VM_BASENAME$i is ERROR"
- else
- #This VM not yet active
- count=$(($count+1))
- sleep 5
- continue
- fi
- done
- [[ $active_count -eq $VM_COUNT ]] && {
- echo "get computer host info..."
- get_compute_host_info
- VMS_ON_FAILED_HOST=$(openstack $as_doctor_user server list --host \
- $COMPUTE_HOST | grep " ${VM_BASENAME}" | wc -l)
- return 0
- }
- #Not all VMs active
- count=$(($count+1))
- sleep 5
- done
- die $LINENO "Time out while waiting for VM launch"
-}
-
-inject_failure() {
- echo "disabling network of compute host [$COMPUTE_HOST] for 3 mins..."
- cat > disable_network.sh << 'END_TXT'
-#!/bin/bash -x
-sleep 1
-if [ -n "@INTERFACE_NAME@" ]; then
- dev=@INTERFACE_NAME@
-else
- dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $NF}')
-fi
-sudo ip link set $dev down
-echo "doctor set link down at" $(date "+%s.%N")
-sleep 180
-sudo ip link set $dev up
-sleep 1
-END_TXT
- sed -i -e "s/@COMPUTE_IP@/$COMPUTE_IP/" disable_network.sh
- sed -i -e "s/@INTERFACE_NAME@/$INTERFACE_NAME/" disable_network.sh
- chmod +x disable_network.sh
- scp $ssh_opts_cpu disable_network.sh "$COMPUTE_USER@$COMPUTE_IP:"
- ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'nohup ./disable_network.sh > disable_network.log 2>&1 &'
- # use host time to get rid of potential time sync deviation between nodes
- triggered=$(date "+%s.%N")
-}
-
-wait_consumer() {
- local interval=1
- local rounds=$(($1 / $interval))
- for i in `seq $rounds`; do
- notified_count=$(grep "doctor consumer notified at" consumer.log | wc -l)
- if [[ $notified_count -eq $VMS_ON_FAILED_HOST ]]; then
- return 0
- fi
- sleep $interval
- done
- die $LINENO "Consumer hasn't received fault notification."
-}
-
-calculate_notification_time() {
- wait_consumer 60
- #keep 'at' as the last keyword just before the value, and
- #use regex to get value instead of the fixed column
- if [ ! -f monitor.log ]; then
- scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:monitor.log" .
- fi
- detected=$(grep "doctor monitor detected at" monitor.log |\
- sed -e "s/^.* at //" | tail -1)
- notified=$(grep "doctor consumer notified at" consumer.log |\
- sed -e "s/^.* at //" | tail -1)
-
- echo "$notified $detected" | \
- awk '{
- d = $1 - $2;
- if (d < 1 && d > 0) { print d " OK"; exit 0 }
- else { print d " NG"; exit 1 }
- }'
-}
-
-check_host_status() {
- # Check host related to first Doctor VM is in wanted state
- # $1 Expected state
- # $2 Seconds to wait to have wanted state
- expected_state=$1
- local interval=5
- local rounds=$(($2 / $interval))
- for i in `seq $rounds`; do
- host_status_line=$(openstack $as_doctor_user --os-compute-api-version \
- 2.16 server show ${VM_BASENAME}1 | grep "host_status")
- host_status=$(echo $host_status_line | awk '{print $4}')
- die_if_not_set $LINENO host_status "host_status not reported by: nova show ${VM_BASENAME}1"
- if [[ "$expected_state" =~ "$host_status" ]] ; then
- echo "${VM_BASENAME}1 showing host_status: $host_status"
- return 0
- else
- sleep $interval
- fi
- done
- if [[ "$expected_state" =~ "$host_status" ]] ; then
- echo "${VM_BASENAME}1 showing host_status: $host_status"
- else
- die $LINENO "host_status:$host_status not equal to expected_state: $expected_state"
- fi
-}
-
-unset_forced_down_hosts() {
- # for debug
- openstack compute service list --service nova-compute
-
- downed_computes=$(openstack compute service list --service nova-compute \
- -f value -c Host -c State | grep ' down$' \
- | sed -e 's/ *down$//')
- echo "downed_computes: $downed_computes"
- for host in $downed_computes
- do
- # TODO(r-mibu): use openstack client
- #openstack compute service set --up $host nova-compute
- nova service-force-down --unset $host nova-compute
- done
-
- echo "waiting disabled compute host back to be enabled..."
- wait_until 'openstack compute service list --service nova-compute
- -f value -c State | grep -q down' 240 5
-
- for host in $downed_computes
- do
- # TODO(r-mibu): improve 'get_compute_ip_from_hostname'
- get_compute_ip_from_hostname $host
- wait_until "! ping -c 1 $COMPUTE_IP" 120 5
- done
-}
-
-collect_logs() {
- if [[ -n "$COMPUTE_IP" ]];then
- scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:disable_network.log" .
- fi
-
- # TODO(yujunz) collect other logs, e.g. nova, aodh
-}
-
-run_profiler() {
- if [[ "$PROFILER_TYPE" == "poc" ]]; then
- linkdown=$(grep "doctor set link down at " disable_network.log |\
- sed -e "s/^.* at //")
- vmdown=$(grep "doctor mark vm.* error at" inspector.log |tail -n 1 |\
- sed -e "s/^.* at //")
- hostdown=$(grep "doctor mark host.* down at" inspector.log |\
- sed -e "s/^.* at //")
-
- # TODO(yujunz) check the actual delay to verify time sync status
- # expected ~1s delay from $trigger to $linkdown
- relative_start=${linkdown}
- export DOCTOR_PROFILER_T00=$(python -c \
- "print(int(($linkdown-$relative_start)*1000))")
- export DOCTOR_PROFILER_T01=$(python -c \
- "print(int(($detected-$relative_start)*1000))")
- export DOCTOR_PROFILER_T03=$(python -c \
- "print(int(($vmdown-$relative_start)*1000))")
- export DOCTOR_PROFILER_T04=$(python -c \
- "print(int(($hostdown-$relative_start)*1000))")
- export DOCTOR_PROFILER_T09=$(python -c \
- "print(int(($notified-$relative_start)*1000))")
-
- python profiler_poc.py > doctor_profiler.log 2>&1
- fi
-}
-
-cleanup() {
- set +e
- echo "cleanup..."
- stop_inspector
- stop_consumer
-
- unset_forced_down_hosts
- stop_monitor
- collect_logs
-
- vms=$(openstack $as_doctor_user server list)
- vmstodel=""
- for i in `seq $VM_COUNT`; do
- $(echo "${vms}" | grep -q " $VM_BASENAME$i ") &&
- vmstodel+=" $VM_BASENAME$i"
- done
- [[ $vmstodel ]] && openstack $as_doctor_user server delete $vmstodel
- alarm_list=$($ceilometer alarm-list)
- for i in `seq $VM_COUNT`; do
- alarm_id=$(echo "${alarm_list}" | grep " $ALARM_BASENAME$i " |
- awk '{print $2}')
- [ -n "$alarm_id" ] && $ceilometer alarm-delete "$alarm_id"
- done
- openstack $as_doctor_user subnet delete $NET_NAME
- sleep 1
- openstack $as_doctor_user network delete $NET_NAME
- sleep 1
-
- image_id=$(openstack image list | grep " $IMAGE_NAME " | awk '{print $2}')
- sleep 1
- #if an existing image was used, there's no need to remove it here
- if [[ "$use_existing_image" == false ]] ; then
- [ -n "$image_id" ] && openstack image delete "$image_id"
- fi
-
- remove_test_user
-
- cleanup_installer
- cleanup_inspector
- cleanup_monitor
-
- # NOTE: Temporal log printer.
- for f in $(find . -name '*.log')
- do
- echo
- echo "[$f]"
- sed -e 's/^/ | /' $f
- echo
- done
-}
-
-setup_python_packages() {
- pip freeze |grep -i flask\= > /dev/null || sudo ${pip_install} flask
- command -v openstack || sudo ${pip_install} python-openstackclient
- command -v ceilometer || sudo ${pip_install} python-ceilometerclient
- command -v congress || sudo ${pip_install} python-congressclient
-}
-
-# Main process
-
-if [[ $PYTHON_ENABLE == [Tt]rue ]]; then
- which tox || sudo ${pip_install} tox
- if [ -f /usr/bin/apt-get ]; then
- sudo apt-get install -y python3-dev
- elif [ -f /usr/bin/yum ] ; then
- sudo yum install -y python3-devel
- fi
-
- cd $TOP_DIR
- echo "executing tox..."
- tox
- exit $?
-fi
-
-echo "Note: doctor/tests/run.sh has been executed."
-git log --oneline -1 || true # ignore even you don't have git installed
-
-trap cleanup EXIT
-
-setup_python_packages
-
-source $TOP_DIR/functions-common
-source $TOP_DIR/lib/installer
-source $TOP_DIR/lib/inspector
-source $TOP_DIR/lib/monitor
-
-rm -f *.log
-
-setup_installer
-
-echo "preparing VM image..."
-download_image
-register_image
-
-echo "creating test user..."
-create_test_user
-
-echo "creating VM..."
-boot_vm
-wait_for_vm_launch
-
-echo "creating alarm..."
-#TODO: change back to use, network problems depends on infra and installers
-#get_consumer_ip
-create_alarm
-
-echo "starting doctor sample components..."
-start_inspector
-start_monitor
-start_consumer
-
-sleep 60
-echo "injecting host failure..."
-inject_failure
-
-check_host_status "(DOWN|UNKNOWN)" 60
-unset_forced_down_hosts
-calculate_notification_time
-collect_logs
-run_profiler
-
-echo "done"
Code Review / doctor.git / commitdiff