*~
+*.pyc
.*.sw?
**.log
/docs_build/
/releng/
/tests/*.img
+#IntelJ Idea
+.idea/
+
#Build results
.tox
-
url: https://review.openstack.org/424340
system: Gerrit
- # WIP
+-
+ url: https://bugs.launchpad.net/python-openstackclient/+bug/1684989
+ system: Launchpad-bug
This directory contains the files necessary to run OpenStack with enabled
OPNFV Doctor in DevStack.
-To configure DevStack to enable OPNFV Doctor edit
+1. To configure DevStack to enable OPNFV Doctor edit
``${DEVSTACK_DIR}/local.conf`` file and add::
enable_plugin aodh http://git.openstack.org/openstack/aodh
.. note:: The order of enabling plugins matters.
-Run DevStack as normal::
+2. To enable Python 3 in DevStack, please add::
+
+ USE_PYTHON3=True
+
+3. To enable Congress as Doctor Inspector, please also add::
+
+ enable_plugin congress https://git.openstack.org/openstack/congress
+
+4. Run DevStack as normal::
$ ./stack.sh
--- /dev/null
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. http://creativecommons.org/licenses/by/4.0
+
+Monitor Types and Limitations
+=============================
+
+Currently there are two monitor types supported: sample and collectd
+
+Sample Monitor
+--------------
+
+Sample monitor type pings the compute host from the control host and calculates the
+notification time after the ping timeout.
+Also if inspector type is sample, the compute node needs to communicate with the control
+node on port 12345. This port needs to be opened for incomming traffic on control node.
+
+Collectd Monitor
+----------------
+
+Collectd monitor type uses collectd daemon running ovs_events plugin. Collectd runs on
+compute to send instant notification to the control node. The notification time is
+calculated by using the difference of time at which compute node sends notification to
+control node and the time at which consumer is notified. The time on control and compute
+node has to be synchronized for this reason. For further details on setting up collectd
+on the compute node, use the following link:
+http://docs.opnfv.org/en/stable-danube/submodules/barometer/docs/release/userguide/feature.userguide.html#id18
+
+Collectd monitors an interface managed by OVS. If the interface is not be assigned
+an IP, the user has to provide the name of interface to be monitored. The command to
+launch the doctor test in that case is:
+MONITOR_TYPE=collectd INSPECTOR_TYPE=sample INTERFACE_NAME=example_iface ./run.sh
+
+If the interface name or IP is not provided, the collectd monitor type will monitor the
+default management interface. This may result in the failure of doctor run.sh test case.
+The test case sets the monitored interface down and if the inspector (sample or congress)
+is running on the same subnet, collectd monitor will not be able to communicate with it.
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+[DEFAULT]
+#image_name = cirros
+#image_format = qcow2
+#image_filename = cirros.img
+#image_download_url = https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img
+
+#glance_version = 2
+#nova_version = 2.34
+#aodh_version = 2
+
+#doctor_user = doctor
+#doctor_passwd = doctor
+#doctor_project = doctor
+#doctor_role = _member_
+#quota_instances = 1
+#quota_cores = 1
+
+#net_name = doctor_net
+#net_cidr = 192.168.168.0/24
+#flavor = m1.tiny
+#instance_count = 1
+#instance_basename = doctor_vm
+
+#alarm_basename = doctor_alarm
+
+[installer]
+#type = local
+#ip = 127.0.0.1
+#username = root
+
+[monitor]
+#type = sample
+
+[inspector]
+#type = sample
+#ip = 127.0.0.1
+#port = 12345
+
+[consumer]
+#type = sample
+#ip = 127.0.0.1
+#port = 12346
oslo.config==3.22.0 # Apache-2.0
python-openstackclient==2.3.0
python-ceilometerclient==2.6.2
+aodhclient==0.7.0
python-keystoneclient==3.5.0
python-neutronclient==6.0.0
python-novaclient==6.0.0
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import aodh_client
+from os_clients import nova_client
+
+OPTS = [
+ cfg.StrOpt('alarm_basename',
+ default='doctor_alarm',
+ help='the base name of alarm',
+ required=True),
+]
+
+
+class Alarm(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.auth = get_identity_auth(username=self.conf.doctor_user,
+ password=self.conf.doctor_passwd,
+ project=self.conf.doctor_project)
+ self.aodh = \
+ aodh_client(conf.aodh_version,
+ get_session(auth=self.auth))
+ self.nova = \
+ nova_client(conf.nova_version,
+ get_session(auth=self.auth))
+ self._init_alarm_name()
+
+ def _init_alarm_name(self):
+ self.alarm_names = []
+ for i in range(0, self.conf.instance_count):
+ alarm_name = '%s%d' % (self.conf.alarm_basename, i)
+ self.alarm_names.append(alarm_name)
+
+ def create(self):
+ self.log.info('alarm create start......')
+
+ alarms = {alarm['name']: alarm for alarm in self.aodh.alarm.list()}
+ servers = \
+ {getattr(server, 'name'): server
+ for server in self.nova.servers.list()}
+
+ for i in range(0, self.conf.instance_count):
+ alarm_name = self.alarm_names[i]
+ if alarm_name in alarms:
+ continue;
+ vm_name = '%s%d' % (self.conf.instance_basename, i)
+ vm_id = getattr(servers[vm_name], 'id')
+ alarm_request = dict(
+ name=alarm_name,
+ description=u'VM failure',
+ enabled=True,
+ alarm_actions=[u'http://%s:%d/failure'
+ % (self.conf.consumer.ip,
+ self.conf.consumer.port)],
+ repeat_actions=False,
+ severity=u'moderate',
+ type=u'event',
+ event_rule=dict(
+ event_type=u'compute.instance.update',
+ query=[
+ dict(field=u'traits.instance_id',
+ type='',
+ op=u'eq',
+ value=vm_id),
+ dict(field=u'traits.state',
+ type='',
+ op=u'eq',
+ value=u'error')]))
+ self.aodh.alarm.create(alarm_request)
+
+ self.log.info('alarm create end......')
+
+ def delete(self):
+ self.log.info('alarm delete start.......')
+
+ alarms = {alarm['name']: alarm for alarm in self.aodh.alarm.list()}
+ for alarm_name in self.alarm_names:
+ if alarm_name in alarms:
+ self.aodh.alarm.delete(alarms[alarm_name]['alarm_id'])
+
+ del self.alarm_names[:]
+
+ self.log.info('alarm delete end.......')
# which accompanies this distribution, and is available at\r
# http://www.apache.org/licenses/LICENSE-2.0\r
##############################################################################\r
+import itertools\r
+\r
from oslo_config import cfg\r
\r
+import alarm\r
+import consumer\r
import image\r
+import instance\r
+import installer\r
+import network\r
+import inspector\r
+import monitor\r
import os_clients\r
+import user\r
\r
\r
def list_opts():\r
return [\r
- ('os_clients', os_clients.OPTS),\r
- ('image', image.IMAGE_OPTS),\r
+ ('installer', installer.OPTS),\r
+ ('monitor', monitor.OPTS),\r
+ ('inspector', inspector.OPTS),\r
+ ('consumer', consumer.OPTS),\r
+ ('DEFAULT', itertools.chain(\r
+ os_clients.OPTS,\r
+ image.OPTS,\r
+ user.OPTS,\r
+ network.OPTS,\r
+ instance.OPTS,\r
+ alarm.OPTS))\r
]\r
\r
\r
-def prepare_conf(conf=None):\r
+def prepare_conf(args=None, conf=None, config_files=None):\r
if conf is None:\r
conf = cfg.ConfigOpts()\r
\r
conf.register_opts(list(options),\r
group=None if group == 'DEFAULT' else group)\r
\r
+ conf(args, project='doctor', validate_default_values=True,\r
+ default_config_files=config_files)\r
+\r
return conf\r
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+from oslo_utils import importutils
+
+
+OPTS = [
+ cfg.StrOpt('type',
+ default='sample',
+ choices=['sample'],
+ help='the component of doctor consumer',
+ required=True),
+ cfg.StrOpt('ip',
+ default='127.0.0.1',
+ help='the ip of consumer',
+ required=True),
+ cfg.IntOpt('port',
+ default='12346',
+ help='the port of doctor consumer',
+ required=True),
+]
+
+
+_consumer_name_class_mapping = {
+ 'sample': 'consumer.sample.SampleConsumer'
+}
+
+
+def get_consumer(conf, log):
+ consumer_class = _consumer_name_class_mapping.get(conf.consumer.type)
+ return importutils.import_object(consumer_class, conf, log)
\ No newline at end of file
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseConsumer(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+
+ @abc.abstractmethod
+ def start(self):
+ pass
+
+ @abc.abstractmethod
+ def stop(self):
+ pass
\ No newline at end of file
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from flask import Flask
+from flask import request
+import json
+import time
+from threading import Thread
+import requests
+
+from consumer.base import BaseConsumer
+
+
+class SampleConsumer(BaseConsumer):
+
+ def __init__(self, conf, log):
+ super(SampleConsumer, self).__init__(conf, log)
+ self.app = None
+
+ def start(self):
+ self.log.info('sample consumer start......')
+ self.app = ConsumerApp(self.conf.consumer.port, self, self.log)
+ self.app.start()
+
+ def stop(self):
+ self.log.info('sample consumer stop......')
+ if not self.app:
+ return
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ }
+ url = 'http://%s:%d/shutdown'\
+ % (self.conf.consumer.ip,
+ self.conf.consumer.port)
+ requests.post(url, data='', headers=headers)
+
+
+class ConsumerApp(Thread):
+
+ def __init__(self, port, consumer, log):
+ Thread.__init__(self)
+ self.port = port
+ self.consumer = consumer
+ self.log = log
+
+ def run(self):
+ app = Flask('consumer')
+
+ @app.route('/failure', methods=['POST'])
+ def event_posted():
+ self.log.info('doctor consumer notified at %s' % time.time())
+ self.log.info('received data = %s' % request.data)
+ data = json.loads(request.data)
+ return "OK"
+
+ @app.route('/shutdown', methods=['POST'])
+ def shutdown():
+ self.log.info('shutdown consumer app server at %s' % time.time())
+ func = request.environ.get('werkzeug.server.shutdown')
+ if func is None:
+ raise RuntimeError('Not running with the Werkzeug Server')
+ func()
+ return 'consumer app shutting down...'
+
+ app.run(host="0.0.0.0", port=self.port)
\ No newline at end of file
from keystoneauth1 import session
-def get_identity_auth():
+def get_identity_auth(username=None, password=None, project=None):
auth_url = os.environ['OS_AUTH_URL']
- username = os.environ['OS_USERNAME']
- password = os.environ['OS_PASSWORD']
+ username = username or os.environ['OS_USERNAME']
+ password = password or os.environ['OS_PASSWORD']
user_domain_name = os.environ.get('OS_USER_DOMAIN_NAME') or 'default'
user_domain_id = os.environ.get('OS_USER_DOMAIN_ID') or 'default'
- project_name = os.environ.get('OS_PROJECT_NAME') or os.environ.get('OS_TENANT_NAME')
+ project_name = project or os.environ.get('OS_PROJECT_NAME') \
+ or os.environ.get('OS_TENANT_NAME')
project_domain_name = os.environ.get('OS_PROJECT_DOMAIN_NAME') or 'default'
project_domain_id = os.environ.get('OS_PROJECT_DOMAIN_ID') or 'default'
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
import os
-import urllib2
+import urllib.request
from oslo_config import cfg
from identity_auth import get_session
from os_clients import glance_client
-import logger as doctor_log
-IMAGE_OPTS = [
- cfg.StrOpt('name',
+OPTS = [
+ cfg.StrOpt('image_name',
default=os.environ.get('IMAGE_NAME', 'cirros'),
help='the name of test image',
required=True),
- cfg.StrOpt('format',
+ cfg.StrOpt('image_format',
default='qcow2',
help='the format of test image',
required=True),
- cfg.StrOpt('file_name',
+ cfg.StrOpt('image_filename',
default='cirros.img',
help='the name of image file',
required=True),
- cfg.StrOpt('url',
+ cfg.StrOpt('image_download_url',
default='https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img',
help='the url where to get the image',
required=True),
]
-LOG = doctor_log.Logger('doctor').getLogger()
-
class Image(object):
- def __init__(self, conf):
+ def __init__(self, conf, log):
self.conf = conf
+ self.log = log
self.glance = \
- glance_client(conf.os_clients.glance_version,
- get_session())
+ glance_client(conf.glance_version, get_session())
self.use_existing_image = False
self.image = None
def create(self):
- LOG.info('image create start......')
+ self.log.info('image create start......')
images = {image.name: image for image in self.glance.images.list()}
- if self.conf.image.name not in images:
- if not os.path.exists(self.conf.image.file_name):
- resp = urllib2.urlopen(self.conf.image.url)
- with open(self.conf.image.file_name, "wb") as file:
+ if self.conf.image_name not in images:
+ if not os.path.exists(self.conf.image_filename):
+ resp = urllib.request.urlopen(self.conf.image_download_url)
+ with open(self.conf.image_filename, "wb") as file:
file.write(resp.read())
- self.image = self.glance.images.create(name=self.conf.image.name,
- disk_format=self.conf.image.format,
+ self.image = self.glance.images.create(name=self.conf.image_name,
+ disk_format=self.conf.image_format,
container_format="bare",
visibility="public")
self.glance.images.upload(self.image['id'],
- open(self.conf.image.file_name, 'rb'))
+ open(self.conf.image_filename, 'rb'))
else:
self.use_existing_image = True
- self.image = images[self.conf.image.name]
+ self.image = images[self.conf.image_name]
- LOG.info('image create end......')
+ self.log.info('image create end......')
def delete(self):
- LOG.info('image delete start.......')
+ self.log.info('image delete start.......')
if not self.use_existing_image and self.image:
self.glance.images.delete(self.image['id'])
- LOG.info('image delete end.......')
+ self.log.info('image delete end.......')
# Pool of novaclients for redundant usage
for i in range(self.NUMBER_OF_CLIENTS):
self.novaclients.append(
- novaclient.Client(self.NOVA_API_VERSION, session=sess,
- connection_pool=True))
+ novaclient.Client(self.NOVA_API_VERSION, session=sess))
# Normally we use this client for non redundant API calls
self.nova=self.novaclients[0]
self.nova.servers.list(detailed=False)
def main():
args = get_args()
- app.run(port=args.port)
-
+ app.run(host='0.0.0.0', port=args.port)
if __name__ == '__main__':
main()
--- /dev/null
+#############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+from oslo_utils import importutils
+
+
+OPTS = [
+ cfg.StrOpt('type',
+ default=os.environ.get('INSPECTOR_TYPE', 'sample'),
+ choices=['sample', 'congress', 'vitrage'],
+ help='the component of doctor inspector',
+ required=True),
+ cfg.StrOpt('ip',
+ default='127.0.0.1',
+ help='the host ip of inspector',
+ required=False),
+ cfg.StrOpt('port',
+ default='12345',
+ help='the port of default for inspector',
+ required=False),
+]
+
+
+_inspector_name_class_mapping = {
+ 'sample': 'inspector.sample.SampleInspector',
+ 'congress': 'inspector.congress.CongressInspector',
+}
+
+
+def get_inspector(conf, log):
+ inspector_class = _inspector_name_class_mapping[conf.inspector.type]
+ return importutils.import_object(inspector_class, conf, log)
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseInspector(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+
+ @abc.abstractmethod
+ def get_inspector_url(self):
+ pass
+
+ @abc.abstractmethod
+ def start(self):
+ pass
+
+ @abc.abstractmethod
+ def stop(self):
+ pass
\ No newline at end of file
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import congress_client
+
+from inspector.base import BaseInspector
+
+
+class CongressInspector(BaseInspector):
+ nova_api_min_version = '2.11'
+ doctor_driver = 'doctor'
+ doctor_datasource = 'doctor'
+ policy = 'classification'
+ rules = {
+ 'host_down':
+ 'host_down(host) :- doctor:events(hostname=host, type="compute.host.down", status="down")',
+ 'active_instance_in_host':
+ 'active_instance_in_host(vmid, host) :- nova:servers(id=vmid, host_name=host, status="ACTIVE")',
+ 'host_force_down':
+ 'execute[nova:services.force_down(host, "nova-compute", "True")] :- host_down(host)',
+ 'error_vm_states':
+ 'execute[nova:servers.reset_state(vmid, "error")] :- host_down(host), active_instance_in_host(vmid, host)'
+ }
+
+ def __init__(self, conf, log):
+ super(CongressInspector, self).__init__(conf, log)
+ self.auth = get_identity_auth()
+ self.congress = congress_client(get_session(auth=self.auth))
+ self._init_driver_and_ds()
+ self.inspector_url = self.get_inspector_url()
+
+ def _init_driver_and_ds(self):
+ datasources = \
+ {ds['name']: ds for ds in self.congress.list_datasources()['results']}
+
+ # check nova_api version
+ nova_api_version = datasources['nova']['config'].get('api_version')
+ if nova_api_version and nova_api_version < self.nova_api_min_version:
+ raise Exception('Congress Nova datasource API version < nova_api_min_version(%s)'
+ % self.nova_api_min_version)
+
+ # create doctor datasource if it's not exist
+ if self.doctor_datasource not in datasources:
+ self.congress.create_datasource(
+ body={'driver': self.doctor_driver,
+ 'name': self.doctor_datasource})
+
+ # check whether doctor driver exist
+ drivers = \
+ {driver['id']: driver for driver in self.congress.list_drivers()['results']}
+ if self.doctor_driver not in drivers:
+ raise Exception('Do not support doctor driver in congress')
+
+ self.policy_rules = \
+ {rule['name']: rule for rule in
+ self.congress.list_policy_rules(self.policy)['results']}
+
+ def get_inspector_url(self):
+ ds = self.congress.list_datasources()['results']
+ doctor_ds = next((item for item in ds if item['driver'] == 'doctor'),
+ None)
+ congress_endpoint = self.congress.httpclient.get_endpoint(auth=self.auth)
+ return ('%s/v1/data-sources/%s/tables/events/rows' %
+ (congress_endpoint, doctor_ds['id']))
+
+ def start(self):
+ self.log.info('congress inspector start......')
+
+ for rule_name, rule in self.rules.items():
+ self._add_rule(rule_name, rule)
+
+ def stop(self):
+ self.log.info('congress inspector stop......')
+
+ for rule_name in self.rules.keys():
+ self._del_rule(rule_name)
+
+ def _add_rule(self, rule_name, rule):
+ if rule_name not in self.policy_rules:
+ self.congress.create_policy_rule(self.policy,
+ body={'name': rule_name,
+ 'rule': rule})
+
+ def _del_rule(self, rule_name):
+ if rule_name in self.policy_rules:
+ rule_id = self.policy_rules[rule_name]['id']
+ self.congress.delete_policy_rule(self.policy, rule_id)
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import collections
+from flask import Flask
+from flask import request
+import json
+import time
+from threading import Thread
+import requests
+
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import nova_client
+from inspector.base import BaseInspector
+
+
+class SampleInspector(BaseInspector):
+ event_type = 'compute.host.down'
+
+ def __init__(self, conf, log):
+ super(SampleInspector, self).__init__(conf, log)
+ self.inspector_url = self.get_inspector_url()
+ self.novaclients = list()
+ self._init_novaclients()
+ # Normally we use this client for non redundant API calls
+ self.nova = self.novaclients[0]
+
+ self.servers = collections.defaultdict(list)
+ self.hostnames = list()
+ self.app = None
+
+ def _init_novaclients(self):
+ self.NUMBER_OF_CLIENTS = self.conf.instance_count
+ auth = get_identity_auth(project=self.conf.doctor_project)
+ session = get_session(auth=auth)
+ for i in range(self.NUMBER_OF_CLIENTS):
+ self.novaclients.append(
+ nova_client(self.conf.nova_version, session))
+
+ def _init_servers_list(self):
+ self.servers.clear()
+ opts = {'all_tenants': True}
+ servers = self.nova.servers.list(search_opts=opts)
+ for server in servers:
+ try:
+ host = server.__dict__.get('OS-EXT-SRV-ATTR:host')
+ self.servers[host].append(server)
+ self.log.debug('get hostname=%s from server=%s' % (host, server))
+ except Exception as e:
+ self.log.info('can not get hostname from server=%s' % server)
+
+ def get_inspector_url(self):
+ return 'http://%s:%s' % (self.conf.inspector.ip, self.conf.inspector.port)
+
+ def start(self):
+ self.log.info('sample inspector start......')
+ self._init_servers_list()
+ self.app = InspectorApp(self.conf.inspector.port, self, self.log)
+ self.app.start()
+
+ def stop(self):
+ self.log.info('sample inspector stop......')
+ if not self.app:
+ return
+ for hostname in self.hostnames:
+ self.nova.services.force_down(hostname, 'nova-compute', False)
+
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ }
+ url = '%s%s' % (self.inspector_url, 'shutdown') \
+ if self.inspector_url.endswith('/') else \
+ '%s%s' % (self.inspector_url, '/shutdown')
+ requests.post(url, data='', headers=headers)
+
+ def handle_events(self, events):
+ for event in events:
+ hostname = event['details']['hostname']
+ event_type = event['type']
+ if event_type == self.event_type:
+ self.hostnames.append(hostname)
+ self.disable_compute_host(hostname)
+
+ def disable_compute_host(self, hostname):
+ threads = []
+ if len(self.servers[hostname]) > self.NUMBER_OF_CLIENTS:
+ # TODO(tojuvone): This could be enhanced in future with dynamic
+ # reuse of self.novaclients when all threads in use
+ self.log.error('%d servers in %s. Can handle only %d'%(
+ self.servers[hostname], hostname, self.NUMBER_OF_CLIENTS))
+ for nova, server in zip(self.novaclients, self.servers[hostname]):
+ t = ThreadedResetState(nova, "error", server, self.log)
+ t.start()
+ threads.append(t)
+ for t in threads:
+ t.join()
+ self.nova.services.force_down(hostname, 'nova-compute', True)
+ self.log.info('doctor mark host(%s) down at %s' % (hostname, time.time()))
+
+
+class ThreadedResetState(Thread):
+
+ def __init__(self, nova, state, server, log):
+ Thread.__init__(self)
+ self.nova = nova
+ self.state = state
+ self.server = server
+ self.log = log
+
+ def run(self):
+ self.nova.servers.reset_state(self.server, self.state)
+ self.log.info('doctor mark vm(%s) error at %s' % (self.server, time.time()))
+
+
+class InspectorApp(Thread):
+
+ def __init__(self, port, inspector, log):
+ Thread.__init__(self)
+ self.port = port
+ self.inspector = inspector
+ self.log = log
+
+ def run(self):
+ app = Flask('inspector')
+
+ @app.route('/events', methods=['PUT'])
+ def event_posted():
+ self.log.info('event posted in sample inspector at %s' % time.time())
+ self.log.info('sample inspector = %s' % self.inspector)
+ self.log.info('sample inspector received data = %s' % request.data)
+ events = json.loads(request.data.decode('utf8'))
+ self.inspector.handle_events(events)
+ return "OK"
+
+ @app.route('/shutdown', methods=['POST'])
+ def shutdown():
+ self.log.info('shutdown inspector app server at %s' % time.time())
+ func = request.environ.get('werkzeug.server.shutdown')
+ if func is None:
+ raise RuntimeError('Not running with the Werkzeug Server')
+ func()
+ return 'inspector app shutting down...'
+
+ app.run(host="0.0.0.0", port=self.port)
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+from oslo_utils import importutils
+
+OPTS = [
+ cfg.StrOpt('type',
+ default=os.environ.get('INSTALLER_TYPE', 'local'),
+ choices=['local', 'apex'],
+ help='the type of installer',
+ required=True),
+ cfg.StrOpt('ip',
+ default=os.environ.get('INSTALLER_IP', '127.0.0.1'),
+ help='the ip of installer'),
+ cfg.StrOpt('username',
+ default='root',
+ help='the user name for login installer server',
+ required=True),
+]
+
+
+_installer_name_class_mapping = {
+ 'local': 'installer.local.LocalInstaller',
+ 'apex': 'installer.apex.ApexInstaller'
+}
+
+
+def get_installer(conf, log):
+ installer_class = _installer_name_class_mapping[conf.installer.type]
+ return importutils.import_object(installer_class, conf, log)
\ No newline at end of file
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import getpass
+import grp
+import os
+import pwd
+import stat
+import sys
+
+from installer.common.congress import set_doctor_driver_conf
+from installer.common.congress import restore_doctor_driver_conf
+from installer.base import BaseInstaller
+from utils import SSHClient
+
+
+class ApexInstaller(BaseInstaller):
+ node_user_name = 'heat-admin'
+ cm_set_script = 'set_ceilometer.py'
+ cm_restore_script = 'restore_ceilometer.py'
+
+ def __init__(self, conf, log):
+ super(ApexInstaller, self).__init__(conf, log)
+ self.client = SSHClient(self.conf.installer.ip,
+ self.conf.installer.username,
+ look_for_keys=True)
+ self.key_file = None
+ self.controllers = list()
+ self.controller_clients = list()
+
+ def setup(self):
+ self.log.info('Setup Apex installer start......')
+
+ self.key_file = self.get_ssh_key_from_installer()
+ self.get_controller_ips()
+ self.set_apply_patches()
+
+ def cleanup(self):
+ self.restore_apply_patches()
+
+ def get_ssh_key_from_installer(self):
+ self.log.info('Get SSH keys from Apex installer......')
+
+ self.client.scp('/home/stack/.ssh/id_rsa', './instack_key', method='get')
+ user = getpass.getuser()
+ uid = pwd.getpwnam(user).pw_uid
+ gid = grp.getgrnam(user).gr_gid
+ os.chown('./instack_key', uid, gid)
+ os.chmod('./instack_key', stat.S_IREAD)
+ current_dir = sys.path[0]
+ return '{0}/{1}'.format(current_dir, 'instack_key')
+
+ def get_controller_ips(self):
+ self.log.info('Get controller ips from Apex installer......')
+
+ command = "source stackrc; " \
+ "nova list | grep ' overcloud-controller-[0-9] ' " \
+ "| sed -e 's/^.*ctlplane=//' |awk '{print $1}'"
+ ret, controllers = self.client.ssh(command)
+ if ret:
+ raise Exception('Exec command to get controller ips in Apex installer failed'
+ 'ret=%s, output=%s' % (ret, controllers))
+ self.controllers = controllers
+
+ def set_apply_patches(self):
+ self.log.info('Set apply patches start......')
+
+ for node_ip in self.controllers:
+ client = SSHClient(node_ip, self.node_user_name, key_filename=self.key_file)
+ self.controller_clients.append(client)
+ self._ceilometer_apply_patches(client, self.cm_set_script)
+ cmd = 'sudo systemctl restart openstack-congress-server.service'
+ set_doctor_driver_conf(client, cmd)
+
+ def restore_apply_patches(self):
+ self.log.info('restore apply patches start......')
+
+ for client in self.controller_clients:
+ self._ceilometer_apply_patches(client, self.cm_restore_script)
+ cmd = 'sudo systemctl restart openstack-congress-server.service'
+ restore_doctor_driver_conf(client, cmd)
+
+ def _ceilometer_apply_patches(self, ssh_client, script_name):
+ installer_dir = os.path.dirname(os.path.realpath(__file__))
+ script_abs_path = '{0}/{1}/{2}'.format(installer_dir, 'common', script_name)
+
+ ssh_client.scp(script_abs_path, script_name)
+ cmd = 'sudo python %s' % script_name
+ ret, output = ssh_client.ssh(cmd)
+ if ret:
+ raise Exception('Do the ceilometer command in controller node failed....'
+ 'ret=%s, cmd=%s, output=%s' % (ret, cmd, output))
+ ssh_client.ssh('sudo systemctl restart openstack-ceilometer-notification.service')
+
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseInstaller(object):
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+
+ @abc.abstractproperty
+ def node_user_name(self):
+ """user name for login to cloud node"""
+
+ @abc.abstractmethod
+ def get_ssh_key_from_installer(self):
+ pass
+
+ @abc.abstractmethod
+ def setup(self):
+ pass
+
+ @abc.abstractmethod
+ def cleanup(self):
+ pass
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+def set_doctor_driver_conf(ssh_client, restart_cmd):
+ cg_set_cmd = '''#!/bin/bash
+co_conf=/etc/congress/congress.conf
+co_conf_bak=/etc/congress/congress.conf.bak
+co_entry="congress.datasources.doctor_driver.DoctorDriver"
+if sudo grep -e "^drivers.*$co_entry" $co_conf; then
+ echo "NOTE: congress is configured as we needed"
+else
+ echo "modify the congress config"
+ sudo cp $co_conf $co_conf_bak
+ sudo sed -i -e "/^drivers/s/$/,$co_entry/" $co_conf
+ %s
+fi
+ ''' % (restart_cmd)
+
+ ret, output = ssh_client.ssh(cg_set_cmd)
+ if ret:
+ raise Exception('Do the congress command in controller node failed....'
+ 'ret=%s, cmd=%s, output=%s' % (ret, cg_set_cmd, output))
+
+
+def restore_doctor_driver_conf(ssh_client, restart_cmd):
+ cg_restore_cmd = '''#!/bin/bash
+co_conf=/etc/congress/congress.conf
+co_conf_bak=/etc/congress/congress.conf.bak
+if [ -e $co_conf_bak ]; then
+ echo "restore the congress config"
+ sudo cp $co_conf_bak $co_conf
+ sudo rm $co_conf_bak
+ %s
+else
+ echo "Do not need to restore the congress config"
+fi
+ ''' % (restart_cmd)
+
+ ret, output = ssh_client.ssh(cg_restore_cmd)
+ if ret:
+ raise Exception('Do the congress command in controller node failed....'
+ 'ret=%s, cmd=%s, output=%s' % (ret, cg_restore_cmd, output))
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+
+ep_file = '/etc/ceilometer/event_pipeline.yaml'
+ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
+
+
+def restore_ep_config():
+
+ if not os.path.isfile(ep_file_bak):
+ print('Bak_file:%s does not exist.' % ep_file_bak)
+ else:
+ print('restore')
+ shutil.copyfile(ep_file_bak, ep_file)
+ os.remove(ep_file_bak)
+ return
+
+
+restore_ep_config()
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+import yaml
+
+ep_file = '/etc/ceilometer/event_pipeline.yaml'
+ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
+event_notifier_topic = 'notifier://?topic=alarm.all'
+
+
+def set_notifier_topic():
+ config_modified = False
+
+ if not os.path.isfile(ep_file):
+ raise Exception("File doesn't exist: %s." % ep_file)
+
+ with open(ep_file, 'r') as file:
+ config = yaml.safe_load(file)
+
+ sinks = config['sinks']
+ for sink in sinks:
+ if sink['name'] == 'event_sink':
+ publishers = sink['publishers']
+ if event_notifier_topic not in publishers:
+ print('Add event notifier in ceilometer')
+ publishers.append(event_notifier_topic)
+ config_modified = True
+ else:
+ print('NOTE: event notifier is configured in ceilometer as we needed')
+
+ if config_modified:
+ shutil.copyfile(ep_file, ep_file_bak)
+ with open(ep_file, 'w+') as file:
+ file.write(yaml.safe_dump(config))
+
+
+set_notifier_topic()
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+
+from installer.base import BaseInstaller
+from utils import load_json_file
+from utils import write_json_file
+
+
+class LocalInstaller(BaseInstaller):
+ node_user_name = 'root'
+
+ nova_policy_file = '/etc/nova/policy.json'
+ nova_policy_file_backup = '%s%s' % (nova_policy_file, '.bak')
+
+ def __init__(self, conf, log):
+ super(LocalInstaller, self).__init__(conf, log)
+ self.policy_modified = False
+ self.add_policy_file = False
+
+ def setup(self):
+ self.get_ssh_key_from_installer()
+ self.set_apply_patches()
+
+ def cleanup(self):
+ self.restore_apply_patches()
+
+ def get_ssh_key_from_installer(self):
+ self.log.info('Assuming SSH keys already exchanged with computer for local installer type')
+ return
+
+ def set_apply_patches(self):
+ self._set_nova_policy()
+
+ def restore_apply_patches(self):
+ self._restore_nova_policy()
+
+ def _set_nova_policy(self):
+ host_status_policy = 'os_compute_api:servers:show:host_status'
+ host_status_rule = 'rule:admin_or_owner'
+ policy_data = {
+ 'context_is_admin': 'role:admin',
+ 'owner': 'user_id:%(user_id)s',
+ 'admin_or_owner': 'rule:context_is_admin or rule:owner',
+ host_status_policy: host_status_rule
+ }
+
+ if os.path.isfile(self.nova_policy_file):
+ data = load_json_file(self.nova_policy_file)
+ if host_status_policy in data:
+ rule_origion = data[host_status_policy]
+ if host_status_rule == rule_origion:
+ self.log.info('Do not need to modify nova policy.')
+ self.policy_modified = False
+ else:
+ # update the host_status_policy
+ data[host_status_policy] = host_status_rule
+ self.policy_modified = True
+ else:
+ # add the host_status_policy, if the admin_or_owner is not
+ # defined, add it also
+ for policy, rule in policy_data.items():
+ if policy not in data:
+ data[policy] = rule
+ self.policy_modified = True
+ if self.policy_modified:
+ self.log.info('Nova policy is Modified.')
+ shutil.copyfile(self.nova_policy_file,
+ self.nova_policy_file_backup)
+ else:
+ # file does not exit, create a new one and add the policy
+ self.log.info('Nova policy file not exist. Creating a new one')
+ data = policy_data
+ self.add_policy_file = True
+
+ if self.policy_modified or self.add_policy_file:
+ write_json_file(self.nova_policy_file, data)
+ os.system('screen -S stack -p n-api -X stuff "^C^M^[[A^M"')
+
+ def _restore_nova_policy(self):
+ if self.policy_modified:
+ shutil.copyfile(self.nova_policy_file_backup, self.nova_policy_file)
+ os.remove(self.nova_policy_file_backup)
+ elif self.add_policy_file:
+ os.remove(self.nova_policy_file)
+
+ if self.add_policy_file or self.policy_modified:
+ os.system('screen -S stack -p n-api -X stuff "^C^M^[[A^M"')
+ self.add_policy_file = False
+ self.policy_modified = False
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import time
+
+from oslo_config import cfg
+
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import neutron_client
+from os_clients import nova_client
+
+OPTS = [
+ cfg.StrOpt('flavor',
+ default='m1.tiny',
+ help='the name of flavor',
+ required=True),
+ cfg.IntOpt('instance_count',
+ default=os.environ.get('VM_COUNT', 1),
+ help='the count of instance',
+ required=True),
+ cfg.StrOpt('instance_basename',
+ default='doctor_vm',
+ help='the base name of instance',
+ required=True),
+]
+
+
+class Instance(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.auth = get_identity_auth(username=self.conf.doctor_user,
+ password=self.conf.doctor_passwd,
+ project=self.conf.doctor_project)
+ self.nova = \
+ nova_client(conf.nova_version,
+ get_session(auth=self.auth))
+ self.neutron = neutron_client(get_session(auth=self.auth))
+ self.servers = {}
+ self.vm_names = []
+
+ def create(self):
+ self.log.info('instance create start......')
+
+ # get flavor, image and network for vm boot
+ flavors = {flavor.name: flavor for flavor in self.nova.flavors.list()}
+ flavor = flavors.get(self.conf.flavor)
+ image = self.nova.glance.find_image(self.conf.image_name)
+ network = self.neutron.list_networks(name=self.conf.net_name)['networks'][0]
+ nics = {'net-id': network['id']}
+
+ self.servers = \
+ {getattr(server, 'name'): server
+ for server in self.nova.servers.list()}
+ for i in range(0, self.conf.instance_count):
+ vm_name = "%s%d"%(self.conf.instance_basename, i)
+ self.vm_names.append(vm_name)
+ if vm_name not in self.servers:
+ server = self.nova.servers.create(vm_name, image,
+ flavor, nics=[nics])
+ self.servers[vm_name] = server
+ time.sleep(0.1)
+
+ self.log.info('instance create end......')
+
+ def delete(self):
+ self.log.info('instance delete start.......')
+
+ for vm_name in self.vm_names:
+ if vm_name in self.servers:
+ self.nova.servers.delete(self.servers[vm_name])
+ time.sleep(0.1)
+
+ # check that all vms are deleted
+ while self.nova.servers.list():
+ time.sleep(0.1)
+ self.servers.clear()
+ del self.vm_names[:]
+
+ self.log.info('instance delete end.......')
+
+ def wait_for_vm_launch(self):
+ self.log.info('wait for vm launch start......')
+
+ wait_time = 60
+ count = 0
+ while count < wait_time:
+ active_count = 0
+ for vm_name in self.vm_names:
+ server = self.nova.servers.get(self.servers[vm_name])
+ server_status = getattr(server, 'status').lower()
+ if 'active' == server_status:
+ active_count += 1
+ elif 'error' == server_status:
+ raise Exception('vm launched with error state')
+ else:
+ time.sleep(2)
+ count += 1
+ continue
+ if active_count == self.conf.instance_count:
+ self.log.info('wait for vm launch end......')
+ return
+ count += 1
+ time.sleep(2)
+ raise Exception('time out for vm launch')
+
function get_installer_ip {
is_set INSTALLER_IP && return
- INSTALLER_IP=$(get_first_vnic_ip instack)
+ INSTALLER_IP=$(get_first_vnic_ip undercloud)
}
function installer_get_ssh_keys {
fi
co_conf=/etc/congress/congress.conf
+ co_conf_bak=/etc/congress/congress.conf.bak
co_entry="congress.datasources.doctor_driver.DoctorDriver"
if sudo grep -e "^drivers.*$co_entry" $co_conf; then
echo "NOTE: congress is configured as we needed"
else
echo "modify the congress config"
- sudo sed -i -e "/^drivers/s/$/,$co_entry # added by doctor script/" \
- $co_conf
+ sudo cp $co_conf $co_conf_bak
+ sudo sed -i -e "/^drivers/s/$/,$co_entry/" $co_conf
sudo systemctl restart openstack-congress-server.service
fi
' > installer_apply_patches_$node.log 2>&1
date
co_conf=/etc/congress/congress.conf
- co_entry="congress.datasources.doctor_driver.DoctorDriver"
- if sudo grep -q -e "# added by doctor script" $co_conf; then
- echo "modify the congress config"
- sudo sed -i -e "/^drivers/s/^\(.*\),$co_entry # added by doctor script/\1/" $co_conf
+ co_conf_bak=/etc/congress/congress.conf.bak
+ if [ -e $co_conf_bak ]; then
+ echo "restore the congress config"
+ sudo mv $co_conf_bak $co_conf
sudo systemctl restart openstack-congress-server.service
fi
service nova-api restart
fi
else
- # TODO(tojuvone) policy.json might not exists in Ocata.
- echo "$np_conf does not exist!!!"
+ # policy.json does not exist in Ocata.
+ echo "$np_conf does not exist. Creating new one."
+ echo -e "{\n \"context_is_admin\": \"role:admin\"," > $np_conf
+ echo -e " \"owner\" : \"user_id:%(user_id)s\"," >> $np_conf
+ echo -e " \"admin_or_owner\": \"rule:context_is_admin or rule:owner\"," >> $np_conf
+ echo -e " \"os_compute_api:servers:show:host_status\": \"rule:admin_or_owner\" \n}" >> $np_conf
+ np_rm="${np_conf}-doctor-rm"
+ cp $np_conf $np_rm
+ service nova-api restart
fi
' > installer_apply_patches_$node.log 2>&1
done
fi
np_conf=/etc/nova/policy.json
- entry="os_compute_api:servers:show:host_status"
- if [ -e $np_conf ]; then
- np_backup="${np_conf}-doctor-saved"
- if [ -e $np_backup ]; then
- cp -f $np_backup $np_conf
- rm $np_backup
- service nova-api restart
- fi
+ np_backup="${np_conf}-doctor-saved"
+ np_rm="${np_conf}-doctor-rm"
+ if [ -e $np_backup ]; then
+ cp -f $np_backup $np_conf
+ rm $np_backup
+ service nova-api restart
+ elif [ -e $np_rm ]; then
+ rm $np_conf
+ rm $np_rm
+ service nova-api restart
fi
' >> installer_apply_patches_$node.log 2>&1
done
}
function installer_apply_patches {
- # Noop
+ set -x
+ date
+ echo "### apply patches (installer=local)"
+ np_conf=/etc/nova/policy.json
+ if [ -e $np_conf ]; then
+ entry="os_compute_api:servers:show:host_status"
+ new="rule:admin_or_owner"
+ np_backup="${np_conf}-doctor-saved"
+ if grep -q "${entry}.*${new}" $np_conf; then
+ echo "Not modifying nova policy"
+ elif grep -q "${entry}" $np_conf; then
+ echo "modify nova policy"
+ cp $np_conf $np_backup
+ oldline=$(grep "$entry" $np_conf)
+ newline=$(echo "$oldline" | sed "s/rule.*\"/$new\"/")
+ sed -i "s/$oldline/$newline/" $np_conf
+ # TODO(umar): Update to systemd when screen is no more used for devstack
+ screen -S stack -p n-api -X stuff "^C^M^[[A^M" # restart n-api service
+ else
+ echo "add nova policy"
+ cp $np_conf $np_backup
+ sed -i "/{/a \ \"${entry}\": \"$new\"" $np_conf
+ screen -S stack -p n-api -X stuff "^C^M^[[A^M"
+ fi
+ else
+ # policy.json does not exist in Ocata.
+ echo "$np_conf does not exist. Creating a new one"
+ echo -e '{\n "context_is_admin": "role:admin",' > $np_conf
+ echo -e ' "owner" : "user_id:%(user_id)s",' >> $np_conf
+ echo -e ' "admin_or_owner": "rule:context_is_admin or rule:owner",' >> $np_conf
+ echo -e ' "os_compute_api:servers:show:host_status": "rule:admin_or_owner"\n}' >> $np_conf
+ np_rm="${np_conf}-doctor-rm"
+ cp $np_conf $np_rm
+ screen -S stack -p n-api -X stuff "^C^M^[[A^M"
+ fi
+
return
}
}
function cleanup_installer {
- # Noop
+ set -x
+ echo "### revert patches (installer=local)"
+ date
+
+ np_conf=/etc/nova/policy.json
+ np_backup="${np_conf}-doctor-saved"
+ np_rm="${np_conf}-doctor-rm"
+ if [ -e $np_backup ]; then
+ cp -f $np_backup $np_conf
+ rm $np_backup
+ screen -S stack -p n-api -X stuff "^C^M^[[A^M"
+ elif [ -e $np_rm ]; then
+ rm $np_conf
+ rm $np_rm
+ screen -S stack -p n-api -X stuff "^C^M^[[A^M"
+ fi
+
return
}
--- /dev/null
+#!/bin/bash
+
+MONITOR_TYPE=${MONITOR_TYPE:-sample}
+
+function is_monitor_supported {
+ local monitor="$1"
+ [[ -f $TOP_DIR/lib/monitors/$monitor/$monitor ]]
+}
+
+function is_monitor {
+ local monitor="$1"
+ [[ $monitor == $MONITOR_TYPE ]]
+}
+
+function start_monitor {
+ start_monitor_$MONITOR_TYPE
+}
+
+function stop_monitor {
+ stop_monitor_$MONITOR_TYPE
+}
+
+function cleanup_monitor {
+ cleanup_monitor_$MONITOR_TYPE
+}
+
+if ! is_monitor_supported $MONITOR_TYPE; then
+ die $LINENO "MONITOR_TYPE=$MONITOR_TYPE is not supported."
+fi
+
+source $TOP_DIR/lib/monitors/$MONITOR_TYPE/$MONITOR_TYPE
--- /dev/null
+#!/bin/bash
+
+function start_monitor_collectd {
+ ## CONTROL_IP is the IP of primary interface of control node i.e.
+ ## eth0, eno1. It is used by collectd monitor to communicate with
+ ## sample inspector.
+ ## @TODO (umar) see if mgmt IP of control is a better option. Also
+ ## primary interface may not be the right option
+ CONTROL_IP="$(ip a | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | sed -n 1p)"
+ #CONTROL_IP=192.168.98.6
+
+ echo "
+Hostname \"$COMPUTE_HOST\"
+FQDNLookup false
+Interval 1
+MaxReadInterval 2
+
+<LoadPlugin python>
+ Globals true
+</LoadPlugin>
+LoadPlugin ovs_events
+LoadPlugin logfile
+
+<Plugin logfile>
+ File \"/var/log/collectd.log\"
+ Timestamp true
+ LogLevel \"info\"
+</Plugin>
+
+<Plugin python>
+ ModulePath \"/home/$COMPUTE_USER\"
+ LogTraces true
+ Interactive false
+ Import \"collectd_plugin\"
+ <Module \"collectd_plugin\">
+ control_ip \"$CONTROL_IP\"
+ compute_ip \"$COMPUTE_IP\"
+ compute_host \"$COMPUTE_HOST\"
+ compute_user \"$COMPUTE_USER\"
+ inspector_type \"$INSPECTOR_TYPE\"
+ os_auth_url \"$OS_AUTH_URL\"
+ os_username \"$OS_USERNAME\"
+ os_password \"$OS_PASSWORD\"
+ os_project_name \"$OS_PROJECT_NAME\"
+ os_user_domain_name \"$OS_USER_DOMAIN_NAME\"
+ os_user_domain_id \"$OS_USER_DOMAIN_ID\"
+ os_project_domain_name \"$OS_PROJECT_DOMAIN_NAME\"
+ os_project_domain_id \"$OS_PROJECT_DOMAIN_ID\"
+ </Module>
+</Plugin>
+
+<Plugin ovs_events>
+ Port 6640
+ Socket \"/var/run/openvswitch/db.sock\"
+ Interfaces \"@INTERFACE_NAME@\"
+ SendNotification true
+ DispatchValues false
+</Plugin>
+
+" > $TOP_DIR/lib/monitors/collectd.conf
+
+ scp $ssh_opts_cpu $TOP_DIR/lib/monitors/collectd.conf $COMPUTE_USER@$COMPUTE_IP:
+ ## @TODO (umar) Always assuming that the interface is assigned an IP if
+ ## interface name is not provided. See if there is a better approach
+ ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" "
+ if [ -n \"$INTERFACE_NAME\" ]; then
+ dev=$INTERFACE_NAME
+ else
+ dev=\$(sudo ip a | awk '/ $COMPUTE_IP\//{print \$NF}')
+ fi
+ sed -i -e \"s/@INTERFACE_NAME@/\$dev/\" collectd.conf
+ collectd_conf=/opt/collectd/etc/collectd.conf
+ if [ -e \$collectd_conf ]; then
+ sudo cp \$collectd_conf \${collectd_conf}-doctor-saved
+ else
+ sudo touch \${collectd_conf}-doctor-created
+ fi
+ sudo mv collectd.conf /opt/collectd/etc/collectd.conf"
+
+ scp $ssh_opts_cpu $TOP_DIR/lib/monitors/collectd/collectd_plugin.py $COMPUTE_USER@$COMPUTE_IP:collectd_plugin.py
+ ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" "sudo pkill collectd
+ sudo /opt/collectd/sbin/collectd"
+}
+
+function stop_monitor_collectd {
+ ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'sudo pkill collectd'
+}
+
+function cleanup_monitor_collectd {
+ ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" "
+ collectd_conf=/opt/collectd/etc/collectd.conf
+ if [ -e \"\${collectd_conf}-doctor-created\" ]; then
+ sudo rm \"\${collectd_conf}-doctor-created\"
+ sudo rm \$collectd_conf
+ elif [ -e \"\${collectd_conf}-doctor-saved\" ]; then
+ sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf
+ sudo rm \"\${collectd_conf}-doctor-saved\"
+ fi"
+
+ rm $TOP_DIR/lib/monitors/collectd.conf
+}
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 NEC Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import collectd
+import sys
+from netifaces import interfaces, ifaddresses, AF_INET
+from datetime import datetime
+import json
+import requests
+import time
+from requests.exceptions import ConnectionError
+
+from keystoneauth1 import loading
+from keystoneauth1 import session
+from congressclient.v1 import client
+
+
+def write_debug(str_write, write_type, compute_user):
+ file_name = ('/home/%s/monitor.log' % compute_user)
+ file_tmp = open(file_name, write_type)
+ file_tmp.write( "%s" % str_write)
+ file_tmp.close()
+
+
+class DoctorMonitorCollectd(object):
+ def __init__(self):
+ self.control_ip = ''
+ self.compute_user = ''
+ self.compute_ip = ''
+ self.host_name = ''
+ self.inspector_type = ''
+ self.inspector_url = ''
+ self.os_auth_url = ''
+ self.os_username = ''
+ self.os_password = ''
+ self.os_project_name = ''
+ self.os_user_domain_name = ''
+ self.os_user_domain_id = ''
+ self.os_project_domain_name = ''
+ self.os_project_domain_id = ''
+ self.sess = ''
+ self.auth = ''
+ self.inspector_notified = 0
+ self.start_notifications = 0
+ self.monitor_type = 'sample'
+
+ def config_func(self, config):
+ for node in config.children:
+ key = node.key.lower()
+ val = node.values[0]
+
+ if key == 'compute_host':
+ self.host_name = val
+ elif key == 'control_ip':
+ self.control_ip = val
+ elif key == 'compute_ip':
+ self.compute_ip = val
+ elif key == 'compute_user':
+ self.compute_user = val
+ elif key == 'inspector_type':
+ self.inspector_type = val
+ elif key == 'os_auth_url':
+ self.os_auth_url = val
+ elif key == 'os_username':
+ self.os_username = val
+ elif key == 'os_password':
+ self.os_password = val
+ elif key == 'os_project_name':
+ self.os_project_name = val
+ elif key == 'os_user_domain_name':
+ self.os_user_domain_name = val
+ elif key == 'os_user_domain_id':
+ self.os_user_domain_id = val
+ elif key == 'os_project_domain_name':
+ self.os_project_domain_name = val
+ elif key == 'os_project_domain_id':
+ self.os_project_domain_id = val
+ else:
+ collectd.info('Unknown config key "%s"' % key)
+
+ def init_collectd(self):
+ write_debug("Compute node collectd monitor start at %s\n\n" % datetime.now().isoformat(), "w", self.compute_user)
+
+ if self.inspector_type == 'sample':
+ self.inspector_url = ('http://%s:12345/events' % self.control_ip)
+ elif self.inspector_type == 'congress':
+ loader = loading.get_plugin_loader('password')
+ self.auth = loader.load_from_options(auth_url=self.os_auth_url,
+ username=self.os_username,
+ password=self.os_password,
+ project_name=self.os_project_name,
+ user_domain_name=self.os_user_domain_name,
+ user_domain_id=self.os_user_domain_id,
+ project_domain_name=self.os_project_domain_name,
+ project_domain_id=self.os_project_domain_id)
+ self.sess=session.Session(auth=self.auth)
+ congress = client.Client(session=self.sess, service_type='policy')
+ ds = congress.list_datasources()['results']
+ doctor_ds = next((item for item in ds if item['driver'] == 'doctor'),
+ None)
+
+ congress_endpoint = congress.httpclient.get_endpoint(auth=self.auth)
+ self.inspector_url = ('%s/v1/data-sources/%s/tables/events/rows' %
+ (congress_endpoint, doctor_ds['id']))
+ else:
+ sys.exit()
+ self.start_notifications = 1
+
+
+ def notify_inspector(self):
+ event_type = "compute.host.down"
+ payload = [
+ {
+ 'id': ("monitor_%s_id1" % self.monitor_type),
+ 'time': datetime.now().isoformat(),
+ 'type': event_type,
+ 'details': {
+ 'hostname': self.host_name,
+ 'status': 'down',
+ 'monitor': ("monitor_%s" % self.monitor_type),
+ 'monitor_event_id': ("monitor_%s_event1" % self.monitor_type)
+ },
+ },
+ ]
+ data = json.dumps(payload)
+ self.inspector_notified = 1
+
+ if self.inspector_type == 'sample':
+ headers = {'content-type': 'application/json'}
+ try:
+ requests.post(self.inspector_url, data=data, headers=headers)
+ except ConnectionError as err:
+ print err
+ elif self.inspector_type == 'congress':
+ # TODO(umar) enhance for token expiry case
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ 'X-Auth-Token': self.sess.get_token()
+ }
+ requests.put(self.inspector_url, data=data, headers=headers)
+
+
+ def handle_notif(self, notification, data=None):
+ if (notification.severity == collectd.NOTIF_FAILURE or
+ notification.severity == collectd.NOTIF_WARNING):
+ if (self.start_notifications == 1 and self.inspector_notified == 0):
+ write_debug("Received down notification: doctor monitor detected at %s\n" % time.time(), "a", self.compute_user)
+ self.notify_inspector()
+
+ elif notification.severity == collectd.NOTIF_OKAY:
+ collectd.info("Interface status: UP again %s\n" % time.time())
+ else:
+ collectd.info("Unknown notification severity %s\n" % notification.severity)
+
+
+monitor = DoctorMonitorCollectd()
+
+collectd.register_config(monitor.config_func)
+collectd.register_init(monitor.init_collectd)
+collectd.register_notification(monitor.handle_notif)
--- /dev/null
+#!/bin/bash
+
+function start_monitor_sample {
+ cp $TOP_DIR/lib/monitors/sample/monitor.py $TOP_DIR/monitor.py
+ pgrep -f "python monitor.py" && return 0
+ sudo -E python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" "$INSPECTOR_TYPE" \
+ > monitor.log 2>&1 &
+}
+
+function stop_monitor_sample {
+ pgrep -f "python monitor.py" || return 0
+ sudo kill $(pgrep -f "python monitor.py")
+}
+
+function cleanup_monitor_sample {
+ rm monitor.py
+ return
+}
CI_DEBUG = os.getenv('CI_DEBUG')
+ logging.basicConfig(filemode='w')
self.logger = logging.getLogger(logger_name)
self.logger.propagate = 0
self.logger.setLevel(logging.DEBUG)
file_handler.setLevel(logging.DEBUG)
self.logger.addHandler(file_handler)
-
def getLogger(self):
return self.logger
-
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+import os
+from os.path import isfile, join
import sys
+from alarm import Alarm
import config
+from consumer import get_consumer
from image import Image
+from instance import Instance
+from inspector import get_inspector
+from installer import get_installer
import logger as doctor_log
+from user import User
+from network import Network
+from monitor import get_monitor
LOG = doctor_log.Logger('doctor').getLogger()
def __init__(self, conf):
self.conf = conf
- self.image = Image(self.conf)
+ self.image = Image(self.conf, LOG)
+ self.user = User(self.conf, LOG)
+ self.network = Network(self.conf, LOG)
+ self.instance = Instance(self.conf, LOG)
+ self.alarm = Alarm(self.conf, LOG)
+ self.inspector = get_inspector(self.conf, LOG)
+ self.monitor = get_monitor(self.conf,
+ self.inspector.get_inspector_url(),
+ LOG)
+ self.consumer = get_consumer(self.conf, LOG)
+ self.installer = get_installer(self.conf, LOG)
+
+ def setup(self):
+ # prepare the cloud env
+ self.installer.setup()
+
+ # preparing VM image...
+ self.image.create()
+
+ # creating test user...
+ self.user.create()
+ self.user.update_quota()
+
+ # creating VM...
+ self.network.create()
+ self.instance.create()
+ self.instance.wait_for_vm_launch()
+
+ # creating alarm...
+ self.alarm.create()
+
+ # starting doctor sample components...
+ self.inspector.start()
+ self.monitor.start()
+ self.consumer.start()
def run(self):
"""run doctor test"""
try:
LOG.info('doctor test starting.......')
- # prepare the cloud env
- # preparing VM image...
- self.image.create()
-
- # creating test user...
-
- # creating VM...
-
- # creating alarm...
-
- # starting doctor sample components...
+ self.setup()
# injecting host failure...
+ # NOTE (umar) add INTERFACE_NAME logic to host injection
# verify the test results
+ # NOTE (umar) copy remote monitor.log file when monitor=collectd
+
except Exception as e:
LOG.error('doctor test failed, Exception=%s' % e)
sys.exit(1)
finally:
- self.image.delete()
+ self.cleanup()
+
+ def cleanup(self):
+ self.alarm.delete()
+ self.instance.delete()
+ self.network.delete()
+ self.image.delete()
+ self.inspector.stop()
+ self.user.delete()
+ self.monitor.stop()
+ self.consumer.stop()
+ self.installer.cleanup()
def main():
"""doctor main"""
- conf = config.prepare_conf()
+ doctor_root_dir = os.path.dirname(sys.path[0])
+ config_file_dir = '{0}/{1}'.format(doctor_root_dir, 'etc/')
+ config_files = [join(config_file_dir, f) for f in os.listdir(config_file_dir)
+ if isfile(join(config_file_dir, f))]
+
+ conf = config.prepare_conf(args=sys.argv[1:],
+ config_files=config_files)
doctor = DoctorTest(conf)
doctor.run()
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+from oslo_utils import importutils
+
+OPTS = [
+ cfg.StrOpt('type',
+ default='sample',
+ choices=['sample', 'collectd'],
+ help='the type of doctor monitor component',
+ required=True),
+]
+
+
+_monitor_name_class_mapping = {
+ 'sample': 'monitor.sample.SampleMonitor',
+ 'collectd': 'monitor.collectd.CollectdMonitor'
+}
+
+def get_monitor(conf, inspector_url, log):
+ monitor_class = _monitor_name_class_mapping.get(conf.monitor.type)
+ return importutils.import_object(monitor_class, conf,
+ inspector_url, log)
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseMonitor(object):
+ """Monitor computer fault and report error to the inspector"""
+ def __init__(self, conf, inspector_url, log):
+ self.conf = conf
+ self.log = log
+ self.inspector_url = inspector_url
+
+ @abc.abstractmethod
+ def start(self):
+ pass
+
+ @abc.abstractmethod
+ def stop(self):
+ pass
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 NEC Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import os
+import socket
+import getpass
+import sys
+
+from identity_auth import get_session
+from os_clients import nova_client
+from monitor.base import BaseMonitor
+
+
+class CollectdMonitor(BaseMonitor):
+ def __init__(self, conf, inspector_url, log):
+ super(CollectdMonitor, self).__init__(conf, inspector_url, log)
+ self.top_dir = os.path.dirname(sys.path[0])
+ self.session = get_session()
+ self.nova = nova_client(conf.nova_version, self.session)
+ self.compute_hosts = self.nova.hypervisors.list(detailed=True)
+ for host in self.compute_hosts:
+ host_dict = host.__dict__
+ self.compute_host = host_dict['hypervisor_hostname']
+ self.compute_ip = host_dict['host_ip']
+ tmp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ tmp_sock.connect(("8.8.8.8", 80))
+
+ ## control_ip is the IP of primary interface of control node i.e.
+ ## eth0, eno1. It is used by collectd monitor to communicate with
+ ## sample inspector.
+ ## TODO (umar) see if mgmt IP of control is a better option. Also
+ ## primary interface may not be the right option
+ self.control_ip = tmp_sock.getsockname()[0]
+ self.compute_user = getpass.getuser()
+ self.interface_name = os.environ.get('INTERFACE_NAME') or ''
+ self.inspector_type = os.environ.get('INSPECTOR_TYPE', 'sample')
+ self.auth_url = os.environ.get('OS_AUTH_URL')
+ self.username = os.environ.get('OS_USERNAME')
+ self.password = os.environ.get('OS_PASSWORD')
+ self.project_name = os.environ.get('OS_PROJECT_NAME')
+ self.user_domain_name = os.environ.get('OS_USER_DOMAIN_NAME') or 'default'
+ self.user_domain_id = os.environ.get('OS_USER_DOMAIN_ID')
+ self.project_domain_name = os.environ.get('OS_PROJECT_DOMAIN_NAME') or 'default'
+ self.project_domain_id = os.environ.get('OS_PROJECT_DOMAIN_ID')
+ self.ssh_opts_cpu = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+
+ def start(self):
+ self.log.info("Collectd monitor start.........")
+ f = open("%s/tests/collectd.conf" % self.top_dir, 'w')
+ collectd_conf_file = """
+Hostname %s
+FQDNLookup false
+Interval 1
+MaxReadInterval 2
+
+<LoadPlugin python>
+Globals true
+</LoadPlugin>
+LoadPlugin ovs_events
+LoadPlugin logfile
+
+<Plugin logfile>
+ File \"/var/log/collectd.log\"
+ Timestamp true
+ LogLevel \"info\"
+</Plugin>
+
+<Plugin python>
+ ModulePath \"/home/%s\"
+ LogTraces true
+ Interactive false
+ Import \"collectd_plugin\"
+ <Module \"collectd_plugin\">
+ control_ip \"%s\"
+ compute_ip \"%s\"
+ compute_host \"%s\"
+ compute_user \"%s\"
+ inspector_type \"%s\"
+ os_auth_url \"%s\"
+ os_username \"%s\"
+ os_password \"%s\"
+ os_project_name \"%s\"
+ os_user_domain_name \"%s\"
+ os_user_domain_id \"%s\"
+ os_project_domain_name \"%s\"
+ os_project_domain_id \"%s\"
+ </Module>
+</Plugin>
+
+<Plugin ovs_events>
+ Port 6640
+ Socket \"/var/run/openvswitch/db.sock\"
+ Interfaces \"@INTERFACE_NAME@\"
+ SendNotification true
+ DispatchValues false
+</Plugin>
+ """ % (self.compute_host, self.compute_user, self.control_ip, self.compute_ip, self.compute_host, self.compute_user,
+ self.inspector_type, self.auth_url, self.username, self.password, self.project_name, self.user_domain_name,
+ self.user_domain_id, self.project_domain_name, self.project_domain_id)
+ f.write(collectd_conf_file)
+ f.close()
+
+ os.system(" scp %s %s/tests/collectd.conf %s@%s: " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip))
+ self.log.info("after first scp")
+ ## @TODO (umar) Always assuming that the interface is assigned an IP if
+ ## interface name is not provided. See if there is a better approach
+ os.system(""" ssh %s %s@%s \"if [ -n \"%s\" ]; then
+ dev=%s
+ else
+ dev=\$(sudo ip a | awk '/ %s\//{print \$NF}')
+ fi
+ sed -i -e \"s/@INTERFACE_NAME@/\$dev/\" collectd.conf
+ collectd_conf=/opt/collectd/etc/collectd.conf
+ if [ -e \$collectd_conf ]; then
+ sudo cp \$collectd_conf \${collectd_conf}-doctor-saved
+ else
+ sudo touch \${collectd_conf}-doctor-created
+ fi
+ sudo mv collectd.conf /opt/collectd/etc/collectd.conf\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip, self.interface_name, self.interface_name, self.compute_ip))
+ self.log.info("after first ssh")
+ os.system(" scp %s %s/tests/lib/monitors/collectd/collectd_plugin.py %s@%s:collectd_plugin.py " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip))
+ self.log.info("after sec scp")
+ os.system(" ssh %s %s@%s \"sudo pkill collectd; sudo /opt/collectd/sbin/collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
+ self.log.info("after sec ssh")
+
+ def stop(self):
+ os.system(" ssh %s %s@%s \"sudo pkill collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
+
+ def cleanup(self):
+ os.system(""" ssh %s %s@%s \"
+ collectd_conf=/opt/collectd/etc/collectd.conf
+ if [ -e \"\${collectd_conf}-doctor-created\" ]; then
+ sudo rm \"\${collectd_conf}-doctor-created\"
+ sudo rm \$collectd_conf
+ elif [ -e \"\${collectd_conf}-doctor-saved\" ]; then
+ sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf
+ sudo rm \"\${collectd_conf}-doctor-saved\"
+ fi\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
+ os.remove("%s/tests/collectd.conf" % self.top_dir)
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from datetime import datetime
+import json
+import requests
+import socket
+from threading import Thread
+import time
+
+from identity_auth import get_session
+from os_clients import nova_client
+from monitor.base import BaseMonitor
+
+
+class SampleMonitor(BaseMonitor):
+ event_type = "compute.host.down"
+
+ def __init__(self, conf, inspector_url, log):
+ super(SampleMonitor, self).__init__(conf, inspector_url, log)
+ self.session = get_session()
+ self.nova = nova_client(conf.nova_version, self.session)
+ self.hosts = self.nova.hypervisors.list(detailed=True)
+ self.pingers = []
+
+ def start(self):
+ self.log.info('sample monitor start......')
+ for host in self.hosts:
+ host_dict = host.__dict__
+ host_name = host_dict['hypervisor_hostname']
+ host_ip = host_dict['host_ip']
+ pinger = Pinger(host_name, host_ip, self, self.log)
+ pinger.start()
+ self.pingers.append(pinger)
+
+ def stop(self):
+ self.log.info('sample monitor stop......')
+ for pinger in self.pingers:
+ pinger.stop()
+ pinger.join()
+ del self.pingers
+
+ def report_error(self, hostname):
+ self.log.info('sample monitor report error......')
+ data = [
+ {
+ 'id': 'monitor_sample_id1',
+ 'time': datetime.now().isoformat(),
+ 'type': self.event_type,
+ 'details': {
+ 'hostname': hostname,
+ 'status': 'down',
+ 'monitor': 'monitor_sample',
+ 'monitor_event_id': 'monitor_sample_event1'
+ },
+ },
+ ]
+
+ auth_token = self.session.get_token() if \
+ self.conf.inspector.type != 'sample' else None
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ 'X-Auth-Token': auth_token,
+ }
+
+ url = '%s%s' % (self.inspector_url, 'events') \
+ if self.inspector_url.endswith('/') else \
+ '%s%s' % (self.inspector_url, '/events')
+ requests.put(url, data=json.dumps(data), headers=headers)
+
+
+class Pinger(Thread):
+ interval = 0.1 # second
+ timeout = 0.1 # second
+ ICMP_ECHO_MESSAGE = bytes([0x08, 0x00, 0xf7, 0xff, 0x00, 0x00, 0x00, 0x00])
+
+ def __init__(self, host_name, host_ip, monitor, log):
+ Thread.__init__(self)
+ self.monitor = monitor
+ self.hostname = host_name
+ self.ip_addr = host_ip or socket.gethostbyname(self.hostname)
+ self.log = log
+ self._stopped = False
+
+ def run(self):
+ self.log.info("Starting Pinger host_name(%s), host_ip(%s)"
+ % (self.hostname, self.ip_addr))
+
+ sock = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+ socket.IPPROTO_ICMP)
+ sock.settimeout(self.timeout)
+ while True:
+ if self._stopped:
+ return
+ try:
+ sock.sendto(self.ICMP_ECHO_MESSAGE, (self.ip_addr, 0))
+ sock.recv(4096)
+ except socket.timeout:
+ self.log.info("doctor monitor detected at %s" % time.time())
+ self.monitor.report_error(self.hostname)
+ self.log.info("ping timeout, quit monitoring...")
+ self._stopped = True
+ return
+ time.sleep(self.interval)
+
+ def stop(self):
+ self.log.info("Stopping Pinger host_name(%s), host_ip(%s)"
+ % (self.hostname, self.ip_addr))
+ self._stopped = True
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import neutron_client
+
+
+OPTS = [
+ cfg.StrOpt('net_name',
+ default='doctor_net',
+ help='the name of test net',
+ required=True),
+ cfg.StrOpt('net_cidr',
+ default='192.168.168.0/24',
+ help='the cidr of test subnet',
+ required=True),
+]
+
+
+class Network(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.auth = get_identity_auth(username=self.conf.doctor_user,
+ password=self.conf.doctor_passwd,
+ project=self.conf.doctor_project)
+ self.neutron = neutron_client(get_session(auth=self.auth))
+ self.net = None
+ self.subnet = None
+
+ def create(self):
+ self.log.info('network create start.......')
+ net_name = self.conf.net_name
+ networks = self.neutron.list_networks(name=net_name)['networks']
+ self.net = networks[0] if networks \
+ else self.neutron.create_network(
+ {'network': {'name': net_name}})['network']
+ self.log.info('network create end.......')
+
+ self.log.info('subnet create start.......')
+ subnets = self.neutron.list_subnets(network_id=self.net['id'])['subnets']
+ subnet_param = {'name': net_name, 'network_id': self.net['id'],
+ 'cidr': self.conf.net_cidr, 'ip_version': 4,
+ 'enable_dhcp': False}
+ self.subnet = subnets[0] if subnets \
+ else self.neutron.create_subnet(
+ {'subnet': subnet_param})['subnet']
+ self.log.info('subnet create end.......')
+
+ def delete(self):
+ self.log.info('subnet delete start.......')
+ if self.subnet:
+ self.neutron.delete_subnet(self.subnet['id'])
+ self.log.info('subnet delete end.......')
+
+ self.log.info('network delete start.......')
+ if self.net:
+ self.neutron.delete_network(self.net['id'])
+ self.log.info('network delete end.......')
##############################################################################\r
from oslo_config import cfg\r
\r
+import aodhclient.client as aodhclient\r
+from congressclient.v1 import client as congressclient\r
import glanceclient.client as glanceclient\r
+from keystoneclient.v2_0 import client as ks_client\r
+from neutronclient.v2_0 import client as neutronclient\r
+import novaclient.client as novaclient\r
\r
\r
OPTS = [\r
cfg.StrOpt('glance_version', default='2', help='glance version'),\r
+ cfg.StrOpt('nova_version', default='2.34', help='Nova version'),\r
+ cfg.StrOpt('aodh_version', default='2', help='aodh version'),\r
]\r
\r
\r
def glance_client(version, session):\r
return glanceclient.Client(version=version,\r
session=session)\r
+\r
+\r
+def keystone_client(session):\r
+ return ks_client.Client(session=session)\r
+\r
+\r
+def nova_client(version, session):\r
+ return novaclient.Client(version=version,\r
+ session=session)\r
+\r
+\r
+def neutron_client(session):\r
+ return neutronclient.Client(session=session)\r
+\r
+\r
+def aodh_client(version, session):\r
+ return aodhclient.Client(version, session=session)\r
+\r
+\r
+def congress_client(session):\r
+ return congressclient.Client(session=session,\r
+ service_type='policy')\r
DOCTOR_PW=doctor
DOCTOR_PROJECT=doctor
DOCTOR_ROLE=_member_
-PROFILER_TYPE=${PROFILER_TYPE:-none}
+PROFILER_TYPE=${PROFILER_TYPE:-poc}
PYTHON_ENABLE=${PYTHON_ENABLE:-false}
TOP_DIR=$(cd $(dirname "$0") && pwd)
get_compute_host_info() {
# get computer host info which first VM boot in as admin user
COMPUTE_HOST=$(openstack $as_admin_user server show ${VM_BASENAME}1 |
- grep "OS-EXT-SRV-ATTR:host" | awk '{ print $4 }')
+ grep "OS-EXT-SRV-ATTR:host " | awk '{ print $4 }')
compute_host_in_undercloud=${COMPUTE_HOST%%.*}
die_if_not_set $LINENO COMPUTE_HOST "Failed to get compute hostname"
done
}
-start_monitor() {
- pgrep -f "python monitor.py" && return 0
- sudo -E python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" "$INSPECTOR_TYPE" \
- > monitor.log 2>&1 &
-}
-
-stop_monitor() {
- pgrep -f "python monitor.py" || return 0
- sudo kill $(pgrep -f "python monitor.py")
-}
-
start_consumer() {
pgrep -f "python consumer.py" && return 0
python consumer.py "$CONSUMER_PORT" > consumer.log 2>&1 &
echo "disabling network of compute host [$COMPUTE_HOST] for 3 mins..."
cat > disable_network.sh << 'END_TXT'
#!/bin/bash -x
-dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $NF}')
sleep 1
+if [ -n "@INTERFACE_NAME@" ]; then
+ dev=@INTERFACE_NAME@
+else
+ dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $NF}')
+fi
sudo ip link set $dev down
echo "doctor set link down at" $(date "+%s.%N")
sleep 180
sleep 1
END_TXT
sed -i -e "s/@COMPUTE_IP@/$COMPUTE_IP/" disable_network.sh
+ sed -i -e "s/@INTERFACE_NAME@/$INTERFACE_NAME/" disable_network.sh
chmod +x disable_network.sh
scp $ssh_opts_cpu disable_network.sh "$COMPUTE_USER@$COMPUTE_IP:"
ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'nohup ./disable_network.sh > disable_network.log 2>&1 &'
wait_consumer 60
#keep 'at' as the last keyword just before the value, and
#use regex to get value instead of the fixed column
+ if [ ! -f monitor.log ]; then
+ scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:monitor.log" .
+ fi
detected=$(grep "doctor monitor detected at" monitor.log |\
- sed -e "s/^.* at //")
+ sed -e "s/^.* at //" | tail -1)
notified=$(grep "doctor consumer notified at" consumer.log |\
sed -e "s/^.* at //" | tail -1)
export DOCTOR_PROFILER_T09=$(python -c \
"print(int(($notified-$relative_start)*1000))")
- python profiler-poc.py >doctor_profiler.log 2>&1
+ python profiler-poc.py > doctor_profiler.log 2>&1
fi
}
cleanup() {
set +e
echo "cleanup..."
- stop_monitor
stop_inspector
stop_consumer
unset_forced_down_hosts
+ stop_monitor
collect_logs
vms=$(openstack $as_doctor_user server list)
cleanup_installer
cleanup_inspector
+ cleanup_monitor
# NOTE: Temporal log printer.
for f in $(find . -name '*.log')
done
}
+setup_python_packages() {
+ sudo pip install flask==0.10.1
+ command -v openstack || sudo pip install python-openstackclient==2.3.0
+ command -v ceilometer || sudo pip install python-ceilometerclient==2.6.2
+ command -v congress || sudo pip install python-congressclient==1.5.0
+}
+
# Main process
-if $PYTHON_ENABLE; then
+if [[ $PYTHON_ENABLE == [Tt]rue ]]; then
+ which tox || sudo pip install tox
+ if [ -f /usr/bin/apt-get ]; then
+ sudo apt-get install -y python3-dev
+ elif [ -f /usr/bin/yum ] ; then
+ sudo yum install -y python3-devel
+ fi
+
cd $TOP_DIR
echo "executing tox..."
tox
trap cleanup EXIT
+setup_python_packages
+
source $TOP_DIR/functions-common
source $TOP_DIR/lib/installer
source $TOP_DIR/lib/inspector
+source $TOP_DIR/lib/monitor
+
+rm -f *.log
setup_installer
inject_failure
check_host_status "(DOWN|UNKNOWN)" 60
-calculate_notification_time
unset_forced_down_hosts
+calculate_notification_time
collect_logs
run_profiler
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+
+from identity_auth import get_session
+from os_clients import keystone_client
+from os_clients import nova_client
+
+
+OPTS = [
+ cfg.StrOpt('doctor_user',
+ default='doctor',
+ help='the name of test user',
+ required=True),
+ cfg.StrOpt('doctor_passwd',
+ default='doctor',
+ help='the password of test user',
+ required=True),
+ cfg.StrOpt('doctor_project',
+ default='doctor',
+ help='the name of test project',
+ required=True),
+ cfg.StrOpt('doctor_role',
+ default='_member_',
+ help='the role of test user',
+ required=True),
+ cfg.IntOpt('quota_instances',
+ default=os.environ.get('VM_COUNT', 1),
+ help='the quota of instances in test user',
+ required=True),
+ cfg.IntOpt('quota_cores',
+ default=os.environ.get('VM_COUNT', 1),
+ help='the quota of cores in test user',
+ required=True),
+]
+
+
+class User(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.keystone = \
+ keystone_client(get_session())
+ self.nova = \
+ nova_client(conf.nova_version, get_session())
+ self.users = {}
+ self.projects = {}
+ self.roles = {}
+ self.roles_for_user = {}
+ self.roles_for_admin = {}
+
+ def create(self):
+ """create test user, project and etc"""
+ self.log.info('user create start......')
+
+ self._create_project()
+ self._create_user()
+ self._create_role()
+ self._add_user_role_in_project(is_admin=False)
+ self._add_user_role_in_project(is_admin=True)
+
+ self.log.info('user create end......')
+
+ def _create_project(self):
+ """create test project"""
+ self.projects = {project.name: project
+ for project in self.keystone.tenants.list()}
+ if self.conf.doctor_project not in self.projects:
+ test_project = \
+ self.keystone.tenants.create(self.conf.doctor_project)
+ self.projects[test_project.name] = test_project
+
+ def _create_user(self):
+ """create test user"""
+ project = self.projects.get(self.conf.doctor_project)
+ self.users = {user.name: user for user in self.keystone.users.list()}
+ if self.conf.doctor_user not in self.users:
+ test_user = self.keystone.users.create(
+ self.conf.doctor_user,
+ password=self.conf.doctor_passwd,
+ tenant_id=project.id)
+ self.users[test_user.name] = test_user
+
+ def _create_role(self):
+ """create test role"""
+ self.roles = {role.name: role for role in self.keystone.roles.list()}
+ if self.conf.doctor_role not in self.roles:
+ test_role = self.keystone.roles.create(self.conf.doctor_role)
+ self.roles[test_role.name] = test_role
+
+ def _add_user_role_in_project(self, is_admin=False):
+ """add test user with test role in test project"""
+ project = self.projects.get(self.conf.doctor_project)
+
+ user_name = 'admin' if is_admin else self.conf.doctor_user
+ user = self.users.get(user_name)
+
+ role_name = 'admin' if is_admin else self.conf.doctor_role
+ role = self.roles.get(role_name)
+
+ roles_for_user = self.roles_for_admin \
+ if is_admin else self.roles_for_user
+
+ roles_for_user = \
+ {role.name: role for role in
+ self.keystone.roles.roles_for_user(user, tenant=project)}
+ if role_name not in roles_for_user:
+ self.keystone.roles.add_user_role(user, role, tenant=project)
+ roles_for_user[role_name] = role
+
+ def delete(self):
+ """delete the test user, project and role"""
+ self.log.info('user delete start......')
+
+ project = self.projects.get(self.conf.doctor_project)
+ user = self.users.get(self.conf.doctor_user)
+ role = self.roles.get(self.conf.doctor_role)
+
+ if project:
+ if 'admin' in self.roles_for_admin:
+ self.keystone.roles.remove_user_role(
+ self.users['admin'],
+ self.roles['admin'],
+ tenant=project)
+
+ if user:
+ if role and self.conf.doctor_role in self.roles_for_user:
+ self.keystone.roles.remove_user_role(
+ user, role, tenant=project)
+ self.keystone.roles.delete(role)
+ self.keystone.users.delete(user)
+
+ self.keystone.tenants.delete(project)
+ self.log.info('user delete end......')
+
+ def update_quota(self):
+ self.log.info('user quota update start......')
+ project = self.projects.get(self.conf.doctor_project)
+ user = self.users.get(self.conf.doctor_user)
+
+ if project and user:
+ self.quota = self.nova.quotas.get(project.id,
+ user_id=user.id)
+ if self.conf.quota_instances > self.quota.instances:
+ self.nova.quotas.update(project.id,
+ instances=self.conf.quota_instances,
+ user_id=user.id)
+ if self.conf.quota_cores > self.quota.cores:
+ self.nova.quotas.update(project.id,
+ cores=self.conf.quota_cores,
+ user_id=user.id)
+ self.log.info('user quota update end......')
+ else:
+ raise Exception('No project or role for update quota')
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import json
+import os
+import paramiko
+
+
+def load_json_file(full_path):
+ """Loads JSON from file
+ :param target_filename:
+ :return:
+ """
+ if not os.path.isfile(full_path):
+ raise Exception('File(%s) does not exist' % full_path)
+
+ with open(full_path, 'r') as file:
+ return json.load(file)
+
+
+def write_json_file(full_path, data):
+ """write JSON from file
+ :param target_filename:
+ :return:
+ """
+
+ with open(full_path, 'w+') as file:
+ file.write(json.dumps(data))
+
+
+class SSHClient(object):
+ def __init__(self, ip, username, password=None, pkey=None,
+ key_filename=None, log=None, look_for_keys=False,
+ allow_agent=False):
+ self.client = paramiko.SSHClient()
+ self.client.load_system_host_keys()
+ self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ self.client.connect(ip, username=username, password=password,
+ pkey=pkey, key_filename=key_filename,
+ look_for_keys=look_for_keys,
+ allow_agent=allow_agent)
+ self.log = log
+
+ def __del__(self):
+ self.client.close()
+
+ def ssh(self, command):
+ if self.log:
+ self.log.debug("Executing: %s" % command)
+ stdin, stdout, stderr = self.client.exec_command(command)
+ ret = stdout.channel.recv_exit_status()
+ output = list()
+ for line in stdout.read().splitlines():
+ output.append(line.decode('utf-8'))
+ if ret:
+ if self.log:
+ self.log.debug("*** FAILED to run command %s (%s)" % (command, ret))
+ raise Exception(
+ "Unable to run \ncommand: %s\nret: %s"
+ % (command, ret))
+ if self.log:
+ self.log.debug("*** SUCCESSFULLY run command %s" % command)
+ return ret, output
+
+ def scp(self, source, dest, method='put'):
+ if self.log:
+ self.log.info("Copy %s -> %s" % (source, dest))
+ ftp = self.client.open_sftp()
+ if method == 'put':
+ ftp.put(source, dest)
+ elif method == 'get':
+ ftp.get(source, dest)
+ ftp.close()
[tox]
minversion = 2.3.1
-envlist = verify
+envlist = py34
skipsdist = True
[testenv]
PROFILER_TYPE
PYTHON_ENABLE
CI_DEBUG
-
-[testenv:verify]
+ INSTALLER_TYPE
+ INSTALLER_IP
changedir = {toxinidir}/tests
commands = python main.py
+
Code Review / doctor.git / commitdiff