# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+import datetime
+import json
+import requests
+import time
+
+from doctor_tests.admin_tool import get_admin_tool
+from doctor_tests.app_manager import get_app_manager
from doctor_tests.common.utils import get_doctor_test_root_dir
from doctor_tests.identity_auth import get_identity_auth
from doctor_tests.identity_auth import get_session
+from doctor_tests.inspector import get_inspector
from doctor_tests.os_clients import keystone_client
from doctor_tests.os_clients import neutron_client
from doctor_tests.os_clients import nova_client
class Maintenance(object):
- def __init__(self, conf, log):
+ def __init__(self, trasport_url, conf, log):
self.conf = conf
self.log = log
self.keystone = keystone_client(
auth = get_identity_auth(project=self.conf.doctor_project)
self.neutron = neutron_client(get_session(auth=auth))
self.stack = Stack(self.conf, self.log)
+ if self.conf.admin_tool.type == 'sample':
+ self.admin_tool = get_admin_tool(trasport_url, self.conf, self.log)
+ self.endpoint = 'maintenance'
+ else:
+ self.endpoint = 'v1/maintenance'
+ self.app_manager = get_app_manager(self.stack, self.conf, self.log)
+ self.inspector = get_inspector(self.conf, self.log, trasport_url)
def get_external_network(self):
ext_net = None
ext_net = network['name']
break
if ext_net is None:
- raise Exception("externl network not defined")
+ raise Exception("external network not defined")
return ext_net
def setup_maintenance(self, user):
# need to be free before test
hvisors = self.nova.hypervisors.list(detailed=True)
prev_vcpus = 0
- prev_hostname = ""
+ prev_hostname = ''
self.log.info('checking hypervisors.......')
for hvisor in hvisors:
- vcpus = hvisor.__getattr__("vcpus")
- vcpus_used = hvisor.__getattr__("vcpus_used")
- hostname = hvisor.__getattr__("hypervisor_hostname")
+ vcpus = hvisor.__getattr__('vcpus')
+ vcpus_used = hvisor.__getattr__('vcpus_used')
+ hostname = hvisor.__getattr__('hypervisor_hostname')
if vcpus < 2:
- raise Exception('not enough vcpus on %s' % hostname)
+ raise Exception('not enough vcpus (%d) on %s' %
+ (vcpus, hostname))
if vcpus_used > 0:
- raise Exception('%d vcpus used on %s'
- % (vcpus_used, hostname))
+ if self.conf.test_case == 'all':
+ # VCPU might not yet be free after fault_management test
+ self.log.info('%d vcpus used on %s, retry...'
+ % (vcpus_used, hostname))
+ time.sleep(15)
+ hvisor = self.nova.hypervisors.get(hvisor.id)
+ vcpus_used = hvisor.__getattr__('vcpus_used')
+ if vcpus_used > 0:
+ raise Exception('%d vcpus used on %s'
+ % (vcpus_used, hostname))
if prev_vcpus != 0 and prev_vcpus != vcpus:
raise Exception('%d vcpus on %s does not match to'
'%d on %s'
parameters=parameters,
files=files)
+ if self.conf.admin_tool.type == 'sample':
+ self.admin_tool.start()
+ else:
+ # TBD Now we expect Fenix is running in self.conf.admin_tool.port
+ pass
+ self.app_manager.start()
+ self.inspector.start()
+
+ def start_maintenance(self):
+ self.log.info('start maintenance.......')
+ hvisors = self.nova.hypervisors.list(detailed=True)
+ maintenance_hosts = list()
+ for hvisor in hvisors:
+ hostname = hvisor.__getattr__('hypervisor_hostname')
+ maintenance_hosts.append(hostname)
+
+ url = ('http://%s:%s/%s' %
+ (self.conf.admin_tool.ip,
+ self.conf.admin_tool.port,
+ self.endpoint))
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json'}
+
+ retries = 12
+ while retries > 0:
+ # let's start maintenance 20sec from now, so projects will have
+ # time to ACK to it before that
+ maintenance_at = (datetime.datetime.utcnow() +
+ datetime.timedelta(seconds=30)
+ ).strftime('%Y-%m-%d %H:%M:%S')
+
+ data = {'state': 'MAINTENANCE',
+ 'maintenance_at': maintenance_at,
+ 'metadata': {'openstack_version': 'Rocky'},
+ 'workflow': 'default'}
+
+ if self.conf.admin_tool.type == 'sample':
+ data['hosts'] = maintenance_hosts
+ else:
+ data['hosts'] = []
+ try:
+ ret = requests.post(url, data=json.dumps(data),
+ headers=headers)
+ except:
+ if retries == 0:
+ raise Exception('admin tool did not respond in 120s')
+ else:
+ self.log.info('admin tool not ready, retry in 10s')
+ retries = retries - 1
+ time.sleep(10)
+ continue
+ break
+ if ret.status_code != 200:
+ raise Exception(ret.text)
+ return ret.json()['session_id']
+
+ def remove_maintenance_session(self, session_id):
+ self.log.info('remove maintenance session %s.......' % session_id)
+ url = ('http://%s:%s/%s/%s' %
+ (self.conf.admin_tool.ip,
+ self.conf.admin_tool.port,
+ self.endpoint,
+ session_id))
+
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json'}
+
+ ret = requests.delete(url, data=None, headers=headers)
+ if ret.status_code != 200:
+ raise Exception(ret.text)
+
+ def get_maintenance_state(self, session_id):
+ url = ('http://%s:%s/%s/%s' %
+ (self.conf.admin_tool.ip,
+ self.conf.admin_tool.port,
+ self.endpoint,
+ session_id))
+
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json'}
+ ret = requests.get(url, data=None, headers=headers)
+ if ret.status_code != 200:
+ raise Exception(ret.text)
+ return ret.json()['state']
+
+ def wait_maintenance_complete(self, session_id):
+ retries = 90
+ state = None
+ time.sleep(300)
+ while (state not in ['MAINTENANCE_DONE', 'MAINTENANCE_FAILED'] and
+ retries > 0):
+ time.sleep(10)
+ state = self.get_maintenance_state(session_id)
+ retries = retries - 1
+ self.remove_maintenance_session(session_id)
+ self.log.info('maintenance %s ended with state %s' %
+ (session_id, state))
+ if state == 'MAINTENANCE_FAILED':
+ raise Exception('maintenance %s failed' % session_id)
+ elif retries == 0:
+ raise Exception('maintenance %s not completed within 20min' %
+ session_id)
+
def cleanup_maintenance(self):
+ if self.conf.admin_tool.type == 'sample':
+ self.admin_tool.stop()
+ self.app_manager.stop()
+ self.inspector.stop()
self.log.info('stack delete start.......')
self.stack.delete()
Code Review / doctor.git / blobdiff