1 ##############################################################################
2 # Copyright (c) 2019 Nokia Corporation and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
14 from doctor_tests.admin_tool import get_admin_tool
15 from doctor_tests.app_manager import get_app_manager
16 from doctor_tests.common.utils import get_doctor_test_root_dir
17 from doctor_tests.identity_auth import get_identity_auth
18 from doctor_tests.identity_auth import get_session
19 from doctor_tests.inspector import get_inspector
20 from doctor_tests.os_clients import keystone_client
21 from doctor_tests.os_clients import neutron_client
22 from doctor_tests.os_clients import nova_client
23 from doctor_tests.stack import Stack
26 class Maintenance(object):
28 def __init__(self, trasport_url, conf, log):
31 self.admin_session = get_session()
32 self.keystone = keystone_client(
33 self.conf.keystone_version, get_session())
34 self.nova = nova_client(conf.nova_version, get_session())
35 auth = get_identity_auth(project=self.conf.doctor_project)
36 self.neutron = neutron_client(get_session(auth=auth))
37 self.stack = Stack(self.conf, self.log)
38 if self.conf.installer.type == "devstack":
39 self.endpoint_ip = trasport_url.split("@", 1)[1].split(":", 1)[0]
41 self.endpoint_ip = self.conf.admin_tool.ip
42 self.endpoint = "http://%s:12347/" % self.endpoint_ip
43 if self.conf.admin_tool.type == 'sample':
44 self.admin_tool = get_admin_tool(trasport_url, self.conf, self.log)
45 self.endpoint += 'maintenance'
47 self.endpoint += 'v1/maintenance'
48 self.app_manager = get_app_manager(self.stack, self.conf, self.log)
49 self.inspector = get_inspector(self.conf, self.log, trasport_url)
51 def get_external_network(self):
53 networks = self.neutron.list_networks()['networks']
54 for network in networks:
55 if network['router:external']:
56 ext_net = network['name']
59 raise Exception("external network not defined")
62 def setup_maintenance(self, user):
63 # each hypervisor needs to have same amount of vcpus and they
64 # need to be free before test
65 hvisors = self.nova.hypervisors.list(detailed=True)
68 self.log.info('checking hypervisors.......')
69 for hvisor in hvisors:
70 vcpus = hvisor.__getattr__('vcpus')
71 vcpus_used = hvisor.__getattr__('vcpus_used')
72 hostname = hvisor.__getattr__('hypervisor_hostname')
74 raise Exception('not enough vcpus (%d) on %s' %
77 if self.conf.test_case == 'all':
78 # VCPU might not yet be free after fault_management test
79 self.log.info('%d vcpus used on %s, retry...'
80 % (vcpus_used, hostname))
82 hvisor = self.nova.hypervisors.get(hvisor.id)
83 vcpus_used = hvisor.__getattr__('vcpus_used')
85 raise Exception('%d vcpus used on %s'
86 % (vcpus_used, hostname))
87 if prev_vcpus != 0 and prev_vcpus != vcpus:
88 raise Exception('%d vcpus on %s does not match to'
91 prev_vcpus, prev_hostname))
93 prev_hostname = hostname
95 # maintenance flavor made so that 2 instances take whole node
96 flavor_vcpus = int(vcpus / 2)
97 compute_nodes = len(hvisors)
98 amount_actstdby_instances = 2
99 amount_noredundancy_instances = 2 * compute_nodes - 2
100 self.log.info('testing %d computes with %d vcpus each'
101 % (compute_nodes, vcpus))
102 self.log.info('testing %d actstdby and %d noredundancy instances'
103 % (amount_actstdby_instances,
104 amount_noredundancy_instances))
105 max_instances = (amount_actstdby_instances +
106 amount_noredundancy_instances)
107 max_cores = compute_nodes * vcpus
109 user.update_quota(max_instances, max_cores)
111 test_dir = get_doctor_test_root_dir()
112 template_file = '{0}/{1}'.format(test_dir, 'maintenance_hot_tpl.yaml')
113 files, template = self.stack.get_hot_tpl(template_file)
115 ext_net = self.get_external_network()
117 parameters = {'ext_net': ext_net,
118 'flavor_vcpus': flavor_vcpus,
119 'maint_image': self.conf.image_name,
120 'nonha_intances': amount_noredundancy_instances,
121 'ha_intances': amount_actstdby_instances}
123 self.log.info('creating maintenance stack.......')
124 self.log.info('parameters: %s' % parameters)
126 self.stack.create('doctor_test_maintenance',
128 parameters=parameters,
131 if self.conf.admin_tool.type == 'sample':
132 self.admin_tool.start()
134 # TBD Now we expect Fenix is running in self.conf.admin_tool.port
136 # Inspector before app_manager, as floating ip might come late
137 self.inspector.start()
138 self.app_manager.start()
140 def start_maintenance(self):
141 self.log.info('start maintenance.......')
142 hvisors = self.nova.hypervisors.list(detailed=True)
143 maintenance_hosts = list()
144 for hvisor in hvisors:
145 hostname = hvisor.__getattr__('hypervisor_hostname')
146 maintenance_hosts.append(hostname)
149 'Content-Type': 'application/json',
150 'Accept': 'application/json'}
151 if self.conf.admin_tool.type == 'fenix':
152 headers['X-Auth-Token'] = self.admin_session.get_token()
153 self.log.info('url %s headers %s' % (url, headers))
157 # let's start maintenance 20sec from now, so projects will have
158 # time to ACK to it before that
159 maintenance_at = (datetime.datetime.utcnow() +
160 datetime.timedelta(seconds=30)
161 ).strftime('%Y-%m-%d %H:%M:%S')
163 data = {'state': 'MAINTENANCE',
164 'maintenance_at': maintenance_at,
165 'metadata': {'openstack_version': 'Train'}}
167 if self.conf.app_manager.type == 'vnfm':
168 data['workflow'] = 'vnf'
170 data['workflow'] = 'default'
172 if self.conf.admin_tool.type == 'sample':
173 data['hosts'] = maintenance_hosts
177 ret = requests.post(url, data=json.dumps(data),
181 raise Exception('admin tool did not respond in 120s')
183 self.log.info('admin tool not ready, retry in 10s')
184 retries = retries - 1
189 raise Exception("admin tool did not respond")
190 if ret.status_code != 200:
191 raise Exception(ret.text)
192 return ret.json()['session_id']
194 def remove_maintenance_session(self, session_id):
195 self.log.info('remove maintenance session %s.......' % session_id)
197 url = ('%s/%s' % (self.endpoint, session_id))
200 'Content-Type': 'application/json',
201 'Accept': 'application/json'}
203 if self.conf.admin_tool.type == 'fenix':
204 headers['X-Auth-Token'] = self.admin_session.get_token()
206 ret = requests.delete(url, data=None, headers=headers)
207 if ret.status_code != 200:
208 raise Exception(ret.text)
210 def get_maintenance_state(self, session_id):
212 url = ('%s/%s' % (self.endpoint, session_id))
215 'Content-Type': 'application/json',
216 'Accept': 'application/json'}
218 if self.conf.admin_tool.type == 'fenix':
219 headers['X-Auth-Token'] = self.admin_session.get_token()
221 ret = requests.get(url, data=None, headers=headers)
222 if ret.status_code != 200:
223 raise Exception(ret.text)
224 return ret.json()['state']
226 def wait_maintenance_complete(self, session_id):
230 while (state not in ['MAINTENANCE_DONE', 'MAINTENANCE_FAILED'] and
233 state = self.get_maintenance_state(session_id)
234 retries = retries - 1
235 self.remove_maintenance_session(session_id)
236 self.log.info('maintenance %s ended with state %s' %
238 if state == 'MAINTENANCE_FAILED':
239 raise Exception('maintenance %s failed' % session_id)
241 raise Exception('maintenance %s not completed within 20min' %
244 def cleanup_maintenance(self):
245 if self.conf.admin_tool.type == 'sample':
246 self.admin_tool.stop()
247 self.app_manager.stop()
248 self.inspector.stop()
249 self.log.info('stack delete start.......')