add0f2437e40b7e13db2041ec6b425f6daf8d466
[functest.git] / functest / opnfv_tests / openstack / rally / rally.py
1 #!/usr/bin/env python
2 #
3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10
11 """Rally testcases implementation."""
12
13 from __future__ import division
14
15 import json
16 import logging
17 import os
18 import re
19 import subprocess
20 import time
21 import uuid
22
23 import pkg_resources
24 import prettytable
25 import yaml
26
27 from functest.core import testcase
28 from functest.energy import energy
29 from functest.opnfv_tests.openstack.snaps import snaps_utils
30 from functest.opnfv_tests.openstack.tempest import conf_utils
31 from functest.utils.constants import CONST
32 from functest.utils import env
33
34 from snaps.config.flavor import FlavorConfig
35 from snaps.config.image import ImageConfig
36 from snaps.config.network import NetworkConfig, SubnetConfig
37 from snaps.config.router import RouterConfig
38
39 from snaps.openstack.create_flavor import OpenStackFlavor
40 from snaps.openstack.utils import deploy_utils
41
42 LOGGER = logging.getLogger(__name__)
43
44
45 class RallyBase(testcase.TestCase):
46     """Base class form Rally testcases implementation."""
47
48     # pylint: disable=too-many-instance-attributes
49     TESTS = ['authenticate', 'glance', 'ceilometer', 'cinder', 'heat',
50              'keystone', 'neutron', 'nova', 'quotas', 'vm', 'all']
51     GLANCE_IMAGE_NAME = getattr(CONST, 'openstack_image_name')
52     GLANCE_IMAGE_FILENAME = getattr(CONST, 'openstack_image_file_name')
53     GLANCE_IMAGE_PATH = os.path.join(getattr(CONST, 'dir_functest_images'),
54                                      GLANCE_IMAGE_FILENAME)
55     GLANCE_IMAGE_FORMAT = getattr(CONST, 'openstack_image_disk_format')
56     GLANCE_IMAGE_USERNAME = getattr(CONST, 'openstack_image_username')
57     GLANCE_IMAGE_EXTRA_PROPERTIES = getattr(CONST,
58                                             'openstack_extra_properties', {})
59     FLAVOR_NAME = getattr(CONST, 'rally_flavor_name')
60     FLAVOR_ALT_NAME = getattr(CONST, 'rally_flavor_alt_name')
61     FLAVOR_RAM = 512
62     FLAVOR_RAM_ALT = 1024
63     FLAVOR_EXTRA_SPECS = getattr(CONST, 'flavor_extra_specs', None)
64     if FLAVOR_EXTRA_SPECS:
65         FLAVOR_RAM = 1024
66         FLAVOR_RAM_ALT = 2048
67
68     RALLY_DIR = pkg_resources.resource_filename(
69         'functest', 'opnfv_tests/openstack/rally')
70     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
71         'functest', 'opnfv_tests/openstack/rally/scenario')
72     TEMPLATE_DIR = pkg_resources.resource_filename(
73         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
74     SUPPORT_DIR = pkg_resources.resource_filename(
75         'functest', 'opnfv_tests/openstack/rally/scenario/support')
76     USERS_AMOUNT = 2
77     TENANTS_AMOUNT = 3
78     ITERATIONS_AMOUNT = 10
79     CONCURRENCY = 4
80     RESULTS_DIR = os.path.join(getattr(CONST, 'dir_results'), 'rally')
81     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.txt")
82     TEMP_DIR = os.path.join(RALLY_DIR, "var")
83
84     RALLY_PRIVATE_NET_NAME = getattr(CONST, 'rally_network_name')
85     RALLY_PRIVATE_SUBNET_NAME = getattr(CONST, 'rally_subnet_name')
86     RALLY_PRIVATE_SUBNET_CIDR = getattr(CONST, 'rally_subnet_cidr')
87     RALLY_ROUTER_NAME = getattr(CONST, 'rally_router_name')
88
89     def __init__(self, **kwargs):
90         """Initialize RallyBase object."""
91         super(RallyBase, self).__init__(**kwargs)
92         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
93         self.guid = '-' + str(uuid.uuid4())
94         self.creators = []
95         self.mode = ''
96         self.summary = []
97         self.scenario_dir = ''
98         self.image_name = None
99         self.ext_net_name = None
100         self.priv_net_id = None
101         self.flavor_name = None
102         self.flavor_alt_name = None
103         self.smoke = None
104         self.test_name = None
105         self.start_time = None
106         self.result = None
107         self.details = None
108         self.compute_cnt = 0
109
110     def _build_task_args(self, test_file_name):
111         """Build arguments for the Rally task."""
112         task_args = {'service_list': [test_file_name]}
113         task_args['image_name'] = self.image_name
114         task_args['flavor_name'] = self.flavor_name
115         task_args['flavor_alt_name'] = self.flavor_alt_name
116         task_args['glance_image_location'] = self.GLANCE_IMAGE_PATH
117         task_args['glance_image_format'] = self.GLANCE_IMAGE_FORMAT
118         task_args['tmpl_dir'] = self.TEMPLATE_DIR
119         task_args['sup_dir'] = self.SUPPORT_DIR
120         task_args['users_amount'] = self.USERS_AMOUNT
121         task_args['tenants_amount'] = self.TENANTS_AMOUNT
122         task_args['use_existing_users'] = False
123         task_args['iterations'] = self.ITERATIONS_AMOUNT
124         task_args['concurrency'] = self.CONCURRENCY
125         task_args['smoke'] = self.smoke
126
127         ext_net = self.ext_net_name
128         if ext_net:
129             task_args['floating_network'] = str(ext_net)
130         else:
131             task_args['floating_network'] = ''
132
133         net_id = self.priv_net_id
134         if net_id:
135             task_args['netid'] = str(net_id)
136         else:
137             task_args['netid'] = ''
138
139         return task_args
140
141     def _prepare_test_list(self, test_name):
142         """Build the list of test cases to be executed."""
143         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
144         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
145                                           test_yaml_file_name)
146
147         if not os.path.exists(scenario_file_name):
148             scenario_file_name = os.path.join(self.scenario_dir,
149                                               test_yaml_file_name)
150
151             if not os.path.exists(scenario_file_name):
152                 raise Exception("The scenario '%s' does not exist."
153                                 % scenario_file_name)
154
155         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
156         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
157
158         if not os.path.exists(self.TEMP_DIR):
159             os.makedirs(self.TEMP_DIR)
160
161         self._apply_blacklist(scenario_file_name, test_file_name)
162         return test_file_name
163
164     @staticmethod
165     def get_task_id(cmd_raw):
166         """
167         Get task id from command rally result.
168
169         :param cmd_raw:
170         :return: task_id as string
171         """
172         taskid_re = re.compile('^Task +(.*): started$')
173         for line in cmd_raw.splitlines(True):
174             line = line.strip()
175             match = taskid_re.match(line)
176             if match:
177                 return match.group(1)
178         return None
179
180     @staticmethod
181     def task_succeed(json_raw):
182         """
183         Parse JSON from rally JSON results.
184
185         :param json_raw:
186         :return: Bool
187         """
188         rally_report = json.loads(json_raw)
189         for report in rally_report:
190             if report is None or report.get('result') is None:
191                 return False
192
193             for result in report.get('result'):
194                 if result is None or len(result.get('error')) > 0:
195                     return False
196
197         return True
198
199     def _migration_supported(self):
200         """Determine if migration is supported."""
201         if self.compute_cnt > 1:
202             return True
203
204         return False
205
206     @staticmethod
207     def get_cmd_output(proc):
208         """Get command stdout."""
209         result = ""
210         for line in proc.stdout:
211             result += line
212         return result
213
214     @staticmethod
215     def excl_scenario():
216         """Exclude scenario."""
217         black_tests = []
218         try:
219             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
220                 black_list_yaml = yaml.safe_load(black_list_file)
221
222             installer_type = env.get('INSTALLER_TYPE')
223             deploy_scenario = env.get('DEPLOY_SCENARIO')
224             if (bool(installer_type) and bool(deploy_scenario) and
225                     'scenario' in black_list_yaml.keys()):
226                 for item in black_list_yaml['scenario']:
227                     scenarios = item['scenarios']
228                     installers = item['installers']
229                     in_it = RallyBase.in_iterable_re
230                     if (in_it(deploy_scenario, scenarios) and
231                             in_it(installer_type, installers)):
232                         tests = item['tests']
233                         black_tests.extend(tests)
234         except Exception:  # pylint: disable=broad-except
235             LOGGER.debug("Scenario exclusion not applied.")
236
237         return black_tests
238
239     @staticmethod
240     def in_iterable_re(needle, haystack):
241         """
242         Check if given needle is in the iterable haystack, using regex.
243
244         :param needle: string to be matched
245         :param haystack: iterable of strings (optionally regex patterns)
246         :return: True if needle is eqial to any of the elements in haystack,
247                  or if a nonempty regex pattern in haystack is found in needle.
248         """
249         # match without regex
250         if needle in haystack:
251             return True
252
253         for pattern in haystack:
254             # match if regex pattern is set and found in the needle
255             if pattern and re.search(pattern, needle) is not None:
256                 return True
257
258         return False
259
260     def excl_func(self):
261         """Exclude functionalities."""
262         black_tests = []
263         func_list = []
264
265         try:
266             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
267                 black_list_yaml = yaml.safe_load(black_list_file)
268
269             if not self._migration_supported():
270                 func_list.append("no_migration")
271
272             if 'functionality' in black_list_yaml.keys():
273                 for item in black_list_yaml['functionality']:
274                     functions = item['functions']
275                     for func in func_list:
276                         if func in functions:
277                             tests = item['tests']
278                             black_tests.extend(tests)
279         except Exception:  # pylint: disable=broad-except
280             LOGGER.debug("Functionality exclusion not applied.")
281
282         return black_tests
283
284     def _apply_blacklist(self, case_file_name, result_file_name):
285         """Apply blacklist."""
286         LOGGER.debug("Applying blacklist...")
287         cases_file = open(case_file_name, 'r')
288         result_file = open(result_file_name, 'w')
289
290         black_tests = list(set(self.excl_func() +
291                                self.excl_scenario()))
292
293         if black_tests:
294             LOGGER.debug("Blacklisted tests: " + str(black_tests))
295
296         include = True
297         for cases_line in cases_file:
298             if include:
299                 for black_tests_line in black_tests:
300                     if re.search(black_tests_line,
301                                  cases_line.strip().rstrip(':')):
302                         include = False
303                         break
304                 else:
305                     result_file.write(str(cases_line))
306             else:
307                 if cases_line.isspace():
308                     include = True
309
310         cases_file.close()
311         result_file.close()
312
313     @staticmethod
314     def file_is_empty(file_name):
315         """Determine is a file is empty."""
316         try:
317             if os.stat(file_name).st_size > 0:
318                 return False
319         except Exception:  # pylint: disable=broad-except
320             pass
321
322         return True
323
324     def _run_task(self, test_name):
325         """Run a task."""
326         LOGGER.info('Starting test scenario "%s" ...', test_name)
327
328         task_file = os.path.join(self.RALLY_DIR, 'task.yaml')
329         if not os.path.exists(task_file):
330             LOGGER.error("Task file '%s' does not exist.", task_file)
331             raise Exception("Task file '%s' does not exist.", task_file)
332
333         file_name = self._prepare_test_list(test_name)
334         if self.file_is_empty(file_name):
335             LOGGER.info('No tests for scenario "%s"', test_name)
336             return
337
338         cmd = (["rally", "task", "start", "--abort-on-sla-failure", "--task",
339                 task_file, "--task-args",
340                 str(self._build_task_args(test_name))])
341         LOGGER.debug('running command: %s', cmd)
342
343         proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
344                                 stderr=subprocess.STDOUT)
345         output = self.get_cmd_output(proc)
346         task_id = self.get_task_id(output)
347
348         LOGGER.debug('task_id : %s', task_id)
349
350         if task_id is None:
351             LOGGER.error('Failed to retrieve task_id, validating task...')
352             cmd = (["rally", "task", "validate", "--task", task_file,
353                     "--task-args", str(self._build_task_args(test_name))])
354             LOGGER.debug('running command: %s', cmd)
355             proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
356                                     stderr=subprocess.STDOUT)
357             output = self.get_cmd_output(proc)
358             LOGGER.error("Task validation result:" + "\n" + output)
359             return
360
361         # check for result directory and create it otherwise
362         if not os.path.exists(self.RESULTS_DIR):
363             LOGGER.debug('%s does not exist, we create it.',
364                          self.RESULTS_DIR)
365             os.makedirs(self.RESULTS_DIR)
366
367         # get and save rally operation JSON result
368         cmd = (["rally", "task", "detailed", task_id])
369         LOGGER.debug('running command: %s', cmd)
370         proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
371                                 stderr=subprocess.STDOUT)
372         json_detailed = self.get_cmd_output(proc)
373         LOGGER.info('%s', json_detailed)
374
375         cmd = (["rally", "task", "results", task_id])
376         LOGGER.debug('running command: %s', cmd)
377         proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
378                                 stderr=subprocess.STDOUT)
379         json_results = self.get_cmd_output(proc)
380         self._append_summary(json_results, test_name)
381         report_json_name = 'opnfv-{}.json'.format(test_name)
382         report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name)
383         with open(report_json_dir, 'w') as r_file:
384             LOGGER.debug('saving json file')
385             r_file.write(json_results)
386
387         # write html report file
388         report_html_name = 'opnfv-{}.html'.format(test_name)
389         report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name)
390         cmd = (["rally", "task", "report", task_id, "--out", report_html_dir])
391         LOGGER.debug('running command: %s', cmd)
392         subprocess.Popen(cmd, stdout=subprocess.PIPE,
393                          stderr=subprocess.STDOUT)
394
395         # parse JSON operation result
396         if self.task_succeed(json_results):
397             LOGGER.info('Test scenario: "{}" OK.'.format(test_name) + "\n")
398         else:
399             LOGGER.info('Test scenario: "{}" Failed.'.format(test_name) + "\n")
400
401     def _append_summary(self, json_raw, test_name):
402         """Update statistics summary info."""
403         nb_tests = 0
404         nb_success = 0
405         overall_duration = 0.0
406
407         rally_report = json.loads(json_raw)
408         for report in rally_report:
409             if report.get('full_duration'):
410                 overall_duration += report.get('full_duration')
411
412             if report.get('result'):
413                 for result in report.get('result'):
414                     nb_tests += 1
415                     if not result.get('error'):
416                         nb_success += 1
417
418         scenario_summary = {'test_name': test_name,
419                             'overall_duration': overall_duration,
420                             'nb_tests': nb_tests,
421                             'nb_success': nb_success}
422         self.summary.append(scenario_summary)
423
424     def _prepare_env(self):
425         """Create resources needed by test scenarios."""
426         LOGGER.debug('Validating the test name...')
427         if self.test_name not in self.TESTS:
428             raise Exception("Test name '%s' is invalid" % self.test_name)
429
430         network_name = self.RALLY_PRIVATE_NET_NAME + self.guid
431         subnet_name = self.RALLY_PRIVATE_SUBNET_NAME + self.guid
432         router_name = self.RALLY_ROUTER_NAME + self.guid
433         self.image_name = self.GLANCE_IMAGE_NAME + self.guid
434         self.flavor_name = self.FLAVOR_NAME + self.guid
435         self.flavor_alt_name = self.FLAVOR_ALT_NAME + self.guid
436         self.ext_net_name = snaps_utils.get_ext_net_name(self.os_creds)
437         self.compute_cnt = snaps_utils.get_active_compute_cnt(self.os_creds)
438
439         LOGGER.debug("Creating image '%s'...", self.image_name)
440         image_creator = deploy_utils.create_image(
441             self.os_creds, ImageConfig(
442                 name=self.image_name,
443                 image_file=self.GLANCE_IMAGE_PATH,
444                 img_format=self.GLANCE_IMAGE_FORMAT,
445                 image_user=self.GLANCE_IMAGE_USERNAME,
446                 public=True,
447                 extra_properties=self.GLANCE_IMAGE_EXTRA_PROPERTIES))
448         if image_creator is None:
449             raise Exception("Failed to create image")
450         self.creators.append(image_creator)
451
452         LOGGER.debug("Creating network '%s'...", network_name)
453
454         rally_network_type = getattr(CONST, 'rally_network_type', None)
455         rally_physical_network = getattr(CONST, 'rally_physical_network', None)
456         rally_segmentation_id = getattr(CONST, 'rally_segmentation_id', None)
457
458         network_creator = deploy_utils.create_network(
459             self.os_creds, NetworkConfig(
460                 name=network_name,
461                 shared=True,
462                 network_type=rally_network_type,
463                 physical_network=rally_physical_network,
464                 segmentation_id=rally_segmentation_id,
465                 subnet_settings=[SubnetConfig(
466                     name=subnet_name,
467                     cidr=self.RALLY_PRIVATE_SUBNET_CIDR)]))
468         if network_creator is None:
469             raise Exception("Failed to create private network")
470         self.priv_net_id = network_creator.get_network().id
471         self.creators.append(network_creator)
472
473         LOGGER.debug("Creating router '%s'...", router_name)
474         router_creator = deploy_utils.create_router(
475             self.os_creds, RouterConfig(
476                 name=router_name,
477                 external_gateway=self.ext_net_name,
478                 internal_subnets=[subnet_name]))
479         if router_creator is None:
480             raise Exception("Failed to create router")
481         self.creators.append(router_creator)
482
483         LOGGER.debug("Creating flavor '%s'...", self.flavor_name)
484         flavor_creator = OpenStackFlavor(
485             self.os_creds, FlavorConfig(
486                 name=self.flavor_name, ram=self.FLAVOR_RAM, disk=1, vcpus=1,
487                 metadata=self.FLAVOR_EXTRA_SPECS))
488         if flavor_creator is None or flavor_creator.create() is None:
489             raise Exception("Failed to create flavor")
490         self.creators.append(flavor_creator)
491
492         LOGGER.debug("Creating flavor '%s'...", self.flavor_alt_name)
493         flavor_alt_creator = OpenStackFlavor(
494             self.os_creds, FlavorConfig(
495                 name=self.flavor_alt_name, ram=self.FLAVOR_RAM_ALT, disk=1,
496                 vcpus=1, metadata=self.FLAVOR_EXTRA_SPECS))
497         if flavor_alt_creator is None or flavor_alt_creator.create() is None:
498             raise Exception("Failed to create flavor")
499         self.creators.append(flavor_alt_creator)
500
501     def _run_tests(self):
502         """Execute tests."""
503         if self.test_name == 'all':
504             for test in self.TESTS:
505                 if test == 'all' or test == 'vm':
506                     continue
507                 self._run_task(test)
508         else:
509             self._run_task(self.test_name)
510
511     def _generate_report(self):
512         """Generate test execution summary report."""
513         total_duration = 0.0
514         total_nb_tests = 0
515         total_nb_success = 0
516         payload = []
517
518         res_table = prettytable.PrettyTable(
519             padding_width=2,
520             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
521         res_table.align['Module'] = "l"
522         res_table.align['Duration'] = "r"
523         res_table.align['Success'] = "r"
524
525         # for each scenario we draw a row for the table
526         for item in self.summary:
527             total_duration += item['overall_duration']
528             total_nb_tests += item['nb_tests']
529             total_nb_success += item['nb_success']
530             try:
531                 success_avg = 100 * item['nb_success'] / item['nb_tests']
532             except ZeroDivisionError:
533                 success_avg = 0
534             success_str = str("{:0.2f}".format(success_avg)) + '%'
535             duration_str = time.strftime("%M:%S",
536                                          time.gmtime(item['overall_duration']))
537             res_table.add_row([item['test_name'], duration_str,
538                                item['nb_tests'], success_str])
539             payload.append({'module': item['test_name'],
540                             'details': {'duration': item['overall_duration'],
541                                         'nb tests': item['nb_tests'],
542                                         'success': success_str}})
543
544         total_duration_str = time.strftime("%H:%M:%S",
545                                            time.gmtime(total_duration))
546         try:
547             self.result = 100 * total_nb_success / total_nb_tests
548         except ZeroDivisionError:
549             self.result = 100
550         success_rate = "{:0.2f}".format(self.result)
551         success_rate_str = str(success_rate) + '%'
552         res_table.add_row(["", "", "", ""])
553         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
554                            success_rate_str])
555
556         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
557         LOGGER.info("Rally '%s' success_rate is %s%%",
558                     self.case_name, success_rate)
559         payload.append({'summary': {'duration': total_duration,
560                                     'nb tests': total_nb_tests,
561                                     'nb success': success_rate}})
562         self.details = payload
563
564     def _clean_up(self):
565         """Cleanup all OpenStack objects. Should be called on completion."""
566         for creator in reversed(self.creators):
567             try:
568                 creator.clean()
569             except Exception as exc:  # pylint: disable=broad-except
570                 LOGGER.error('Unexpected error cleaning - %s', exc)
571
572     @energy.enable_recording
573     def run(self, **kwargs):
574         """Run testcase."""
575         self.start_time = time.time()
576         try:
577             conf_utils.create_rally_deployment()
578             self._prepare_env()
579             self._run_tests()
580             self._generate_report()
581             res = testcase.TestCase.EX_OK
582         except Exception as exc:   # pylint: disable=broad-except
583             LOGGER.error('Error with run: %s', exc)
584             res = testcase.TestCase.EX_RUN_ERROR
585         finally:
586             self._clean_up()
587
588         self.stop_time = time.time()
589         return res
590
591
592 class RallySanity(RallyBase):
593     """Rally sanity testcase implementation."""
594
595     def __init__(self, **kwargs):
596         """Initialize RallySanity object."""
597         if "case_name" not in kwargs:
598             kwargs["case_name"] = "rally_sanity"
599         super(RallySanity, self).__init__(**kwargs)
600         self.mode = 'sanity'
601         self.test_name = 'all'
602         self.smoke = True
603         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
604
605
606 class RallyFull(RallyBase):
607     """Rally full testcase implementation."""
608
609     def __init__(self, **kwargs):
610         """Initialize RallyFull object."""
611         if "case_name" not in kwargs:
612             kwargs["case_name"] = "rally_full"
613         super(RallyFull, self).__init__(**kwargs)
614         self.mode = 'full'
615         self.test_name = 'all'
616         self.smoke = False
617         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')