gerrit.opnfv Code Review - functest.git/blob - functest/opnfv

1 #!/usr/bin/env python

2 #

4 # This program and the accompanying materials

5 # are made available under the terms of the Apache License, Version 2.0

6 # which accompanies this distribution, and is available at

7 #

8 # http://www.apache.org/licenses/LICENSE-2.0

9 #

10

11 """Tempest testcases implementation."""

12

13 from __future__ import division

14

15 import json

16 import logging

17 import os

18 import re

19 import shutil

20 import subprocess

21 import time

22

23 import pkg_resources

24 from six.moves import configparser

25 from xtesting.core import testcase

26 import yaml

27

28 from functest.core import singlevm

29 from functest.opnfv_tests.openstack.rally import rally

30 from functest.utils import config

31 from functest.utils import env

32 from functest.utils import functest_utils

33

34 LOGGER = logging.getLogger(__name__)

35

36

37 class TempestCommon(singlevm.VmReady2):

38 # pylint: disable=too-many-instance-attributes,too-many-public-methods

39 """TempestCommon testcases implementation class."""

40

41 visibility = 'public'

42 filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'

43 shared_network = True

44 TEMPEST_CONF_YAML = pkg_resources.resource_filename(

45 'functest',

46 'opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml')

47 TEMPEST_CUSTOM = pkg_resources.resource_filename(

48 'functest',

49 'opnfv_tests/openstack/tempest/custom_tests/test_list.txt')

50 TEMPEST_BLACKLIST = pkg_resources.resource_filename(

51 'functest',

52 'opnfv_tests/openstack/tempest/custom_tests/blacklist.yaml')

53

54 def __init__(self, **kwargs):

55 if "case_name" not in kwargs:

56 kwargs["case_name"] = 'tempest'

57 super(TempestCommon, self).__init__(**kwargs)

58 assert self.orig_cloud

59 assert self.cloud

60 assert self.project

61 if self.orig_cloud.get_role("admin"):

62 self.role_name = "admin"

63 elif self.orig_cloud.get_role("Admin"):

64 self.role_name = "Admin"

65 else:

66 raise Exception("Cannot detect neither admin nor Admin")

67 self.orig_cloud.grant_role(

68 self.role_name, user=self.project.user.id,

69 project=self.project.project.id,

70 domain=self.project.domain.id)

71 self.orig_cloud.grant_role(

72 self.role_name, user=self.project.user.id,

73 domain=self.project.domain.id)

74 self.deployment_id = None

75 self.verifier_id = None

76 self.verifier_repo_dir = None

77 self.deployment_dir = None

78 self.verification_id = None

79 self.res_dir = os.path.join(

80 getattr(config.CONF, 'dir_results'), self.case_name)

81 self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')

82 self.list = os.path.join(self.res_dir, 'test_list.txt')

83 self.conf_file = None

84 self.image_alt = None

85 self.flavor_alt = None

86 self.services = []

87 try:

88 self.services = kwargs['run']['args']['services']

89 except Exception: # pylint: disable=broad-except

90 pass

91 self.neutron_extensions = []

92 try:

93 self.neutron_extensions = kwargs['run']['args'][

94 'neutron_extensions']

95 except Exception: # pylint: disable=broad-except

96 pass

97 self.deny_skipping = kwargs.get("deny_skipping", False)

98

99 def check_services(self):

100 """Check the mandatory services."""

101 for service in self.services:

102 try:

103 self.cloud.search_services(service)[0]

104 except Exception: # pylint: disable=broad-except

105 self.is_skipped = True

106 break

107

108 def check_extensions(self):

109 """Check the mandatory network extensions."""

110 extensions = self.cloud.get_network_extensions()

111 for network_extension in self.neutron_extensions:

112 if network_extension not in extensions:

113 LOGGER.warning(

114 "Cannot find Neutron extension: %s", network_extension)

115 self.is_skipped = True

116 break

117

118 def check_requirements(self):

119 self.check_services()

120 self.check_extensions()

121 if self.is_skipped:

122 self.project.clean()

123

124 @staticmethod

125 def read_file(filename):

126 """Read file and return content as a stripped list."""

127 with open(filename) as src:

128 return [line.strip() for line in src.readlines()]

129

130 @staticmethod

131 def get_verifier_result(verif_id):

132 """Retrieve verification results."""

133 result = {

134 'num_tests': 0,

135 'num_success': 0,

136 'num_failures': 0,

137 'num_skipped': 0

138 }

139 cmd = ["rally", "verify", "show", "--uuid", verif_id]

140 LOGGER.info("Showing result for a verification: '%s'.", cmd)

141 proc = subprocess.Popen(cmd,

142 stdout=subprocess.PIPE,

143 stderr=subprocess.STDOUT)

144 for line in proc.stdout:

145 LOGGER.info(line.rstrip())

146 new_line = line.replace(' ', '').split('|')

147 if 'Tests' in new_line:

148 break

149 if 'Testscount' in new_line:

150 result['num_tests'] = int(new_line[2])

151 elif 'Success' in new_line:

152 result['num_success'] = int(new_line[2])

153 elif 'Skipped' in new_line:

154 result['num_skipped'] = int(new_line[2])

155 elif 'Failures' in new_line:

156 result['num_failures'] = int(new_line[2])

157 return result

158

159 @staticmethod

160 def backup_tempest_config(conf_file, res_dir):

161 """

162 Copy config file to tempest results directory

163 """

164 if not os.path.exists(res_dir):

165 os.makedirs(res_dir)

166 shutil.copyfile(conf_file,

167 os.path.join(res_dir, 'tempest.conf'))

168

169 @staticmethod

170 def create_verifier():

171 """Create new verifier"""

172 LOGGER.info("Create verifier from existing repo...")

173 cmd = ['rally', 'verify', 'delete-verifier',

174 '--id', str(getattr(config.CONF, 'tempest_verifier_name')),

175 '--force']

176 try:

177 output = subprocess.check_output(cmd)

178 LOGGER.info("%s\n%s", " ".join(cmd), output)

179 except subprocess.CalledProcessError:

180 pass

181

182 cmd = ['rally', 'verify', 'create-verifier',

183 '--source', str(getattr(config.CONF, 'dir_repo_tempest')),

184 '--name', str(getattr(config.CONF, 'tempest_verifier_name')),

185 '--type', 'tempest', '--system-wide']

186 output = subprocess.check_output(cmd)

187 LOGGER.info("%s\n%s", " ".join(cmd), output)

188 return TempestCommon.get_verifier_id()

189

190 @staticmethod

191 def get_verifier_id():

192 """

193 Returns verifier id for current Tempest

194 """

195 cmd = ("rally verify list-verifiers | awk '/" +

196 getattr(config.CONF, 'tempest_verifier_name') +

197 "/ {print $2}'")

198 proc = subprocess.Popen(cmd, shell=True,

199 stdout=subprocess.PIPE,

200 stderr=subprocess.STDOUT)

201 verifier_uuid = proc.stdout.readline().rstrip()

202 return verifier_uuid

203

204 @staticmethod

205 def get_verifier_repo_dir(verifier_id):

206 """

207 Returns installed verifier repo directory for Tempest

208 """

209 return os.path.join(getattr(config.CONF, 'dir_rally_inst'),

210 'verification',

211 'verifier-{}'.format(verifier_id),

212 'repo')

213

214 @staticmethod

215 def get_verifier_deployment_dir(verifier_id, deployment_id):

216 """

217 Returns Rally deployment directory for current verifier

218 """

219 return os.path.join(getattr(config.CONF, 'dir_rally_inst'),

220 'verification',

221 'verifier-{}'.format(verifier_id),

222 'for-deployment-{}'.format(deployment_id))

223

224 @staticmethod

225 def update_tempest_conf_file(conf_file, rconfig):

226 """Update defined paramters into tempest config file"""

227 with open(TempestCommon.TEMPEST_CONF_YAML) as yfile:

228 conf_yaml = yaml.safe_load(yfile)

229 if conf_yaml:

230 sections = rconfig.sections()

231 for section in conf_yaml:

232 if section not in sections:

233 rconfig.add_section(section)

234 sub_conf = conf_yaml.get(section)

235 for key, value in sub_conf.items():

236 rconfig.set(section, key, value)

237

238 with open(conf_file, 'wb') as config_file:

239 rconfig.write(config_file)

240

241 @staticmethod

242 def configure_tempest_update_params(

243 tempest_conf_file, image_id=None, flavor_id=None,

244 compute_cnt=1, image_alt_id=None, flavor_alt_id=None,

245 admin_role_name='admin', cidr='192.168.120.0/24',

246 domain_id='default'):

247 # pylint: disable=too-many-branches,too-many-arguments,

248 # too-many-statements

249 """

250 Add/update needed parameters into tempest.conf file

251 """

252 LOGGER.debug("Updating selected tempest.conf parameters...")

253 rconfig = configparser.RawConfigParser()

254 rconfig.read(tempest_conf_file)

255 rconfig.set(

256 'compute', 'volume_device_name', env.get('VOLUME_DEVICE_NAME'))

257 if image_id is not None:

258 rconfig.set('compute', 'image_ref', image_id)

259 if image_alt_id is not None:

260 rconfig.set('compute', 'image_ref_alt', image_alt_id)

261 if flavor_id is not None:

262 rconfig.set('compute', 'flavor_ref', flavor_id)

263 if flavor_alt_id is not None:

264 rconfig.set('compute', 'flavor_ref_alt', flavor_alt_id)

265 if compute_cnt > 1:

266 # enable multinode tests

267 rconfig.set('compute', 'min_compute_nodes', compute_cnt)

268 rconfig.set('compute-feature-enabled', 'live_migration', True)

269 filters = ['RetryFilter', 'AvailabilityZoneFilter', 'ComputeFilter',

270 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter',

271 'ServerGroupAntiAffinityFilter',

272 'ServerGroupAffinityFilter']

273 rconfig.set(

274 'compute-feature-enabled', 'scheduler_available_filters',

275 functest_utils.convert_list_to_ini(filters))

276 if os.environ.get('OS_REGION_NAME'):

277 rconfig.set('identity', 'region', os.environ.get('OS_REGION_NAME'))

278 if env.get("NEW_USER_ROLE").lower() != "member":

279 rconfig.set(

280 'auth', 'tempest_roles',

281 functest_utils.convert_list_to_ini([env.get("NEW_USER_ROLE")]))

282 if not json.loads(env.get("USE_DYNAMIC_CREDENTIALS").lower()):

283 rconfig.set('auth', 'use_dynamic_credentials', False)

284 account_file = os.path.join(

285 getattr(config.CONF, 'dir_functest_data'), 'accounts.yaml')

286 assert os.path.exists(

287 account_file), "{} doesn't exist".format(account_file)

288 rconfig.set('auth', 'test_accounts_file', account_file)

289 rconfig.set('identity', 'auth_version', 'v3')

290 rconfig.set('identity', 'admin_role', admin_role_name)

291 rconfig.set('identity', 'default_domain_id', domain_id)

292 if not rconfig.has_section('network'):

293 rconfig.add_section('network')

294 rconfig.set('network', 'default_network', cidr)

295 rconfig.set('network', 'project_network_cidr', cidr)

296 rconfig.set('network', 'project_networks_reachable', False)

297 rconfig.set(

298 'identity', 'v3_endpoint_type',

299 os.environ.get('OS_INTERFACE', 'public'))

300

301 sections = rconfig.sections()

302 services_list = [

303 'compute', 'volume', 'image', 'network', 'data-processing',

304 'object-storage', 'orchestration']

305 for service in services_list:

306 if service not in sections:

307 rconfig.add_section(service)

308 rconfig.set(service, 'endpoint_type',

309 os.environ.get('OS_INTERFACE', 'public'))

310

311 LOGGER.debug('Add/Update required params defined in tempest_conf.yaml '

312 'into tempest.conf file')

313 TempestCommon.update_tempest_conf_file(tempest_conf_file, rconfig)

314

315 @staticmethod

316 def configure_verifier(deployment_dir):

317 """

318 Execute rally verify configure-verifier, which generates tempest.conf

319 """

320 cmd = ['rally', 'verify', 'configure-verifier', '--reconfigure',

321 '--id', str(getattr(config.CONF, 'tempest_verifier_name'))]

322 output = subprocess.check_output(cmd)

323 LOGGER.info("%s\n%s", " ".join(cmd), output)

324

325 LOGGER.debug("Looking for tempest.conf file...")

326 tempest_conf_file = os.path.join(deployment_dir, "tempest.conf")

327 if not os.path.isfile(tempest_conf_file):

328 LOGGER.error("Tempest configuration file %s NOT found.",

329 tempest_conf_file)

330 return None

331 return tempest_conf_file

332

333 def generate_test_list(self, **kwargs):

334 """Generate test list based on the test mode."""

335 LOGGER.debug("Generating test case list...")

336 self.backup_tempest_config(self.conf_file, '/etc')

337 if kwargs.get('mode') == 'custom':

338 if os.path.isfile(self.TEMPEST_CUSTOM):

339 shutil.copyfile(

340 self.TEMPEST_CUSTOM, self.list)

341 else:

342 raise Exception("Tempest test list file %s NOT found."

343 % self.TEMPEST_CUSTOM)

344 else:

345 testr_mode = kwargs.get(

346 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')

347 cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(

348 self.verifier_repo_dir, testr_mode, self.list)

349 output = subprocess.check_output(cmd, shell=True)

350 LOGGER.info("%s\n%s", cmd, output)

351 os.remove('/etc/tempest.conf')

352

353 def apply_tempest_blacklist(self):

354 """Exclude blacklisted test cases."""

355 LOGGER.debug("Applying tempest blacklist...")

356 if os.path.exists(self.raw_list):

357 os.remove(self.raw_list)

358 os.rename(self.list, self.raw_list)

359 cases_file = self.read_file(self.raw_list)

360 result_file = open(self.list, 'w')

361 black_tests = []

362 try:

363 deploy_scenario = env.get('DEPLOY_SCENARIO')

364 if bool(deploy_scenario):

365 # if DEPLOY_SCENARIO is set we read the file

366 black_list_file = open(self.TEMPEST_BLACKLIST)

367 black_list_yaml = yaml.safe_load(black_list_file)

368 black_list_file.close()

369 for item in black_list_yaml:

370 scenarios = item['scenarios']

371 if deploy_scenario in scenarios:

372 tests = item['tests']

373 for test in tests:

374 black_tests.append(test)

375 break

376 except Exception: # pylint: disable=broad-except

377 black_tests = []

378 LOGGER.debug("Tempest blacklist file does not exist.")

379

380 for cases_line in cases_file:

381 for black_tests_line in black_tests:

382 if black_tests_line in cases_line:

383 break

384 else:

385 result_file.write(str(cases_line) + '\n')

386 result_file.close()

387

388 def run_verifier_tests(self, **kwargs):

389 """Execute tempest test cases."""

390 cmd = ["rally", "verify", "start", "--load-list",

391 self.list]

392 cmd.extend(kwargs.get('option', []))

393 LOGGER.info("Starting Tempest test suite: '%s'.", cmd)

394

395 f_stdout = open(

396 os.path.join(self.res_dir, "tempest.log"), 'w+')

397

398 proc = subprocess.Popen(

399 cmd,

400 stdout=subprocess.PIPE,

401 stderr=subprocess.STDOUT,

402 bufsize=1)

403

404 with proc.stdout:

405 for line in iter(proc.stdout.readline, b''):

406 if re.search(r"\} tempest\.", line):

407 LOGGER.info(line.rstrip())

408 elif re.search(r'(?=$UUID=(.*)$)', line):

409 self.verification_id = re.search(

410 r'(?=$UUID=(.*)$)', line).group(1)

411 f_stdout.write(line)

412 proc.wait()

413 f_stdout.close()

414

415 if self.verification_id is None:

416 raise Exception('Verification UUID not found')

417 LOGGER.info('Verification UUID: %s', self.verification_id)

418

419 shutil.copy(

420 "{}/tempest.log".format(self.deployment_dir),

421 "{}/tempest.debug.log".format(self.res_dir))

422

423 def parse_verifier_result(self):

424 """Parse and save test results."""

425 stat = self.get_verifier_result(self.verification_id)

426 try:

427 num_executed = stat['num_tests'] - stat['num_skipped']

428 try:

429 self.result = 100 * stat['num_success'] / num_executed

430 except ZeroDivisionError:

431 self.result = 0

432 if stat['num_tests'] > 0:

433 LOGGER.info("All tests have been skipped")

434 else:

435 LOGGER.error("No test has been executed")

436 return

437

438 with open(os.path.join(self.res_dir,

439 "rally.log"), 'r') as logfile:

440 output = logfile.read()

441

442 success_testcases = []

443 for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',

444 output):

445 success_testcases.append(match)

446 failed_testcases = []

447 for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',

448 output):

449 failed_testcases.append(match)

450 skipped_testcases = []

451 for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip(?::| )',

452 output):

453 skipped_testcases.append(match)

454

455 self.details = {"tests_number": stat['num_tests'],

456 "success_number": stat['num_success'],

457 "skipped_number": stat['num_skipped'],

458 "failures_number": stat['num_failures'],

459 "success": success_testcases,

460 "skipped": skipped_testcases,

461 "failures": failed_testcases}

462 except Exception: # pylint: disable=broad-except

463 self.result = 0

464

465 LOGGER.info("Tempest %s success_rate is %s%%",

466 self.case_name, self.result)

467

468 def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):

469 """Set image name as tempest img_name_regex"""

470 rconfig = configparser.RawConfigParser()

471 rconfig.read(rally_conf)

472 if not rconfig.has_section('openstack'):

473 rconfig.add_section('openstack')

474 rconfig.set('openstack', 'img_name_regex', '^{}$'.format(

475 self.image.name))

476 with open(rally_conf, 'wb') as config_file:

477 rconfig.write(config_file)

478

479 def update_default_role(self, rally_conf='/etc/rally/rally.conf'):

480 """Detect and update the default role if required"""

481 role = self.get_default_role(self.cloud)

482 if not role:

483 return

484 rconfig = configparser.RawConfigParser()

485 rconfig.read(rally_conf)

486 if not rconfig.has_section('openstack'):

487 rconfig.add_section('openstack')

488 rconfig.set('openstack', 'swift_operator_role', role.name)

489 with open(rally_conf, 'wb') as config_file:

490 rconfig.write(config_file)

491

492 def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):

493 """Print rally logs in res dir"""

494 if not os.path.exists(self.res_dir):

495 os.makedirs(self.res_dir)

496 rconfig = configparser.RawConfigParser()

497 rconfig.read(rally_conf)

498 rconfig.set('DEFAULT', 'debug', True)

499 rconfig.set('DEFAULT', 'use_stderr', False)

500 rconfig.set('DEFAULT', 'log-file', 'rally.log')

501 rconfig.set('DEFAULT', 'log_dir', self.res_dir)

502 with open(rally_conf, 'wb') as config_file:

503 rconfig.write(config_file)

504

505 @staticmethod

506 def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):

507 """Clean Rally config"""

508 rconfig = configparser.RawConfigParser()

509 rconfig.read(rally_conf)

510 if rconfig.has_option('openstack', 'img_name_regex'):

511 rconfig.remove_option('openstack', 'img_name_regex')

512 if rconfig.has_option('openstack', 'swift_operator_role'):

513 rconfig.remove_option('openstack', 'swift_operator_role')

514 if rconfig.has_option('DEFAULT', 'use_stderr'):

515 rconfig.remove_option('DEFAULT', 'use_stderr')

516 if rconfig.has_option('DEFAULT', 'debug'):

517 rconfig.remove_option('DEFAULT', 'debug')

518 if rconfig.has_option('DEFAULT', 'log-file'):

519 rconfig.remove_option('DEFAULT', 'log-file')

520 if rconfig.has_option('DEFAULT', 'log_dir'):

521 rconfig.remove_option('DEFAULT', 'log_dir')

522 with open(rally_conf, 'wb') as config_file:

523 rconfig.write(config_file)

524

525 def update_network_section(self):

526 """Update network section in tempest.conf"""

527 rconfig = configparser.RawConfigParser()

528 rconfig.read(self.conf_file)

529 if not rconfig.has_section('network'):

530 rconfig.add_section('network')

531 rconfig.set('network', 'public_network_id', self.ext_net.id)

532 rconfig.set('network', 'floating_network_name', self.ext_net.name)

533 with open(self.conf_file, 'wb') as config_file:

534 rconfig.write(config_file)

535

536 def update_compute_section(self):

537 """Update compute section in tempest.conf"""

538 rconfig = configparser.RawConfigParser()

539 rconfig.read(self.conf_file)

540 if not rconfig.has_section('compute'):

541 rconfig.add_section('compute')

542 rconfig.set('compute', 'fixed_network_name', self.network.name)

543 with open(self.conf_file, 'wb') as config_file:

544 rconfig.write(config_file)

545

546 def update_scenario_section(self):

547 """Update scenario section in tempest.conf"""

548 rconfig = configparser.RawConfigParser()

549 rconfig.read(self.conf_file)

550 filename = getattr(

551 config.CONF, '{}_image'.format(self.case_name), self.filename)

552 if not rconfig.has_section('scenario'):

553 rconfig.add_section('scenario')

554 rconfig.set('scenario', 'img_file', os.path.basename(filename))

555 rconfig.set('scenario', 'img_dir', os.path.dirname(filename))

556 rconfig.set('scenario', 'img_disk_format', getattr(

557 config.CONF, '{}_image_format'.format(self.case_name),

558 self.image_format))

559 extra_properties = self.extra_properties.copy()

560 if env.get('IMAGE_PROPERTIES'):

561 extra_properties.update(

562 functest_utils.convert_ini_to_dict(

563 env.get('IMAGE_PROPERTIES')))

564 extra_properties.update(

565 getattr(config.CONF, '{}_extra_properties'.format(

566 self.case_name), {}))

567 rconfig.set(

568 'scenario', 'img_properties',

569 functest_utils.convert_dict_to_ini(extra_properties))

570 with open(self.conf_file, 'wb') as config_file:

571 rconfig.write(config_file)

572

573 def configure(self, **kwargs): # pylint: disable=unused-argument

574 """

575 Create all openstack resources for tempest-based testcases and write

576 tempest.conf.

577 """

578 if not os.path.exists(self.res_dir):

579 os.makedirs(self.res_dir)

580 self.deployment_id = rally.RallyBase.create_rally_deployment(

581 environ=self.project.get_environ())

582 if not self.deployment_id:

583 raise Exception("Deployment create failed")

584 self.verifier_id = self.create_verifier()

585 if not self.verifier_id:

586 raise Exception("Verifier create failed")

587 self.verifier_repo_dir = self.get_verifier_repo_dir(

588 self.verifier_id)

589 self.deployment_dir = self.get_verifier_deployment_dir(

590 self.verifier_id, self.deployment_id)

591

592 compute_cnt = len(self.orig_cloud.list_hypervisors())

593

594 self.image_alt = self.publish_image_alt()

595 self.flavor_alt = self.create_flavor_alt()

596 LOGGER.debug("flavor: %s", self.flavor_alt)

597

598 self.conf_file = self.configure_verifier(self.deployment_dir)

599 if not self.conf_file:

600 raise Exception("Tempest verifier configuring failed")

601 self.configure_tempest_update_params(

602 self.conf_file,

603 image_id=self.image.id,

604 flavor_id=self.flavor.id,

605 compute_cnt=compute_cnt,

606 image_alt_id=self.image_alt.id,

607 flavor_alt_id=self.flavor_alt.id,

608 admin_role_name=self.role_name, cidr=self.cidr,

609 domain_id=self.project.domain.id)

610 self.update_network_section()

611 self.update_compute_section()

612 self.update_scenario_section()

613 self.backup_tempest_config(self.conf_file, self.res_dir)

614

615 def run(self, **kwargs):

616 self.start_time = time.time()

617 try:

618 assert super(TempestCommon, self).run(

619 **kwargs) == testcase.TestCase.EX_OK

620 if not os.path.exists(self.res_dir):

621 os.makedirs(self.res_dir)

622 self.update_rally_regex()

623 self.update_default_role()

624 self.update_rally_logs()

625 shutil.copy("/etc/rally/rally.conf", self.res_dir)

626 self.configure(**kwargs)

627 self.generate_test_list(**kwargs)

628 self.apply_tempest_blacklist()

629 self.run_verifier_tests(**kwargs)

630 self.parse_verifier_result()

631 rally.RallyBase.verify_report(

632 os.path.join(self.res_dir, "tempest-report.html"),

633 self.verification_id)

634 rally.RallyBase.verify_report(

635 os.path.join(self.res_dir, "tempest-report.xml"),

636 self.verification_id, "junit-xml")

637 res = testcase.TestCase.EX_OK

638 except Exception: # pylint: disable=broad-except

639 LOGGER.exception('Error with run')

640 self.result = 0

641 res = testcase.TestCase.EX_RUN_ERROR

642 self.stop_time = time.time()

643 return res

644

645 def clean(self):

646 """

647 Cleanup all OpenStack objects. Should be called on completion.

648 """

649 self.clean_rally_conf()

650 if self.image_alt:

651 self.cloud.delete_image(self.image_alt)

652 if self.flavor_alt:

653 self.orig_cloud.delete_flavor(self.flavor_alt.id)

654 super(TempestCommon, self).clean()

655

656 def is_successful(self):

657 """The overall result of the test."""

658 skips = self.details.get("skipped_number", 0)

659 if skips > 0 and self.deny_skipping:

660 return testcase.TestCase.EX_TESTCASE_FAILED

661 return super(TempestCommon, self).is_successful()