1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
19 from concurrencytest import ConcurrentTestSuite, fork_for_tests
21 from snaps import file_utils
22 from snaps import test_suite_builder as tsb
23 from snaps.openstack.tests import openstack_tests
25 __author__ = 'spisarski'
27 logger = logging.getLogger('test_runner')
29 ARG_NOT_SET = "argument not set"
30 LOG_LEVELS = {'FATAL': logging.FATAL, 'CRITICAL': logging.CRITICAL,
31 'ERROR': logging.ERROR, 'WARN': logging.WARN,
32 'INFO': logging.INFO, 'DEBUG': logging.DEBUG}
35 def __create_concurrent_test_suite(
36 source_filename, ext_net_name, proxy_settings, ssh_proxy_cmd,
37 run_unit_tests, run_connection_tests, run_api_tests,
38 run_integration_tests, run_staging_tests, flavor_metadata,
39 image_metadata, use_keystone, use_floating_ips, continuous_integration,
42 Compiles the tests that can be run concurrently
43 :param source_filename: the OpenStack credentials file (required)
44 :param ext_net_name: the name of the external network to use for floating
46 :param run_unit_tests: when true, the tests not requiring OpenStack will be
47 added to the test suite
48 :param run_connection_tests: when true, the tests that perform simple
49 connections to OpenStack are executed
50 :param run_api_tests: when true, the tests that perform simple API calls to
51 OpenStack are executed
52 :param run_integration_tests: when true, the integration tests are executed
53 :param run_staging_tests: when true, the staging tests are executed
54 :param proxy_settings: <host>:<port> of the proxy server (optional)
55 :param ssh_proxy_cmd: the command used to connect via SSH over some proxy
57 :param flavor_metadata: dict() object containing the metadata for flavors
58 created for test VM instance
59 :param image_metadata: dict() object containing the metadata for overriding
60 default images within the tests
61 :param use_keystone: when true, tests creating users and projects will be
62 exercised and must be run on a host that
63 has access to the cloud's administrative network
64 :param use_floating_ips: when true, tests requiring floating IPs will be
66 :param continuous_integration: when true, tests for CI will be run
67 :param log_level: the logging level
70 suite = unittest.TestSuite()
72 os_creds = openstack_tests.get_credentials(
73 os_env_file=source_filename, proxy_settings_str=proxy_settings,
74 ssh_proxy_cmd=ssh_proxy_cmd)
76 # Tests that do not require a remote connection to an OpenStack cloud
78 tsb.add_unit_tests(suite)
80 # Basic connection tests
81 if run_connection_tests:
82 tsb.add_openstack_client_tests(
83 suite=suite, os_creds=os_creds, ext_net_name=ext_net_name,
84 use_keystone=use_keystone, log_level=log_level)
86 # Tests the OpenStack API calls
88 tsb.add_openstack_api_tests(
89 suite=suite, os_creds=os_creds, ext_net_name=ext_net_name,
90 use_keystone=use_keystone, flavor_metadata=flavor_metadata,
91 image_metadata=image_metadata, log_level=log_level)
93 # Long running integration type tests
94 if run_integration_tests:
95 tsb.add_openstack_integration_tests(
96 suite=suite, os_creds=os_creds, ext_net_name=ext_net_name,
97 use_keystone=use_keystone, flavor_metadata=flavor_metadata,
98 image_metadata=image_metadata, use_floating_ips=use_floating_ips,
101 if run_staging_tests:
102 tsb.add_openstack_staging_tests(
103 suite=suite, os_creds=os_creds, ext_net_name=ext_net_name,
106 if continuous_integration:
107 tsb.add_openstack_ci_tests(
108 suite=suite, os_creds=os_creds, ext_net_name=ext_net_name,
109 use_keystone=use_keystone, flavor_metadata=flavor_metadata,
110 image_metadata=image_metadata, use_floating_ips=use_floating_ips,
115 def __create_sequential_test_suite(
116 source_filename, ext_net_name, proxy_settings, ssh_proxy_cmd,
117 run_integration_tests, flavor_metadata, image_metadata, use_keystone,
118 use_floating_ips, log_level):
120 Compiles the tests that cannot be run in parallel
121 :param source_filename: the OpenStack credentials file (required)
122 :param ext_net_name: the name of the external network to use for floating
124 :param run_integration_tests: when true, the integration tests are executed
125 :param proxy_settings: <host>:<port> of the proxy server (optional)
126 :param ssh_proxy_cmd: the command used to connect via SSH over some proxy
128 :param flavor_metadata: dict() object containing the metadata for flavors
129 created for test VM instance
130 :param image_metadata: dict() object containing the metadata for overriding
131 default images within the tests
132 :param use_keystone: when true, tests creating users and projects will be
133 exercised and must be run on a host that
134 has access to the cloud's administrative network
135 :param use_floating_ips: when true, tests requiring floating IPs will be
137 :param log_level: the logging level
140 if use_floating_ips and run_integration_tests:
141 suite = unittest.TestSuite()
143 os_creds = openstack_tests.get_credentials(
144 os_env_file=source_filename, proxy_settings_str=proxy_settings,
145 ssh_proxy_cmd=ssh_proxy_cmd)
147 tsb.add_ansible_integration_tests(
148 suite=suite, os_creds=os_creds, ext_net_name=ext_net_name,
149 use_keystone=use_keystone, flavor_metadata=flavor_metadata,
150 image_metadata=image_metadata, log_level=log_level)
155 def __output_results(results):
157 Sends the test results to the logger
163 logger.error('Number of errors in test suite - %s',
165 for test, message in results.errors:
166 logger.error(str(test) + " ERROR with " + message)
169 logger.error('Number of failures in test suite - %s',
170 len(results.failures))
171 for test, message in results.failures:
172 logger.error(str(test) + " FAILED with " + message)
177 Begins running unit tests.
178 argv[1] if used must be the source filename else os_env.yaml will be
180 argv[2] if used must be the proxy server <host>:<port>
182 logger.info('Starting test suite')
184 log_level = LOG_LEVELS.get(arguments.log_level, logging.DEBUG)
186 flavor_metadata = None
187 if arguments.flavor_metadata:
188 flavor_metadata = json.loads(arguments.flavor_metadata)
190 image_metadata = None
191 if arguments.image_metadata_file:
192 image_metadata = file_utils.read_yaml(arguments.image_metadata_file)
194 concurrent_suite = None
195 sequential_suite = None
197 if arguments.env and arguments.ext_net:
198 unit = arguments.include_unit != ARG_NOT_SET
199 connection = arguments.include_connection != ARG_NOT_SET
200 api = arguments.include_api != ARG_NOT_SET
201 integration = arguments.include_integration != ARG_NOT_SET
202 ci = arguments.continuous_integration != ARG_NOT_SET
203 staging = arguments.include_staging != ARG_NOT_SET
204 if (not unit and not connection and not api and not integration
205 and not staging and not ci):
211 concurrent_suite = __create_concurrent_test_suite(
212 arguments.env, arguments.ext_net, arguments.proxy,
213 arguments.ssh_proxy_cmd, unit, connection, api,
214 integration, staging, flavor_metadata, image_metadata,
215 arguments.use_keystone != ARG_NOT_SET,
216 arguments.floating_ips != ARG_NOT_SET,
219 if (arguments.include_integration != ARG_NOT_SET
220 and arguments.floating_ips != ARG_NOT_SET):
221 sequential_suite = __create_sequential_test_suite(
222 arguments.env, arguments.ext_net, arguments.proxy,
223 arguments.ssh_proxy_cmd, integration, flavor_metadata,
225 arguments.use_keystone != ARG_NOT_SET,
226 arguments.floating_ips != ARG_NOT_SET, log_level)
228 logger.error('Environment file or external network not defined')
232 while i < int(arguments.num_runs):
236 logger.info('Running Concurrent Tests')
237 concurrent_runner = unittest.TextTestRunner(verbosity=2)
238 concurrent_suite = ConcurrentTestSuite(
239 concurrent_suite, fork_for_tests(int(arguments.threads)))
240 concurrent_results = concurrent_runner.run(concurrent_suite)
241 __output_results(concurrent_results)
243 if ((concurrent_results.errors
244 and len(concurrent_results.errors) > 0)
245 or (concurrent_results.failures
246 and len(concurrent_results.failures) > 0)):
247 logger.error('See above for test failures')
251 'Concurrent tests completed successfully in run #%s', i)
254 logger.info('Running Sequential Tests')
255 sequential_runner = unittest.TextTestRunner(verbosity=2)
256 sequential_results = sequential_runner.run(sequential_suite)
257 __output_results(sequential_results)
259 if ((sequential_results.errors
260 and len(sequential_results.errors) > 0)
261 or (sequential_results.failures
262 and len(sequential_results.failures) > 0)):
263 logger.error('See above for test failures')
267 'Sequential tests completed successfully in run #%s', i)
269 logger.info('Successful completion of %s test runs', i)
273 if __name__ == '__main__':
274 parser = argparse.ArgumentParser()
276 '-e', '--env', dest='env', required=True,
277 help='OpenStack credentials file')
279 '-n', '--net', dest='ext_net', required=True,
280 help='External network name')
282 '-p', '--proxy', dest='proxy', nargs='?', default=None,
283 help='Optonal HTTP proxy socket (<host>:<port>)')
285 '-s', '--ssh-proxy-cmd', dest='ssh_proxy_cmd', nargs='?', default=None,
286 help='Optonal SSH proxy command value')
288 '-l', '--log-level', dest='log_level', default='INFO',
289 help='Logging Level (FATAL|CRITICAL|ERROR|WARN|INFO|DEBUG)')
291 '-u', '--unit-tests', dest='include_unit', default=ARG_NOT_SET,
292 nargs='?', help='When argument is set, all tests not requiring '
293 'OpenStack will be executed')
295 '-c', '--connection-tests', dest='include_connection',
296 default=ARG_NOT_SET, nargs='?',
297 help='When argument is set, simple OpenStack connection tests will be '
300 '-a', '--api-tests', dest='include_api', default=ARG_NOT_SET,
302 help='When argument is set, OpenStack API tests will be executed')
304 '-i', '--integration-tests', dest='include_integration',
305 default=ARG_NOT_SET, nargs='?',
306 help='When argument is set, OpenStack integrations tests will be '
309 '-st', '--staging-tests', dest='include_staging', default=ARG_NOT_SET,
311 help='When argument is set, OpenStack staging tests will be executed')
313 '-f', '--floating-ips', dest='floating_ips', default=ARG_NOT_SET,
314 nargs='?', help='When argument is set, all integration tests requiring'
315 ' Floating IPs will be executed')
317 '-k', '--use-keystone', dest='use_keystone', default=ARG_NOT_SET,
319 help='When argument is set, the tests will exercise the keystone APIs '
320 'and must be run on a machine that has access to the admin '
321 'network and is able to create users and groups')
323 '-fm', '--flavor-meta', dest='flavor_metadata',
324 help='JSON string to be used as flavor metadata for all test instances'
327 '-im', '--image-meta', dest='image_metadata_file', default=None,
328 help='Location of YAML file containing the image metadata')
330 '-ci', '--continuous-integration', dest='continuous_integration',
331 default=ARG_NOT_SET, nargs='?',
332 help='When argument is set, OpenStack integrations tests will be '
335 '-r', '--num-runs', dest='num_runs', default=1,
336 help='Number of test runs to execute (default 1)')
338 '-t', '--threads', dest='threads', default=4,
339 help='Number of threads to execute the tests (default 4)')
341 args = parser.parse_args()