snaps/test_suite_builder.py

   1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
   2 #                    and others.  All rights reserved.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import logging
  17 import unittest
  18
  19 from snaps.domain.test.flavor_tests import FlavorDomainObjectTests
  20 from snaps.domain.test.image_tests import ImageDomainObjectTests
  21 from snaps.domain.test.keypair_tests import KeypairDomainObjectTests
  22 from snaps.domain.test.network_tests import (
  23     SecurityGroupDomainObjectTests, SecurityGroupRuleDomainObjectTests,
  24     PortDomainObjectTests)
  25 from snaps.domain.test.project_tests import ProjectDomainObjectTests
  26 from snaps.domain.test.stack_tests import StackDomainObjectTests
  27 from snaps.domain.test.user_tests import UserDomainObjectTests
  28 from snaps.domain.test.vm_inst_tests import (
  29     VmInstDomainObjectTests, FloatingIpDomainObjectTests)
  30 from snaps.openstack.tests.conf.os_credentials_tests import (
  31     ProxySettingsUnitTests, OSCredsUnitTests)
  32 from snaps.openstack.tests.create_flavor_tests import (
  33     CreateFlavorTests, FlavorSettingsUnitTests)
  34 from snaps.openstack.tests.create_image_tests import (
  35     CreateImageSuccessTests, CreateImageNegativeTests, ImageSettingsUnitTests,
  36     CreateMultiPartImageTests)
  37 from snaps.openstack.tests.create_instance_tests import (
  38     CreateInstanceSingleNetworkTests, CreateInstancePubPrivNetTests,
  39     CreateInstanceOnComputeHost, CreateInstanceSimpleTests,
  40     FloatingIpSettingsUnitTests, InstanceSecurityGroupTests,
  41     VmInstanceSettingsUnitTests, CreateInstancePortManipulationTests,
  42     SimpleHealthCheck, CreateInstanceFromThreePartImage,
  43     CreateInstanceMockOfflineTests)
  44 from snaps.openstack.tests.create_keypairs_tests import (
  45     CreateKeypairsTests, KeypairSettingsUnitTests)
  46 from snaps.openstack.tests.create_network_tests import (
  47     CreateNetworkSuccessTests, NetworkSettingsUnitTests, PortSettingsUnitTests,
  48     SubnetSettingsUnitTests, CreateNetworkTypeTests)
  49 from snaps.openstack.tests.create_project_tests import (
  50     CreateProjectSuccessTests, ProjectSettingsUnitTests,
  51     CreateProjectUserTests)
  52 from snaps.openstack.tests.create_router_tests import (
  53     CreateRouterSuccessTests, CreateRouterNegativeTests)
  54 from snaps.openstack.tests.create_security_group_tests import (
  55     CreateSecurityGroupTests, SecurityGroupRuleSettingsUnitTests,
  56     SecurityGroupSettingsUnitTests)
  57 from snaps.openstack.tests.create_stack_tests import (
  58     StackSettingsUnitTests, CreateStackSuccessTests,  CreateStackNegativeTests)
  59 from snaps.openstack.tests.create_user_tests import (
  60     UserSettingsUnitTests, CreateUserSuccessTests)
  61 from snaps.openstack.tests.os_source_file_test import (
  62     OSComponentTestCase, OSIntegrationTestCase)
  63 from snaps.openstack.utils.tests.glance_utils_tests import (
  64     GlanceSmokeTests, GlanceUtilsTests)
  65 from snaps.openstack.utils.tests.heat_utils_tests import (
  66     HeatUtilsCreateStackTests, HeatSmokeTests)
  67 from snaps.openstack.utils.tests.keystone_utils_tests import (
  68     KeystoneSmokeTests, KeystoneUtilsTests)
  69 from snaps.openstack.utils.tests.neutron_utils_tests import (
  70     NeutronSmokeTests, NeutronUtilsNetworkTests, NeutronUtilsSubnetTests,
  71     NeutronUtilsRouterTests, NeutronUtilsSecurityGroupTests,
  72     NeutronUtilsFloatingIpTests)
  73 from snaps.openstack.utils.tests.nova_utils_tests import (
  74     NovaSmokeTests, NovaUtilsKeypairTests, NovaUtilsFlavorTests,
  75     NovaUtilsInstanceTests)
  76 from snaps.provisioning.tests.ansible_utils_tests import (
  77     AnsibleProvisioningTests)
  78 from snaps.tests.file_utils_tests import FileUtilsTests
  79
  80 __author__ = 'spisarski'
  81
  82
  83 def add_unit_tests(suite):
  84     """
  85     Adds tests that do not require external resources
  86     :param suite: the unittest.TestSuite object to which to add the tests
  87     :return: None as the tests will be adding to the 'suite' parameter object
  88     """
  89     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(FileUtilsTests))
  90     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
  91         SecurityGroupRuleSettingsUnitTests))
  92     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
  93         ProxySettingsUnitTests))
  94     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
  95         OSCredsUnitTests))
  96     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
  97         SecurityGroupSettingsUnitTests))
  98     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
  99         SecurityGroupDomainObjectTests))
 100     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 101         SecurityGroupRuleDomainObjectTests))
 102     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 103         ImageSettingsUnitTests))
 104     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 105         ImageDomainObjectTests))
 106     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 107         FlavorSettingsUnitTests))
 108     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 109         FlavorDomainObjectTests))
 110     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 111         KeypairSettingsUnitTests))
 112     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 113         KeypairDomainObjectTests))
 114     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 115         UserSettingsUnitTests))
 116     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 117         UserDomainObjectTests))
 118     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 119         ProjectSettingsUnitTests))
 120     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 121         ProjectDomainObjectTests))
 122     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 123         NetworkSettingsUnitTests))
 124     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 125         SubnetSettingsUnitTests))
 126     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 127         PortSettingsUnitTests))
 128     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 129         PortDomainObjectTests))
 130     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 131         FloatingIpSettingsUnitTests))
 132     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 133         VmInstanceSettingsUnitTests))
 134     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 135         StackDomainObjectTests))
 136     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 137         StackSettingsUnitTests))
 138     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 139         VmInstDomainObjectTests))
 140     suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
 141         FloatingIpDomainObjectTests))
 142
 143
 144 def add_openstack_client_tests(suite, os_creds, ext_net_name,
 145                                use_keystone=True, log_level=logging.INFO):
 146     """
 147     Adds tests written to exercise OpenStack client retrieval
 148     :param suite: the unittest.TestSuite object to which to add the tests
 149     :param os_creds: and instance of OSCreds that holds the credentials
 150                      required by OpenStack
 151     :param ext_net_name: the name of an external network on the cloud under
 152                          test
 153     :param use_keystone: when True, tests requiring direct access to Keystone
 154                          are added as these need to be running on a host that
 155                          has access to the cloud's private network
 156     :param log_level: the logging level
 157     :return: None as the tests will be adding to the 'suite' parameter object
 158     """
 159     # Basic connection tests
 160     suite.addTest(
 161         OSComponentTestCase.parameterize(
 162             GlanceSmokeTests, os_creds=os_creds, ext_net_name=ext_net_name,
 163             log_level=log_level))
 164
 165     if use_keystone:
 166         suite.addTest(
 167             OSComponentTestCase.parameterize(
 168                 KeystoneSmokeTests, os_creds=os_creds,
 169                 ext_net_name=ext_net_name, log_level=log_level))
 170
 171     suite.addTest(
 172         OSComponentTestCase.parameterize(
 173             NeutronSmokeTests, os_creds=os_creds, ext_net_name=ext_net_name,
 174             log_level=log_level))
 175     suite.addTest(
 176         OSComponentTestCase.parameterize(
 177             NovaSmokeTests, os_creds=os_creds, ext_net_name=ext_net_name,
 178             log_level=log_level))
 179     suite.addTest(
 180         OSComponentTestCase.parameterize(
 181             HeatSmokeTests, os_creds=os_creds, ext_net_name=ext_net_name,
 182             log_level=log_level))
 183
 184
 185 def add_openstack_api_tests(suite, os_creds, ext_net_name, use_keystone=True,
 186                             image_metadata=None, log_level=logging.INFO):
 187     """
 188     Adds tests written to exercise all existing OpenStack APIs
 189     :param suite: the unittest.TestSuite object to which to add the tests
 190     :param os_creds: Instance of OSCreds that holds the credentials
 191                      required by OpenStack
 192     :param ext_net_name: the name of an external network on the cloud under
 193                          test
 194     :param use_keystone: when True, tests requiring direct access to Keystone
 195                          are added as these need to be running on a host that
 196                          has access to the cloud's private network
 197     :param image_metadata: dict() object containing metadata for creating an
 198                            image with custom config
 199                            (see YAML files in examples/image-metadata)
 200     :param log_level: the logging level
 201     :return: None as the tests will be adding to the 'suite' parameter object
 202     """
 203     # Tests the OpenStack API calls
 204     if use_keystone:
 205         suite.addTest(OSComponentTestCase.parameterize(
 206             KeystoneUtilsTests, os_creds=os_creds, ext_net_name=ext_net_name,
 207             log_level=log_level))
 208         suite.addTest(OSComponentTestCase.parameterize(
 209             CreateUserSuccessTests, os_creds=os_creds,
 210             ext_net_name=ext_net_name, log_level=log_level))
 211         suite.addTest(OSComponentTestCase.parameterize(
 212             CreateProjectSuccessTests, os_creds=os_creds,
 213             ext_net_name=ext_net_name, log_level=log_level))
 214         suite.addTest(OSComponentTestCase.parameterize(
 215             CreateProjectUserTests, os_creds=os_creds,
 216             ext_net_name=ext_net_name, log_level=log_level))
 217
 218     suite.addTest(OSComponentTestCase.parameterize(
 219         GlanceUtilsTests, os_creds=os_creds, ext_net_name=ext_net_name,
 220         image_metadata=image_metadata,
 221         log_level=log_level))
 222     suite.addTest(OSComponentTestCase.parameterize(
 223         NeutronUtilsNetworkTests, os_creds=os_creds, ext_net_name=ext_net_name,
 224         log_level=log_level))
 225     suite.addTest(OSComponentTestCase.parameterize(
 226         NeutronUtilsSubnetTests, os_creds=os_creds, ext_net_name=ext_net_name,
 227         log_level=log_level))
 228     suite.addTest(OSComponentTestCase.parameterize(
 229         NeutronUtilsRouterTests, os_creds=os_creds, ext_net_name=ext_net_name,
 230         log_level=log_level))
 231     suite.addTest(OSComponentTestCase.parameterize(
 232         NeutronUtilsSecurityGroupTests, os_creds=os_creds,
 233         ext_net_name=ext_net_name, log_level=log_level))
 234     suite.addTest(OSComponentTestCase.parameterize(
 235         NeutronUtilsFloatingIpTests, os_creds=os_creds,
 236         ext_net_name=ext_net_name, log_level=log_level))
 237     suite.addTest(OSComponentTestCase.parameterize(
 238         NovaUtilsKeypairTests, os_creds=os_creds, ext_net_name=ext_net_name,
 239         log_level=log_level))
 240     suite.addTest(OSComponentTestCase.parameterize(
 241         NovaUtilsFlavorTests, os_creds=os_creds, ext_net_name=ext_net_name,
 242         log_level=log_level))
 243     suite.addTest(OSComponentTestCase.parameterize(
 244         NovaUtilsInstanceTests, os_creds=os_creds, ext_net_name=ext_net_name,
 245         log_level=log_level, image_metadata=image_metadata))
 246     suite.addTest(OSComponentTestCase.parameterize(
 247         CreateFlavorTests, os_creds=os_creds, ext_net_name=ext_net_name,
 248         log_level=log_level))
 249     suite.addTest(OSComponentTestCase.parameterize(
 250         HeatUtilsCreateStackTests, os_creds=os_creds,
 251         ext_net_name=ext_net_name, log_level=log_level,
 252         image_metadata=image_metadata))
 253
 254
 255 def add_openstack_integration_tests(suite, os_creds, ext_net_name,
 256                                     use_keystone=True, flavor_metadata=None,
 257                                     image_metadata=None, use_floating_ips=True,
 258                                     log_level=logging.INFO):
 259     """
 260     Adds tests written to exercise all long-running OpenStack integration tests
 261     meaning they will be creating VM instances and potentially performing some
 262     SSH functions through floatingIPs
 263     :param suite: the unittest.TestSuite object to which to add the tests
 264     :param os_creds: and instance of OSCreds that holds the credentials
 265                      required by OpenStack
 266     :param ext_net_name: the name of an external network on the cloud under
 267                          test
 268     :param use_keystone: when True, tests requiring direct access to Keystone
 269                          are added as these need to be running on a host that
 270                          has access to the cloud's private network
 271     :param image_metadata: dict() object containing metadata for creating an
 272                            image with custom config
 273                            (see YAML files in examples/image-metadata)
 274     :param flavor_metadata: dict() object containing the metadata required by
 275                             your flavor based on your configuration:
 276                             (i.e. {'hw:mem_page_size': 'large'})
 277     :param use_floating_ips: when true, all tests requiring Floating IPs will
 278                              be added to the suite
 279     :param log_level: the logging level
 280     :return: None as the tests will be adding to the 'suite' parameter object
 281     """
 282     # Tests the OpenStack API calls via a creator. If use_keystone, objects
 283     # will be created with a custom user and project
 284
 285     # Creator Object tests
 286     suite.addTest(OSIntegrationTestCase.parameterize(
 287         CreateSecurityGroupTests, os_creds=os_creds, ext_net_name=ext_net_name,
 288         use_keystone=use_keystone,
 289         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 290         log_level=log_level))
 291     suite.addTest(OSIntegrationTestCase.parameterize(
 292         CreateImageSuccessTests, os_creds=os_creds, ext_net_name=ext_net_name,
 293         use_keystone=use_keystone,
 294         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 295         log_level=log_level))
 296     suite.addTest(OSIntegrationTestCase.parameterize(
 297         CreateImageNegativeTests, os_creds=os_creds, ext_net_name=ext_net_name,
 298         use_keystone=use_keystone,
 299         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 300         log_level=log_level))
 301     suite.addTest(OSIntegrationTestCase.parameterize(
 302         CreateMultiPartImageTests, os_creds=os_creds,
 303         ext_net_name=ext_net_name, use_keystone=use_keystone,
 304         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 305         log_level=log_level))
 306     suite.addTest(OSIntegrationTestCase.parameterize(
 307         CreateKeypairsTests, os_creds=os_creds, ext_net_name=ext_net_name,
 308         use_keystone=use_keystone,
 309         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 310         log_level=log_level))
 311     suite.addTest(OSIntegrationTestCase.parameterize(
 312         CreateNetworkSuccessTests, os_creds=os_creds,
 313         ext_net_name=ext_net_name, use_keystone=use_keystone,
 314         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 315         log_level=log_level))
 316     suite.addTest(OSIntegrationTestCase.parameterize(
 317         CreateRouterSuccessTests, os_creds=os_creds, ext_net_name=ext_net_name,
 318         use_keystone=use_keystone,
 319         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 320         log_level=log_level))
 321     suite.addTest(OSIntegrationTestCase.parameterize(
 322         CreateRouterNegativeTests, os_creds=os_creds,
 323         ext_net_name=ext_net_name, use_keystone=use_keystone,
 324         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 325         log_level=log_level))
 326
 327     # VM Instances
 328     suite.addTest(OSIntegrationTestCase.parameterize(
 329         SimpleHealthCheck, os_creds=os_creds, ext_net_name=ext_net_name,
 330         use_keystone=use_keystone,
 331         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 332         log_level=log_level))
 333     suite.addTest(OSIntegrationTestCase.parameterize(
 334         CreateInstanceSimpleTests, os_creds=os_creds,
 335         ext_net_name=ext_net_name, use_keystone=use_keystone,
 336         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 337         log_level=log_level))
 338     suite.addTest(OSIntegrationTestCase.parameterize(
 339         CreateInstancePortManipulationTests, os_creds=os_creds,
 340         ext_net_name=ext_net_name, use_keystone=use_keystone,
 341         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 342         log_level=log_level))
 343     suite.addTest(OSIntegrationTestCase.parameterize(
 344         InstanceSecurityGroupTests, os_creds=os_creds,
 345         ext_net_name=ext_net_name, use_keystone=use_keystone,
 346         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 347         log_level=log_level))
 348     suite.addTest(OSIntegrationTestCase.parameterize(
 349         CreateInstanceOnComputeHost, os_creds=os_creds,
 350         ext_net_name=ext_net_name, use_keystone=use_keystone,
 351         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 352         log_level=log_level))
 353     suite.addTest(OSIntegrationTestCase.parameterize(
 354         CreateInstanceFromThreePartImage, os_creds=os_creds,
 355         ext_net_name=ext_net_name, use_keystone=use_keystone,
 356         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 357         log_level=log_level))
 358     suite.addTest(OSIntegrationTestCase.parameterize(
 359         CreateStackSuccessTests, os_creds=os_creds, ext_net_name=ext_net_name,
 360         use_keystone=use_keystone,
 361         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 362         log_level=log_level))
 363     suite.addTest(OSIntegrationTestCase.parameterize(
 364         CreateStackNegativeTests, os_creds=os_creds, ext_net_name=ext_net_name,
 365         use_keystone=use_keystone,
 366         flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 367         log_level=log_level))
 368
 369     if use_floating_ips:
 370         suite.addTest(OSIntegrationTestCase.parameterize(
 371             CreateInstanceSingleNetworkTests, os_creds=os_creds,
 372             ext_net_name=ext_net_name, use_keystone=use_keystone,
 373             flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 374             log_level=log_level))
 375         suite.addTest(OSIntegrationTestCase.parameterize(
 376             CreateInstancePubPrivNetTests, os_creds=os_creds,
 377             ext_net_name=ext_net_name, use_keystone=use_keystone,
 378             flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 379             log_level=log_level))
 380         suite.addTest(OSIntegrationTestCase.parameterize(
 381             AnsibleProvisioningTests, os_creds=os_creds,
 382             ext_net_name=ext_net_name, use_keystone=use_keystone,
 383             flavor_metadata=flavor_metadata, image_metadata=image_metadata,
 384             log_level=log_level))
 385
 386
 387 def add_openstack_staging_tests(suite, os_creds, ext_net_name,
 388                                 log_level=logging.INFO):
 389     """
 390     Adds tests that are still in development have not been designed to run
 391     successfully against all OpenStack pods
 392     :param suite: the unittest.TestSuite object to which to add the tests
 393     :param os_creds: Instance of OSCreds that holds the credentials
 394                     required by OpenStack
 395     :param ext_net_name: the name of an external network on the cloud under
 396                          test
 397     :param log_level: the logging level
 398     :return: None as the tests will be adding to the 'suite' parameter object
 399     """
 400     suite.addTest(OSComponentTestCase.parameterize(
 401         CreateNetworkTypeTests, os_creds=os_creds, ext_net_name=ext_net_name,
 402         log_level=log_level))
 403     suite.addTest(OSComponentTestCase.parameterize(
 404         CreateInstanceMockOfflineTests, os_creds=os_creds,
 405         ext_net_name=ext_net_name, log_level=log_level))