X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=functest%2Fci%2Ftestcases.yaml;h=a49a79b16a6274ef51f72b91eac1de53d66b0920;hb=bb4c274ab05b2e90788995e92cb5e125c0474683;hp=a9e023e4c67156972f733f80a1e9fc5e2f3c413b;hpb=e1dc928517e4c852b25cbb8f7455e49d7ce3ede7;p=functest.git diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index a9e023e4c..a49a79b16 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -17,9 +17,7 @@ tiers: Keystone, Glance, Neutron and Nova and may perform some simple queries. run: - module: - 'functest.opnfv_tests.openstack.api.connection_check' - class: 'ConnectionCheck' + name: connection_check - case_name: tenantnetwork1 @@ -31,8 +29,7 @@ tiers: required by advanced testcases (subnet, network and router). run: - module: 'functest.core.tenantnetwork' - class: 'TenantNetwork1' + name: tenantnetwork1 - case_name: tenantnetwork2 @@ -44,8 +41,7 @@ tiers: all tenant network ressources required by a testcase (subnet, network and router). run: - module: 'functest.core.tenantnetwork' - class: 'TenantNetwork2' + name: tenantnetwork2 - case_name: vmready1 @@ -57,8 +53,7 @@ tiers: resources and prepares a future VM attached to that network. run: - module: 'functest.core.singlevm' - class: 'VmReady1' + name: vmready1 - case_name: vmready2 @@ -70,8 +65,7 @@ tiers: all tenant network ressources, flavors, images, etc. required by advanced testcases. run: - module: 'functest.core.singlevm' - class: 'VmReady2' + name: vmready2 - case_name: singlevm1 @@ -83,8 +77,7 @@ tiers: resources and completes it by booting a VM attached to that network. run: - module: 'functest.core.singlevm' - class: 'SingleVm1' + name: singlevm1 - case_name: singlevm2 @@ -96,8 +89,7 @@ tiers: all tenant network ressources and vms required by advanced testcases. run: - module: 'functest.core.singlevm' - class: 'SingleVm2' + name: singlevm2 - case_name: vping_ssh @@ -109,8 +101,7 @@ tiers: floating IPs over the public network. 2) Connectivity between 2 instances over a private network. run: - module: 'functest.opnfv_tests.openstack.vping.vping_ssh' - class: 'VPingSSH' + name: vping_ssh - case_name: vping_userdata @@ -121,9 +112,7 @@ tiers: This test case verifies: 1) Boot a VM with given userdata. 2) Connectivity between 2 instances over a private network. run: - module: - 'functest.opnfv_tests.openstack.vping.vping_userdata' - class: 'VPingUserdata' + name: vping_userdata - case_name: cinder_test @@ -136,8 +125,7 @@ tiers: from instance 1, attach it on instance 2 3) Read volume data run: - module: 'functest.opnfv_tests.openstack.cinder.cinder_test' - class: 'CinderCheck' + name: cinder_test - case_name: odl @@ -151,45 +139,44 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'odl' run: - module: 'functest.opnfv_tests.sdn.odl.odl' - class: 'ODLTests' + name: odl args: suites: - /src/odl_test/csit/suites/integration/basic - /src/odl_test/csit/suites/openstack/neutron - - case_name: api_check + case_name: tempest_smoke project_name: functest criteria: 100 - blocking: true + blocking: false description: >- - This test case verifies the retrieval of OpenStack clients: - Keystone, Glance, Neutron and Nova and may perform some - simple queries. When the config value of - snaps.use_keystone is True, functest must have access to - the cloud's private network. - dependencies: - - DEPLOY_SCENARIO: '^((?!lxd).)*$' + This test case runs the smoke subset of the OpenStack + Tempest suite. The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deplopyment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114 run: - module: 'functest.opnfv_tests.openstack.snaps.api_check' - class: 'ApiCheck' + name: tempest_common + args: + mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)' + option: + - '--concurrency=4' - - case_name: snaps_health_check + case_name: tempest_horizon project_name: functest criteria: 100 - blocking: true + blocking: false description: >- - This test case creates executes the SimpleHealthCheck - Python test class which creates an, image, flavor, network, - and Cirros VM instance and observes the console output to - validate the single port obtains the correct IP address. + This test case runs the Tempest suite proposed by the + Horizon project. dependencies: - - DEPLOY_SCENARIO: '^((?!lxd).)*$' + - DASHBOARD_URL: '^(?!\s*$).+' run: - module: 'functest.opnfv_tests.openstack.snaps.health_check' - class: 'HealthCheck' + name: tempest_horizon + args: + mode: '^tempest_horizon.' - name: smoke @@ -198,21 +185,52 @@ tiers: Set of basic Functional tests to validate the OPNFV scenarios. testcases: - - case_name: tempest_smoke + case_name: neutron-tempest-plugin-api project_name: functest criteria: 100 blocking: false description: >- - This test case runs the smoke subset of the OpenStack - Tempest suite. The list of test cases is generated by + This test case runs the Tempest suite proposed by the + Neutron project. The list of test cases is generated by Tempest automatically and depends on the parameters of - the OpenStack deplopyment. + the OpenStack deployment. + run: + name: tempest_common + args: + mode: '^neutron_tempest_plugin\.api' + option: + - '--concurrency=4' + + - + case_name: tempest_cinder + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs the Tempest suite proposed by the + Cinder project. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: - '^(tempest|neutron_tempest_plugin)\.(api|scenario).*\[.*\bsmoke\b.*\]$' + '(?!.*test_incremental_backup)(^cinder_tempest_plugin.)' + option: + - '--concurrency=4' + + - + case_name: tempest_keystone + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs the Tempest suite proposed by the + Keystone project. + run: + name: tempest_common + args: + mode: 'keystone_tempest_plugin.' + option: + - '--concurrency=4' - case_name: rally_sanity @@ -223,88 +241,138 @@ tiers: This test case runs a sub group of tests of the OpenStack Rally suite in smoke mode. run: - module: 'functest.opnfv_tests.openstack.rally.rally' - class: 'RallySanity' + name: rally_sanity + args: + optional: + - 'gnocchi' + - 'barbican' - - case_name: refstack_defcore + case_name: refstack_compute project_name: functest - enabled: false criteria: 100 blocking: false + deny_skipping: true + tests_count: 222 description: >- This test case runs a sub group of tests of the OpenStack - Defcore testcases. + Compute testcases. run: - module: - 'functest.opnfv_tests.openstack.refstack.refstack' - class: 'Refstack' + name: refstack + args: + target: compute + option: + - '--concurrency=4' - - case_name: patrole + case_name: refstack_object project_name: functest criteria: 100 blocking: false + deny_skipping: true + tests_count: 58 description: >- - Test suite from Patrole project. + This test case runs a sub group of tests of the OpenStack + Object testcases. run: - module: 'functest.opnfv_tests.openstack.patrole.patrole' - class: 'Patrole' + name: refstack args: - role: 'admin' - services: - - 'image' - - 'network' - exclude: - - 'test_networks_multiprovider_rbac' + target: object + option: + - '--concurrency=4' - - case_name: vmtp + case_name: refstack_platform project_name: functest criteria: 100 blocking: false + deny_skipping: true + tests_count: 278 description: >- - VMTP is a small python application that will automatically - perform ping connectivity, round trip time measurement - (latency) and TCP/UDP throughput + This test case runs a sub group of tests of the OpenStack + Platform testcases. run: - module: - 'functest.opnfv_tests.openstack.vmtp.vmtp' - class: 'Vmtp' + name: refstack + args: + target: platform + option: + - '--concurrency=4' - - case_name: shaker + case_name: tempest_full project_name: functest criteria: 100 blocking: false description: >- - Shaker wraps around popular system network testing tools - like iperf, iperf3 and netperf (with help of flent). Shaker - is able to deploy OpenStack instances and networks in - different topologies. + The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83 run: - module: - 'functest.opnfv_tests.openstack.shaker.shaker' - class: 'Shaker' + name: tempest_common + args: + mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' + option: + - '--concurrency=4' - - case_name: snaps_smoke + case_name: tempest_scenario project_name: functest criteria: 100 blocking: false description: >- - This test case contains tests that setup and destroy - environments with VMs with and without Floating IPs - with a newly created user and project. Set the config - value snaps.use_floating_ips (True|False) to toggle - this functionality. When the config value of - snaps.use_keystone is True, functest must have access to - the cloud's private network. - dependencies: - - DEPLOY_SCENARIO: '^((?!lxd).)*$' + The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84 + run: + name: tempest_scenario + args: + mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' + option: + - '--concurrency=1' + + - + case_name: tempest_slow + project_name: functest + criteria: 100 + blocking: false + description: >- + The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84 run: - module: 'functest.opnfv_tests.openstack.snaps.smoke' - class: 'SnapsSmoke' + name: tempest_common + args: + mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)' + option: + - '--concurrency=2' + + - + case_name: patrole + project_name: functest + criteria: 100 + blocking: false + description: >- + Test suite from Patrole project. + run: + name: patrole + args: + role: 'admin' + includes: + - 'image' + - 'network' + excludes: + - 'network.test_flavors_rbac' + - 'network.test_network_ip_availability_rbac' + - 'test_networks_multiprovider_rbac' + - 'test_auto_allocated_topology_rbac' + - 'test_create_network_provider_segmentation_id' + - 'test_update_network_provider_physical_network' + - 'test_update_network_provider_segmentation_id' + - 'network.test_agents_rbac' + - 'test_list_metadef_namespaces' - case_name: neutron_trunk @@ -317,145 +385,148 @@ tiers: generated by Tempest having as input the relevant testcase list file. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: 'neutron_tempest_plugin.(api|scenario).test_trunk' neutron_extensions: - trunk - - trunk_details + - trunk-details - - case_name: barbican + case_name: networking-bgpvpn project_name: functest criteria: 100 blocking: false description: >- - It leverages on the tempest plugin containing tests used to - verify the functionality of a barbican installation. + This test case runs OpenStack networking-bgpvpn Tempest + suite. The list of test cases is generated by Tempest + having as input the relevant regex. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: - mode: 'barbican_tempest_plugin.tests.(api|scenario)' - services: - - barbican + mode: '^neutron_tempest_plugin.bgpvpn.api\.' + neutron_extensions: + - bgpvpn + option: + - '--concurrency=4' - - - name: features - order: 2 - description: >- - Test suites from feature projects - integrated in functest - testcases: - - case_name: doctor-notification - project_name: doctor + case_name: networking-sfc + project_name: functest criteria: 100 blocking: false description: >- - Test suite from Doctor project. - dependencies: - - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)' - - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$' + This test case runs OpenStack networking-sfc Tempest + suite. The list of test cases is generated by Tempest + having as input the relevant regex. run: - module: 'xtesting.core.feature' - class: 'BashFeature' + name: tempest_common args: - cmd: 'doctor-test' + mode: + '^(?:neutron_tempest_plugin\.sfc\.tests\.api).*$' + neutron_extensions: + - sfc + option: + - '--concurrency=0' - - case_name: bgpvpn - project_name: sdnvpn + case_name: barbican + project_name: functest criteria: 100 blocking: false description: >- - Test suite from SDNVPN project. - dependencies: - - DEPLOY_SCENARIO: 'bgpvpn' + It leverages on the tempest plugin containing tests used to + verify the functionality of a barbican installation. run: - module: 'sdnvpn.test.functest.run_sdnvpn_tests' - class: 'SdnvpnFunctest' + name: barbican + args: + mode: + '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$' + services: + - barbican - - case_name: functest-odl-sfc - project_name: sfc + case_name: octavia + project_name: functest criteria: 100 blocking: false description: >- - Test suite for odl-sfc to test two chains with one SF and - one chain with two SFs - dependencies: - - DEPLOY_SCENARIO: 'odl.*sfc' + It leverages on the tempest plugin containing tests used to + verify the functionality of an octavia installation. run: - module: 'sfc.tests.functest.run_sfc_tests' - class: 'SfcFunctest' + name: tempest_common + args: + mode: + '^octavia_tempest_plugin.tests.api.v2' + services: + - octavia + option: + - '--concurrency=2' + - + name: benchmarking + order: 2 + description: >- + Run several OpenStack performance tools + https://docs.openstack.org/performance-docs/latest/methodologies/tools.html + testcases: - - case_name: barometercollectd - project_name: barometer + case_name: rally_full + project_name: functest criteria: 100 blocking: false description: >- - Test suite for the Barometer project. Separate tests verify - the proper configuration and basic functionality of all the - collectd plugins as described in the Project Release Plan - dependencies: - - DEPLOY_SCENARIO: 'bar' + This test case runs the full suite of scenarios of the + OpenStack Rally suite using several threads and iterations. run: - module: 'baro_tests.barometer' - class: 'BarometerCollectd' + name: rally_full + args: + optional: + - 'gnocchi' + - 'barbican' - - case_name: fds - project_name: fastdatastacks + case_name: rally_jobs + project_name: functest criteria: 100 blocking: false description: >- - Test Suite for the OpenDaylight SDN Controller when GBP - features are installed. It integrates some test suites from - upstream using Robot as the test framework. - dependencies: - - DEPLOY_SCENARIO: 'odl.*-fdio' + This test case runs a group of Rally jobs used in + OpenStack gating run: - module: 'functest.opnfv_tests.sdn.odl.odl' - class: 'ODLTests' + name: rally_jobs args: - suites: - - /src/fds/testing/robot + optional: + - 'gnocchi' - - - name: components - order: 3 - description: >- - Extensive testing of OpenStack API. - testcases: - - case_name: tempest_full + case_name: vmtp project_name: functest - criteria: 80 + criteria: 100 blocking: false description: >- - The list of test cases is generated by - Tempest automatically and depends on the parameters of - the OpenStack deplopyment. + VMTP is a small python application that will automatically + perform ping connectivity, round trip time measurement + (latency) and TCP/UDP throughput + dependencies: + - POD_ARCH: '^(?!aarch64$)' run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' - args: - mode: '^(tempest|neutron_tempest_plugin)\.' + name: vmtp - - case_name: rally_full + case_name: shaker project_name: functest criteria: 100 blocking: false description: >- - This test case runs the full suite of scenarios of the - OpenStack Rally suite using several threads and iterations. + Shaker wraps around popular system network testing tools + like iperf, iperf3 and netperf (with help of flent). Shaker + is able to deploy OpenStack instances and networks in + different topologies. + dependencies: + - POD_ARCH: '^(?!aarch64$)' run: - module: 'functest.opnfv_tests.openstack.rally.rally' - class: 'RallyFull' + name: shaker - name: vnf @@ -470,15 +541,12 @@ tiers: blocking: false description: >- This test case deploys the Cloudify orchestrator. - dependencies: - - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.core.cloudify' - class: 'Cloudify' + name: cloudify - case_name: cloudify_ims project_name: functest - criteria: 80 + criteria: 100 blocking: false description: >- This test case deploys an OpenSource vIMS solution from @@ -487,8 +555,19 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.opnfv_tests.vnf.ims.cloudify_ims' - class: 'CloudifyIms' + name: cloudify_ims + + - + case_name: heat_ims + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case deploys an OpenSource vIMS solution from + Clearwater using the OpenStack Heat orchestrator. + It also runs some signaling traffic. + run: + name: heat_ims - case_name: vyos_vrouter @@ -497,11 +576,8 @@ tiers: blocking: false description: >- This test case is vRouter testing. - dependencies: - - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.opnfv_tests.vnf.router.cloudify_vrouter' - class: 'CloudifyVrouter' + name: vyos_vrouter - case_name: juju_epc @@ -511,8 +587,5 @@ tiers: description: >- vEPC validation with Juju as VNF manager and ABoT as test executor. - dependencies: - - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.opnfv_tests.vnf.epc.juju_epc' - class: 'JujuEpc' + name: juju_epc