X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;ds=sidebyside;f=functest%2Fci%2Ftestcases.yaml;h=32fbacd51071916a340d239ebdf53e3e7d8784c7;hb=5beb864bf6e1409e367a3fdf33289cb5386079fa;hp=2b348d5870052ea7d774c3bb01c1156074f6fe15;hpb=213ca8e4e4d155ed6d5626020e56c3a241d1dace;p=functest.git diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index 2b348d587..32fbacd51 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -3,7 +3,6 @@ tiers: - name: healthcheck order: 0 - ci_loop: '(merge)|(verify)|(daily)|(weekly)' description: >- First tier to be executed to verify the basic operations in the VIM. @@ -16,17 +15,149 @@ tiers: description: >- This test case verifies the retrieval of OpenStack clients: Keystone, Glance, Neutron and Nova and may perform some - simple queries. When the config value of - snaps.use_keystone is True, functest must have access to - the cloud's private network. - dependencies: - installer: '' - scenario: '' + simple queries. run: module: - 'functest.opnfv_tests.openstack.snaps.connection_check' + 'functest.opnfv_tests.openstack.api.connection_check' class: 'ConnectionCheck' + - + case_name: tenantnetwork1 + project_name: functest + criteria: 100 + blocking: true + description: >- + It creates and configures all tenant network ressources + required by advanced testcases (subnet, network and + router). + run: + module: 'functest.core.tenantnetwork' + class: 'TenantNetwork1' + + - + case_name: tenantnetwork2 + project_name: functest + criteria: 100 + blocking: true + description: >- + It creates new user/project before creating and configuring + all tenant network ressources required by a testcase + (subnet, network and router). + run: + module: 'functest.core.tenantnetwork' + class: 'TenantNetwork2' + + - + case_name: vmready1 + project_name: functest + criteria: 100 + blocking: true + description: >- + It inherits from TenantNetwork1 which creates all network + resources and prepares a future VM attached to that + network. + run: + module: 'functest.core.singlevm' + class: 'VmReady1' + + - + case_name: vmready2 + project_name: functest + criteria: 100 + blocking: true + description: >- + It creates new user/project before creating and configuring + all tenant network ressources, flavors, images, etc. + required by advanced testcases. + run: + module: 'functest.core.singlevm' + class: 'VmReady2' + + - + case_name: singlevm1 + project_name: functest + criteria: 100 + blocking: true + description: >- + It inherits from TenantNetwork1 which creates all network + resources and completes it by booting a VM attached to that + network. + run: + module: 'functest.core.singlevm' + class: 'SingleVm1' + + - + case_name: singlevm2 + project_name: functest + criteria: 100 + blocking: true + description: >- + It creates new user/project before creating and configuring + all tenant network ressources and vms required by advanced + testcases. + run: + module: 'functest.core.singlevm' + class: 'SingleVm2' + + - + case_name: vping_ssh + project_name: functest + criteria: 100 + blocking: true + description: >- + This test case verifies: 1) SSH to an instance using + floating IPs over the public network. 2) Connectivity + between 2 instances over a private network. + run: + module: 'functest.opnfv_tests.openstack.vping.vping_ssh' + class: 'VPingSSH' + + - + case_name: vping_userdata + project_name: functest + criteria: 100 + blocking: true + description: >- + This test case verifies: 1) Boot a VM with given userdata. + 2) Connectivity between 2 instances over a private network. + run: + module: + 'functest.opnfv_tests.openstack.vping.vping_userdata' + class: 'VPingUserdata' + + - + case_name: cinder_test + project_name: functest + criteria: 100 + blocking: true + description: >- + This test case verifies: 1) Attach volume and to 1 + instance; 2) Write data on volume 3) Detach volume + from instance 1, attach it on instance 2 3) Read volume + data + run: + module: 'functest.opnfv_tests.openstack.cinder.cinder_test' + class: 'CinderCheck' + + - + case_name: odl + project_name: functest + criteria: 100 + blocking: true + description: >- + Test Suite for the OpenDaylight SDN Controller. It + integrates some test suites from upstream using + Robot as the test framework. + dependencies: + - DEPLOY_SCENARIO: 'odl' + run: + module: 'functest.opnfv_tests.sdn.odl.odl' + class: 'ODLTests' + args: + suites: + - /src/odl_test/csit/suites/integration/basic + - /src/odl_test/csit/suites/openstack/neutron + - case_name: api_check project_name: functest @@ -39,8 +170,7 @@ tiers: snaps.use_keystone is True, functest must have access to the cloud's private network. dependencies: - installer: '' - scenario: '^((?!lxd).)*$' + - DEPLOY_SCENARIO: '^((?!lxd).)*$' run: module: 'functest.opnfv_tests.openstack.snaps.api_check' class: 'ApiCheck' @@ -56,8 +186,7 @@ tiers: and Cirros VM instance and observes the console output to validate the single port obtains the correct IP address. dependencies: - installer: '' - scenario: '^((?!lxd).)*$' + - DEPLOY_SCENARIO: '^((?!lxd).)*$' run: module: 'functest.opnfv_tests.openstack.snaps.health_check' class: 'HealthCheck' @@ -65,58 +194,45 @@ tiers: - name: smoke order: 1 - ci_loop: '(daily)|(weekly)' description: >- Set of basic Functional tests to validate the OPNFV scenarios. testcases: - - case_name: vping_ssh - project_name: functest - criteria: 100 - blocking: false - description: >- - This test case verifies: 1) SSH to an instance using - floating IPs over the public network. 2) Connectivity - between 2 instances over a private network. - dependencies: - installer: '' - scenario: '^((?!lxd).)*$' - run: - module: 'functest.opnfv_tests.openstack.vping.vping_ssh' - class: 'VPingSSH' - - - - case_name: vping_userdata + case_name: tempest_smoke project_name: functest criteria: 100 blocking: false description: >- - This test case verifies: 1) Boot a VM with given userdata. - 2) Connectivity between 2 instances over a private network. - dependencies: - installer: '' - scenario: '^((?!lxd).)*$' + This test case runs the smoke subset of the OpenStack + Tempest suite. The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deplopyment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114 run: - module: - 'functest.opnfv_tests.openstack.vping.vping_userdata' - class: 'VPingUserdata' + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)' + option: + - '--concurrency=4' - - case_name: tempest_smoke_serial + case_name: neutron-tempest-plugin-api project_name: functest criteria: 100 blocking: false description: >- - This test case runs the smoke subset of the OpenStack - Tempest suite. The list of test cases is generated by + This test case runs the Tempest suite proposed by the + Neutron project. The list of test cases is generated by Tempest automatically and depends on the parameters of - the OpenStack deplopyment. - dependencies: - installer: '' - scenario: '' + the OpenStack deployment. run: module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestSmokeSerial' + class: 'TempestCommon' + args: + mode: '^neutron_tempest_plugin\.api' + option: + - '--concurrency=4' - case_name: rally_sanity @@ -126,9 +242,6 @@ tiers: description: >- This test case runs a sub group of tests of the OpenStack Rally suite in smoke mode. - dependencies: - installer: '' - scenario: '' run: module: 'functest.opnfv_tests.openstack.rally.rally' class: 'RallySanity' @@ -142,9 +255,6 @@ tiers: description: >- This test case runs a sub group of tests of the OpenStack Defcore testcases. - dependencies: - installer: '' - scenario: '' run: module: 'functest.opnfv_tests.openstack.refstack.refstack' @@ -157,39 +267,47 @@ tiers: blocking: false description: >- Test suite from Patrole project. - dependencies: - installer: '' - scenario: '' run: module: 'functest.opnfv_tests.openstack.patrole.patrole' class: 'Patrole' args: role: 'admin' - services: + includes: - 'image' - 'network' - exclude: + excludes: - 'test_networks_multiprovider_rbac' - - case_name: odl + case_name: vmtp project_name: functest criteria: 100 blocking: false description: >- - Test Suite for the OpenDaylight SDN Controller. It - integrates some test suites from upstream using - Robot as the test framework. + VMTP is a small python application that will automatically + perform ping connectivity, round trip time measurement + (latency) and TCP/UDP throughput dependencies: - installer: '' - scenario: 'odl' + - POD_ARCH: '^(?!aarch64$)' run: - module: 'functest.opnfv_tests.sdn.odl.odl' - class: 'ODLTests' - args: - suites: - - /src/odl_test/csit/suites/integration/basic - - /src/odl_test/csit/suites/openstack/neutron + module: + 'functest.opnfv_tests.openstack.vmtp.vmtp' + class: 'Vmtp' + + - + case_name: shaker + project_name: functest + criteria: 100 + blocking: false + description: >- + Shaker wraps around popular system network testing tools + like iperf, iperf3 and netperf (with help of flent). Shaker + is able to deploy OpenStack instances and networks in + different topologies. + run: + module: + 'functest.opnfv_tests.openstack.shaker.shaker' + class: 'Shaker' - case_name: snaps_smoke @@ -204,10 +322,8 @@ tiers: this functionality. When the config value of snaps.use_keystone is True, functest must have access to the cloud's private network. - dependencies: - installer: '' - scenario: '^((?!lxd).)*$' + - DEPLOY_SCENARIO: '^((?!lxd).)*$' run: module: 'functest.opnfv_tests.openstack.snaps.smoke' class: 'SnapsSmoke' @@ -222,17 +338,74 @@ tiers: OpenStack Tempest suite. The list of test cases is generated by Tempest having as input the relevant testcase list file. - dependencies: - installer: '(fuel)|(apex)|(compass)' - scenario: '' run: module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestNeutronTrunk' + class: 'TempestCommon' + args: + mode: 'neutron_tempest_plugin.(api|scenario).test_trunk' + neutron_extensions: + - trunk + - trunk-details + + - + case_name: networking-bgpvpn + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs OpenStack networking-bgpvpn Tempest + suite. The list of test cases is generated by Tempest + having as input the relevant regex. + run: + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: '^networking_bgpvpn_tempest\.' + neutron_extensions: + - bgpvpn + option: + - '--concurrency=4' + + - + case_name: networking-sfc + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs OpenStack networking-sfc Tempest + suite. The list of test cases is generated by Tempest + having as input the relevant regex. + run: + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: + '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$' + neutron_extensions: + - sfc + option: + - '--concurrency=0' + + - + case_name: barbican + project_name: functest + criteria: 100 + blocking: false + description: >- + It leverages on the tempest plugin containing tests used to + verify the functionality of a barbican installation. + run: + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: + 'barbican_tempest_plugin.tests.(api|scenario).(?!test_image_signing)' + services: + - barbican - name: features order: 2 - ci_loop: '(daily)|(weekly)' description: >- Test suites from feature projects integrated in functest @@ -245,8 +418,8 @@ tiers: description: >- Test suite from Doctor project. dependencies: - installer: '(apex)|(fuel)|(daisy)' - scenario: '^((?!fdio|nofeature).)*$' + - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)' + - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$' run: module: 'xtesting.core.feature' class: 'BashFeature' @@ -261,8 +434,7 @@ tiers: description: >- Test suite from SDNVPN project. dependencies: - installer: '' - scenario: 'bgpvpn' + - DEPLOY_SCENARIO: 'bgpvpn' run: module: 'sdnvpn.test.functest.run_sdnvpn_tests' class: 'SdnvpnFunctest' @@ -276,8 +448,7 @@ tiers: Test suite for odl-sfc to test two chains with one SF and one chain with two SFs dependencies: - installer: '' - scenario: 'odl.*sfc' + - DEPLOY_SCENARIO: 'odl.*sfc' run: module: 'sfc.tests.functest.run_sfc_tests' class: 'SfcFunctest' @@ -292,8 +463,7 @@ tiers: the proper configuration and basic functionality of all the collectd plugins as described in the Project Release Plan dependencies: - installer: '' - scenario: 'bar' + - DEPLOY_SCENARIO: 'bar' run: module: 'baro_tests.barometer' class: 'BarometerCollectd' @@ -308,8 +478,7 @@ tiers: features are installed. It integrates some test suites from upstream using Robot as the test framework. dependencies: - installer: '' - scenario: 'odl.*-fdio' + - DEPLOY_SCENARIO: 'odl.*-fdio' run: module: 'functest.opnfv_tests.sdn.odl.odl' class: 'ODLTests' @@ -320,25 +489,44 @@ tiers: - name: components order: 3 - ci_loop: 'weekly' description: >- Extensive testing of OpenStack API. testcases: - - case_name: tempest_full_parallel + case_name: tempest_full project_name: functest - criteria: 80 + criteria: 100 blocking: false description: >- The list of test cases is generated by Tempest automatically and depends on the parameters of - the OpenStack deplopyment. - dependencies: - installer: '' - scenario: '' + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83 + run: + module: 'functest.opnfv_tests.openstack.tempest.tempest' + class: 'TempestCommon' + args: + mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' + option: + - '--concurrency=4' + + - + case_name: tempest_scenario + project_name: functest + criteria: 100 + blocking: false + description: >- + The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84 run: module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestFullParallel' + class: 'TempestCommon' + args: + mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' + option: + - '--concurrency=1' - case_name: rally_full @@ -348,9 +536,6 @@ tiers: description: >- This test case runs the full suite of scenarios of the OpenStack Rally suite using several threads and iterations. - dependencies: - installer: '' - scenario: '' run: module: 'functest.opnfv_tests.openstack.rally.rally' class: 'RallyFull' @@ -358,26 +543,51 @@ tiers: - name: vnf order: 4 - ci_loop: '(daily)|(weekly)' description: >- Collection of VNF test cases. testcases: + - + case_name: cloudify + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case deploys the Cloudify orchestrator. + dependencies: + - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' + run: + module: 'functest.core.cloudify' + class: 'Cloudify' - case_name: cloudify_ims project_name: functest - criteria: 80 + criteria: 100 blocking: false description: >- This test case deploys an OpenSource vIMS solution from Clearwater using the Cloudify orchestrator. It also runs some signaling traffic. dependencies: - installer: '' - scenario: 'os-.*-nofeature-.*ha' + - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: module: 'functest.opnfv_tests.vnf.ims.cloudify_ims' class: 'CloudifyIms' + - + case_name: heat_ims + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case deploys an OpenSource vIMS solution from + Clearwater using the OpenStack Heat orchestrator. + It also runs some signaling traffic. + dependencies: + - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' + run: + module: 'functest.opnfv_tests.vnf.ims.heat_ims' + class: 'HeatIms' + - case_name: vyos_vrouter project_name: functest @@ -386,8 +596,7 @@ tiers: description: >- This test case is vRouter testing. dependencies: - installer: '' - scenario: 'os-.*-nofeature-.*ha' + - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: module: 'functest.opnfv_tests.vnf.router.cloudify_vrouter' class: 'CloudifyVrouter' @@ -401,8 +610,7 @@ tiers: vEPC validation with Juju as VNF manager and ABoT as test executor. dependencies: - installer: '' - scenario: 'os-.*-nofeature-.*ha' + - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: module: 'functest.opnfv_tests.vnf.epc.juju_epc' class: 'JujuEpc'