Code Review / functest.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | review | tree
raw | inline | side by side
Skip Patrole test_show_auto_allocated_topology
[functest.git] / functest / ci / testcases.yaml
diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index 23f0620..8fe3327 100644 (file)
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -146,31 +146,37 @@ tiers:
                             - /src/odl_test/csit/suites/openstack/neutron
 
             -
                             - /src/odl_test/csit/suites/openstack/neutron
 
             -
-                case_name: api_check
+                case_name: tempest_smoke
                 project_name: functest
                 criteria: 100
                 project_name: functest
                 criteria: 100
-                blocking: true
+                blocking: false
                 description: >-
                 description: >-
-                    This test case verifies the retrieval of OpenStack clients:
-                    Keystone, Glance, Neutron and Nova and may perform some
-                    simple queries. When the config value of
-                    snaps.use_keystone is True, functest must have access to
-                    the cloud's private network.
+                    This test case runs the smoke subset of the OpenStack
+                    Tempest suite. The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deplopyment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
                 run:
                 run:
-                    name: api_check
+                    name: tempest_common
+                    args:
+                        mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
+                        option:
+                            - '--concurrency=4'
 
             -
 
             -
-                case_name: snaps_health_check
+                case_name: tempest_horizon
                 project_name: functest
                 criteria: 100
                 project_name: functest
                 criteria: 100
-                blocking: true
+                blocking: false
                 description: >-
                 description: >-
-                    This test case creates executes the SimpleHealthCheck
-                    Python test class which creates an, image, flavor, network,
-                    and Cirros VM instance and observes the console output to
-                    validate the single port obtains the correct IP address.
+                    This test case runs the Tempest suite proposed by the
+                    Horizon project.
+                dependencies:
+                    - DASHBOARD_URL: '^(?!\s*$).+'
                 run:
                 run:
-                    name: snaps_health_check
+                    name: tempest_horizon
+                    args:
+                        mode: '^tempest_horizon.'
 
     -
         name: smoke
 
     -
         name: smoke
@@ -179,37 +185,85 @@ tiers:
             Set of basic Functional tests to validate the OPNFV scenarios.
         testcases:
             -
             Set of basic Functional tests to validate the OPNFV scenarios.
         testcases:
             -
-                case_name: tempest_smoke
+                case_name: tempest_neutron
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
-                    This test case runs the smoke subset of the OpenStack
-                    Tempest suite. The list of test cases is generated by
+                    This test case runs the Tempest suite proposed by the
+                    Neutron project. The list of test cases is generated by
                     Tempest automatically and depends on the parameters of
                     Tempest automatically and depends on the parameters of
-                    the OpenStack deplopyment.
-                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
+                    the OpenStack deployment.
                 run:
                     name: tempest_common
                     args:
                 run:
                     name: tempest_common
                     args:
-                        mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
+                        mode: '^neutron_tempest_plugin\.api'
                         option:
                             - '--concurrency=4'
 
             -
                         option:
                             - '--concurrency=4'
 
             -
-                case_name: neutron-tempest-plugin-api
+                case_name: tempest_cinder
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
                     This test case runs the Tempest suite proposed by the
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
                     This test case runs the Tempest suite proposed by the
-                    Neutron project. The list of test cases is generated by
-                    Tempest automatically and depends on the parameters of
-                    the OpenStack deployment.
+                    Cinder project.
                 run:
                     name: tempest_common
                     args:
                 run:
                     name: tempest_common
                     args:
-                        mode: '^neutron_tempest_plugin\.api'
+                        mode:
+                            '(?!.*test_incremental_backup)(^cinder_tempest_plugin.)'
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: tempest_keystone
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs the Tempest suite proposed by the
+                    Keystone project.
+                run:
+                    name: tempest_common
+                    args:
+                        mode: 'keystone_tempest_plugin.'
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: tempest_heat
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs the Tempest suite proposed by the
+                    Heat project.
+                run:
+                    name: tempest_heat
+                    args:
+                        mode: '^heat_tempest_plugin.tests'
+                        option:
+                            - '--concurrency=1'
+
+            -
+                case_name: tempest_telemetry
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs the Tempest suite proposed by the
+                    Telemetry project.
+                run:
+                    name: tempest_common
+                    args:
+                        mode: "(?!.*test_autoscaling)(?!.*test_live)\
+                            (^telemetry_tempest_plugin)"
+                        services:
+                            - aodh
+                            - gnocchi
+                            - panko
                         option:
                             - '--concurrency=4'
 
                         option:
                             - '--concurrency=4'
 
@@ -223,138 +277,172 @@ tiers:
                     Rally suite in smoke mode.
                 run:
                     name: rally_sanity
                     Rally suite in smoke mode.
                 run:
                     name: rally_sanity
+                    args:
+                        optional:
+                            - 'gnocchi'
+                            - 'barbican'
 
             -
 
             -
-                case_name: refstack_defcore
+                case_name: refstack_compute
                 project_name: functest
                 criteria: 100
                 blocking: false
                 project_name: functest
                 criteria: 100
                 blocking: false
+                deny_skipping: true
+                tests_count: 222
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
-                    Defcore testcases.
+                    Compute testcases.
                 run:
                 run:
-                    name: refstack_defcore
+                    name: refstack
                     args:
                     args:
+                        target: compute
                         option:
                             - '--concurrency=4'
 
             -
                         option:
                             - '--concurrency=4'
 
             -
-                case_name: patrole
+                case_name: refstack_object
                 project_name: functest
                 criteria: 100
                 blocking: false
                 project_name: functest
                 criteria: 100
                 blocking: false
+                deny_skipping: true
+                tests_count: 58
                 description: >-
                 description: >-
-                    Test suite from Patrole project.
+                    This test case runs a sub group of tests of the OpenStack
+                    Object testcases.
                 run:
                 run:
-                    name: patrole
+                    name: refstack
                     args:
                     args:
-                        role: 'admin'
-                        includes:
-                            - 'image'
-                            - 'network'
-                        excludes:
-                            - 'test_networks_multiprovider_rbac'
+                        target: object
+                        option:
+                            - '--concurrency=4'
 
             -
 
             -
-                case_name: vmtp
+                case_name: refstack_platform
                 project_name: functest
                 criteria: 100
                 blocking: false
                 project_name: functest
                 criteria: 100
                 blocking: false
+                deny_skipping: true
+                tests_count: 278
                 description: >-
                 description: >-
-                    VMTP is a small python application that will automatically
-                    perform ping connectivity, round trip time measurement
-                    (latency) and TCP/UDP throughput
-                dependencies:
-                    - POD_ARCH: '^(?!aarch64$)'
+                    This test case runs a sub group of tests of the OpenStack
+                    Platform testcases.
                 run:
                 run:
-                    name: vmtp
+                    name: refstack
+                    args:
+                        target: platform
+                        option:
+                            - '--concurrency=4'
 
             -
 
             -
-                case_name: shaker
+                case_name: tempest_full
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
-                    Shaker wraps around popular system network testing tools
-                    like iperf, iperf3 and netperf (with help of flent). Shaker
-                    is able to deploy OpenStack instances and networks in
-                    different topologies.
+                    The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
                 run:
                 run:
-                    name: shaker
+                    name: tempest_common
+                    args:
+                        mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
+                        option:
+                            - '--concurrency=4'
 
             -
 
             -
-                case_name: snaps_smoke
+                case_name: tempest_scenario
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
-                    This test case contains tests that setup and destroy
-                    environments with VMs with and without Floating IPs
-                    with a newly created user and project. Set the config
-                    value snaps.use_floating_ips (True|False) to toggle
-                    this functionality. When the config value of
-                    snaps.use_keystone is True, functest must have access to
-                    the cloud's private network.
+                    The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
                 run:
                 run:
-                    name: snaps_smoke
+                    name: tempest_common
+                    args:
+                        mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
+                        option:
+                            - '--concurrency=1'
 
             -
 
             -
-                case_name: neutron_trunk
+                case_name: tempest_slow
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
-                    This test case runs the neutron trunk subtest of the
-                    OpenStack Tempest suite. The list of test cases is
-                    generated by Tempest having as input the relevant
-                    testcase list file.
+                    The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
                 run:
                     name: tempest_common
                     args:
                 run:
                     name: tempest_common
                     args:
-                        mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
-                        neutron_extensions:
-                            - trunk
-                            - trunk-details
+                        mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)'
+                        option:
+                            - '--concurrency=1'
 
             -
 
             -
-                case_name: networking-bgpvpn
+                case_name: patrole_admin
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
-                    This test case runs OpenStack networking-bgpvpn Tempest
-                    suite. The list of test cases is generated by Tempest
-                    having as input the relevant regex.
+                    Test suite from Patrole project.
                 run:
                 run:
-                    name: tempest_common
+                    name: patrole
                     args:
                     args:
-                        mode: '^networking_bgpvpn_tempest\.'
-                        neutron_extensions:
-                            - bgpvpn
+                        roles: 'admin'
+                        mode: "(?!.*test_unmanage_snapshot_rbac)\
+                          (?!.*test_show_auto_allocated_topology)\
+                          (?!.*test_delete_auto_allocated_topology)\
+                          (?=.*[.*\bslow\b.*])\
+                          (^patrole_tempest_plugin.tests.api.(image|network|volume))"
                         option:
                             - '--concurrency=4'
 
             -
                         option:
                             - '--concurrency=4'
 
             -
-                case_name: networking-sfc
+                case_name: patrole_member
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
                 project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
-                    This test case runs OpenStack networking-sfc Tempest
-                    suite. The list of test cases is generated by Tempest
-                    having as input the relevant regex.
+                    Test suite from Patrole project.
                 run:
                 run:
-                    name: tempest_common
+                    name: patrole
                     args:
                     args:
-                        mode:
-                            '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$'
-                        neutron_extensions:
-                            - sfc
+                        roles: 'member'
+                        mode: "(?!.*test_unmanage_snapshot_rbac)\
+                          (?!.*test_show_auto_allocated_topology)\
+                          (?!.*test_delete_auto_allocated_topology)\
+                          (?=.*[.*\bslow\b.*])\
+                          (^patrole_tempest_plugin.tests.api.(image|network|volume))"
                         option:
                         option:
-                            - '--concurrency=0'
+                            - '--concurrency=4'
 
             -
 
             -
-                case_name: barbican
+                case_name: patrole_reader
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    Test suite from Patrole project.
+                run:
+                    name: patrole
+                    args:
+                        roles: 'reader'
+                        mode: "(?!.*test_unmanage_snapshot_rbac)\
+                          (?!.*test_show_auto_allocated_topology)\
+                          (?!.*test_delete_auto_allocated_topology)\
+                          (?=.*[.*\bslow\b.*])\
+                          (^patrole_tempest_plugin.tests.api.(image|network|volume))"
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: tempest_barbican
                 project_name: functest
                 criteria: 100
                 blocking: false
                 project_name: functest
                 criteria: 100
                 blocking: false
@@ -362,126 +450,220 @@ tiers:
                     It leverages on the tempest plugin containing tests used to
                     verify the functionality of a barbican installation.
                 run:
                     It leverages on the tempest plugin containing tests used to
                     verify the functionality of a barbican installation.
                 run:
-                    name: tempest_common
+                    name: barbican
                     args:
                         mode:
                     args:
                         mode:
-                            'barbican_tempest_plugin.tests.(api|scenario).(?!test_image_signing)'
+                            '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
                         services:
                             - barbican
                         services:
                             - barbican
+                        option:
+                            - '--concurrency=4'
 
 
-    -
-        name: features
-        order: 2
-        description: >-
-            Test suites from feature projects
-            integrated in functest
-        testcases:
             -
             -
-                case_name: doctor-notification
-                project_name: doctor
+                case_name: tempest_octavia
+                project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
                 criteria: 100
                 blocking: false
                 description: >-
-                    Test suite from Doctor project.
-                dependencies:
-                    - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)'
-                    - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$'
+                    It leverages on the tempest plugin containing tests used to
+                    verify the functionality of an octavia installation.
                 run:
                 run:
-                    name: bashfeature
+                    name: tempest_common
                     args:
                     args:
-                        cmd: 'doctor-test'
+                        mode: "(?!.*api.v2.test_availability_zone)\
+                            (?!.*api.v2.test_availability_zone_profile)\
+                            (?!.*api.v2.test_member.MemberAPITest.test_member_ipv4_create)\
+                            (?!.*api.v2.test_member.MemberAPITest.test_member_ipv6_create)\
+                            (^octavia_tempest_plugin.tests.(api|scenario))"
+                        services:
+                            - octavia
+                        option:
+                            - '--concurrency=2'
 
             -
 
             -
-                case_name: bgpvpn
-                project_name: sdnvpn
+                case_name: tempest_cyborg
+                project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
                 criteria: 100
                 blocking: false
                 description: >-
-                    Test suite from SDNVPN project.
-                dependencies:
-                    - DEPLOY_SCENARIO: 'bgpvpn'
+                    It leverages on the tempest plugin containing tests used to
+                    verify the functionality of a cyborg installation.
                 run:
                 run:
-                    name: bgpvpn
+                    name: tempest_common
+                    args:
+                        mode: '^cyborg_tempest_plugin'
+                        services:
+                            - cyborg
+                        option:
+                            - '--concurrency=4'
 
 
+    -
+        name: smoke_cntt
+        order: 2
+        description: >-
+            Set of basic Functional tests to validate the OPNFV scenarios.
+        testcases:
             -
             -
-                case_name: functest-odl-sfc
-                project_name: sfc
+                case_name: tempest_neutron_cntt
+                project_name: functest
                 criteria: 100
                 blocking: false
                 criteria: 100
                 blocking: false
+                deny_skipping: true
+                tests_count: 515
                 description: >-
                 description: >-
-                    Test suite for odl-sfc to test two chains with one SF and
-                    one chain with two SFs
-                dependencies:
-                    - DEPLOY_SCENARIO: 'odl.*sfc'
+                    This test case runs the Tempest suite proposed by the
+                    Neutron project. The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
                 run:
                 run:
-                    name: functest-odl-sfc
+                    name: tempest_common
+                    args:
+                        mode: "(?!.*admin.test_agent_availability_zone)\
+                            (?!.*admin.test_dhcp_agent_scheduler)\
+                            (?!.*admin.test_l3_agent_scheduler)\
+                            (?!.*admin.test_logging)\
+                            (?!.*admin.test_logging_negative)\
+                            (?!.*admin.test_network_segment_range)\
+                            (?!.*admin.test_ports.PortTestCasesAdmin.test_regenerate_mac_address)\
+                            (?!.*admin.test_ports.PortTestCasesResourceRequest)\
+                            (?!.*admin.test_routers_dvr)\
+                            (?!.*admin.test_routers_flavors)\
+                            (?!.*admin.test_routers_ha)\
+                            (?!.*test_conntrack_helper)\
+                            (?!.*test_floating_ips.FloatingIPPoolTestJSON)\
+                            (?!.*test_floating_ips.FloatingIPTestJSON.test_create_update_floatingip_port_details)\
+                            (?!.*test_metering_extensions)\
+                            (?!.*test_metering_negative)\
+                            (?!.*test_networks.NetworksSearchCriteriaTest.test_list_validation_filters)\
+                            (?!.*test_networks.NetworksTestAdmin.test_create_tenant_network_vxlan)\
+                            (?!.*test_networks.NetworksTestJSON.test_create_update_network_dns_domain)\
+                            (?!.*test_port_forwardings)\
+                            (?!.*test_ports.PortsTaggingOnCreation)\
+                            (?!.*test_ports.PortsTestJSON.test_create_port_with_propagate_uplink_status)\
+                            (?!.*test_ports.PortsTestJSON.test_create_port_without_propagate_uplink_status)\
+                            (?!.*test_ports.PortsTestJSON.test_create_update_port_with_dns_domain)\
+                            (?!.*test_ports.PortsTestJSON.test_create_update_port_with_dns_name)\
+                            (?!.*test_ports.PortsTestJSON.test_create_update_port_with_no_dns_name)\
+                            (?!.*test_qos.QosMinimumBandwidthRuleTestJSON)\
+                            (?!.*test_revisions.TestRevisions.test_update_dns_domain_bumps_revision)\
+                            (?!.*test_revisions.TestRevisions.test_update_router_extra_attributes_bumps_revision)\
+                            (?!.*test_router_interface_fip)\
+                            (?!.*test_routers.DvrRoutersTest)\
+                            (?!.*test_routers.HaRoutersTest)\
+                            (?!.*test_routers.RoutersIpV6Test.test_extra_routes_atomic)\
+                            (?!.*test_routers.RoutersTest.test_extra_routes_atomic)\
+                            (?!.*test_routers_negative.DvrRoutersNegativeTest)\
+                            (?!.*test_routers_negative.DvrRoutersNegativeTestExtended)\
+                            (?!.*test_routers_negative.HaRoutersNegativeTest)\
+                            (?!.*test_security_groups.RbacSharedSecurityGroupTest)\
+                            (?!.*test_subnetpool_prefix_ops)\
+                            (?!.*test_subnetpools.RbacSubnetPoolTest)\
+                            (?!.*test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_validation_filters)\
+                            (?!.*test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_tenant_create_subnetpool_associate_shared_address_scope)\
+                            (?!.*test_subnets.SubnetsSearchCriteriaTest.test_list_validation_filters)\
+                            (?!.*test_timestamp.TestTimeStamp.test_segment_with_timestamp)\
+                            (?!.*test_trunk.TrunkTestInheritJSONBase.test_add_subport)\
+                            (?!.*test_trunk.TrunkTestMtusJSON)\
+                            (?!.*test_trunk_negative.TrunkTestJSON.test_create_subport_invalid_inherit_network_segmentation_type)\
+                            (?!.*test_trunk_negative.TrunkTestMtusJSON)\
+                            (^neutron_tempest_plugin.api)"
+                        option:
+                            - '--concurrency=4'
 
             -
 
             -
-                case_name: barometercollectd
-                project_name: barometer
+                case_name: tempest_cinder_cntt
+                project_name: functest
                 criteria: 100
                 blocking: false
                 criteria: 100
                 blocking: false
+                deny_skipping: true
+                tests_count: 8
                 description: >-
                 description: >-
-                    Test suite for the Barometer project. Separate tests verify
-                    the proper configuration and basic functionality of all the
-                    collectd plugins as described in the Project Release Plan
-                dependencies:
-                    - DEPLOY_SCENARIO: 'bar'
+                    This test case runs the Tempest suite proposed by the
+                    Cinder project.
                 run:
                 run:
-                    name: barometercollectd
+                    name: tempest_common
+                    args:
+                        mode: "(?!.*test_incremental_backup)\
+                            (?!.*test_consistencygroups)\
+                            (^cinder_tempest_plugin.)"
+                        option:
+                            - '--concurrency=4'
 
             -
 
             -
-                case_name: fds
-                project_name: fastdatastacks
+                case_name: tempest_keystone_cntt
+                project_name: functest
                 criteria: 100
                 blocking: false
                 criteria: 100
                 blocking: false
+                deny_skipping: true
+                tests_count: 27
                 description: >-
                 description: >-
-                    Test Suite for the OpenDaylight SDN Controller when GBP
-                    features are installed. It integrates some test suites from
-                    upstream using Robot as the test framework.
-                dependencies:
-                    - DEPLOY_SCENARIO: 'odl.*-fdio'
+                    This test case runs the Tempest suite proposed by the
+                    Keystone project.
                 run:
                 run:
-                    name: odl
+                    name: tempest_common
                     args:
                     args:
-                        suites:
-                            - /src/fds/testing/robot
+                        mode: "(?!.*api.identity.v3.test_oauth1_tokens)\
+                            (?!.*scenario.test_federated_authentication)\
+                            keystone_tempest_plugin."
+                        option:
+                            - '--concurrency=4'
+
             -
             -
-                case_name: vgpu
+                case_name: tempest_heat_cntt
                 project_name: functest
                 criteria: 100
                 blocking: false
                 project_name: functest
                 criteria: 100
                 blocking: false
+                deny_skipping: true
+                tests_count: 119
                 description: >-
                 description: >-
-                    Test suite for the OpenStack vGPU feature
-                dependencies:
-                    - DEPLOY_SCENARIO: 'vgpu'
+                    This test case runs the Tempest suite proposed by the
+                    Heat project.
                 run:
                 run:
-                    name: vgpu
+                    name: tempest_heat
+                    args:
+                        mode: "(?!.*functional.test_lbaasv2)\
+                          (?!.*functional.test_encryption_vol_type)\
+                          (?!.*RemoteStackTest.test_stack_create_with_cloud_credential)\
+                          (?!.*scenario.test_aodh_alarm)\
+                          (?!.*tests.scenario.test_autoscaling_lb)\
+                          (?!.*scenario.test_autoscaling_lbv2)\
+                          (?!.*scenario.test_server_software_config)\
+                          (?!.*test_volumes.VolumeBackupRestoreIntegrationTest)\
+                          (?!.*scenario.test_octavia_lbaas)\
+                          (?!.*scenario.test_server_cfn_init)\
+                          ^heat_tempest_plugin.tests"
+                        option:
+                            - '--concurrency=1'
 
             -
 
             -
-                case_name: stor4nfv_os
-                project_name: stor4nfv
+                case_name: rally_sanity_cntt
+                project_name: functest
                 criteria: 100
                 blocking: false
                 description: >-
                 criteria: 100
                 blocking: false
                 description: >-
-                    This tests are necessary demonstrate conformance
-                    of the OpenStack+Stor4NFV deployment.
-                dependencies:
-                    - DEPLOY_SCENARIO: 'stor4nfv'
+                    This test case runs a sub group of tests of the OpenStack
+                    Rally suite in smoke mode.
                 run:
                 run:
-                    name: stor4nfv_os
+                    name: rally_sanity
+                    args:
+                        tests:
+                            - 'authenticate'
+                            - 'glance'
+                            - 'cinder'
+                            - 'heat'
+                            - 'keystone'
+                            - 'neutron'
+                            - 'nova'
+                            - 'quotas'
+                            - 'swift'
 
 
-    -
-        name: components
-        order: 3
-        description: >-
-            Extensive testing of OpenStack API.
-        testcases:
             -
             -
-                case_name: tempest_full
+                case_name: tempest_full_cntt
                 project_name: functest
                 criteria: 100
                 blocking: false
                 project_name: functest
                 criteria: 100
                 blocking: false
+                deny_skipping: true
+                tests_count: 1280
                 description: >-
                     The list of test cases is generated by
                     Tempest automatically and depends on the parameters of
                 description: >-
                     The list of test cases is generated by
                     Tempest automatically and depends on the parameters of
@@ -490,15 +672,88 @@ tiers:
                 run:
                     name: tempest_common
                     args:
                 run:
                     name: tempest_common
                     args:
-                        mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
+                        mode: "(?!.*test_fixed_ips)\
+                          (?!.*test_fixed_ips_negative)\
+                          (?!.*test_auto_allocate_network)(?!.*test_floating_ips_bulk)\
+                          (?!.*test_flavors_microversions.FlavorsV255TestJSON)\
+                          (?!.*test_flavors_microversions.FlavorsV261TestJSON)\
+                          (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_iscsi_volume)\
+                          (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_volume_backed_live_migration)\
+                          (?!.*test_live_migration.LiveMigrationTest.test_iscsi_volume)\
+                          (?!.*test_live_migration.LiveMigrationTest.test_volume_backed_live_migration)\
+                          (?!.*test_live_migration.LiveMigrationRemoteConsolesV26Test)\
+                          (?!.*test_quotas.QuotasAdminTestV257)\
+                          (?!.*certificates.test_certificates)\
+                          (?!.*test_quotas_negative.QuotasSecurityGroupAdminNegativeTest)\
+                          (?!.*test_novnc)(?!.*test_server_personality)\
+                          (?!.*test_servers.ServerShowV263Test.test_show_update_rebuild_list_server)\
+                          (?!.*test_servers_microversions.ServerShowV254Test)\
+                          (?!.*test_servers_microversions.ServerShowV257Test)\
+                          (?!.*test_servers_negative.ServersNegativeTestJSON.test_personality_file_contents_not_encoded)\
+                          (?!.*servers.test_virtual_interfaces)\
+                          (?!.*test_server_actions.ServerActionsTestJSON.test_change_server_password)\
+                          (?!.*test_server_actions.ServerActionsTestJSON.test_get_vnc_console)\
+                          (?!.*test_server_actions.ServerActionsTestJSON.test_reboot_server_soft)\
+                          (?!.*test_server_rescue.ServerBootFromVolumeStableRescueTest)\
+                          (?!.*test_server_rescue.ServerStableDeviceRescueTest)\
+                          (?!.*test_security_group_default_rules)\
+                          (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_duplicate_name)\
+                          (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_description)\
+                          (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_name)\
+                          (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_des)\
+                          (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_id)\
+                          (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_name)\
+                          (?!.*test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_ip_regex)\
+                          (?!.*compute.test_virtual_interfaces)(?!.*compute.test_virtual_interfaces_negative)\
+                          (?!.*compute.test_networks)\
+                          (?!.*test_attach_volume.AttachVolumeMultiAttach)\
+                          (?!.*identity.admin.v2)(?!.*identity.v2)\
+                          (?!.*image.v1)\
+                          (?!.*admin.test_dhcp_agent_scheduler)\
+                          (?!.*admin.test_routers_dvr)\
+                          (?!.*test_metering_extensions)(?!.*network.test_tags)\
+                          (?!.*test_routers_negative.DvrRoutersNegativeTest)\
+                          (?!.*test_routers.RoutersIpV6Test.test_create_router_set_gateway_with_fixed_ip)\
+                          (?!.*test_routers.RoutersTest.test_create_router_set_gateway_with_fixed_ip)\
+                          (?!.*test_group_snapshots.GroupSnapshotsV319Test.test_reset_group_snapshot_status)\
+                          (?!.*test_multi_backend)\
+                          (?!.*test_volume_retype.VolumeRetypeWithMigrationTest)\
+                          (?!.*test_volume_delete_cascade.VolumesDeleteCascade.test_volume_from_snapshot_cascade_delete)\
+                          (?!.*test_volumes_backup.VolumesBackupsTest.test_volume_backup_create_get_detailed_list_restore_delete)\
+                          (?!.*test_volumes_negative.UpdateMultiattachVolumeNegativeTest.test_multiattach_rw_volume_update_failure)\
+                          (?!.*test_volumes_extend.VolumesExtendAttachedTest.test_extend_attached_volume)\
+                          (?!.*\\[.*\\bslow\\b.*\\])(^tempest.api)"
                         option:
                             - '--concurrency=4'
 
             -
                         option:
                             - '--concurrency=4'
 
             -
-                case_name: tempest_scenario
+                case_name: tempest_scenario_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                deny_skipping: true
+                tests_count: 9
+                description: >-
+                    The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
+                run:
+                    name: tempest_scenario
+                    args:
+                        mode: "\
+                          (?!.*test_volume_boot_pattern.TestVolumeBootPattern.test_boot_server_from_encrypted_volume_luks)\
+                          (?!.*\\[.*\\bslow\\b.*\\])(^tempest.scenario)"
+                        option:
+                            - '--concurrency=1'
+
+            -
+                case_name: tempest_slow_cntt
                 project_name: functest
                 criteria: 100
                 blocking: false
                 project_name: functest
                 criteria: 100
                 blocking: false
+                deny_skipping: true
+                tests_count: 43
                 description: >-
                     The list of test cases is generated by
                     Tempest automatically and depends on the parameters of
                 description: >-
                     The list of test cases is generated by
                     Tempest automatically and depends on the parameters of
@@ -507,10 +762,27 @@ tiers:
                 run:
                     name: tempest_common
                     args:
                 run:
                     name: tempest_common
                     args:
-                        mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
+                        mode: "(?!.*test_volume_swap)\
+                          (?!.*test_server_personality)\
+                          (?!.*test_server_rescue.ServerBootFromVolumeStableRescueTest)\
+                          (?!.*test_container_sync.ContainerSyncTest.test_container_synchronization)\
+                          (?!.*test_container_sync_middleware.ContainerSyncMiddlewareTest.test_container_synchronization)\
+                          (?!.*test_encrypted_cinder_volumes)\
+                          (?!.*test_minbw_allocation_placement)\
+                          (?!.*test_network_basic_ops.TestNetworkBasicOps.test_router_rescheduling)\
+                          (?!.*test_volume_migrate_attached)\
+                          (?!.*test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_cold_migration_revert)\
+                          (?=.*\\[.*\\bslow\\b.*\\])(^tempest.)"
                         option:
                             - '--concurrency=1'
 
                         option:
                             - '--concurrency=1'
 
+    -
+        name: benchmarking
+        order: 3
+        description: >-
+            Run several OpenStack performance tools
+            https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
+        testcases:
             -
                 case_name: rally_full
                 project_name: functest
             -
                 case_name: rally_full
                 project_name: functest
@@ -521,10 +793,100 @@ tiers:
                     OpenStack Rally suite using several threads and iterations.
                 run:
                     name: rally_full
                     OpenStack Rally suite using several threads and iterations.
                 run:
                     name: rally_full
+                    args:
+                        optional:
+                            - 'gnocchi'
+                            - 'barbican'
+
+            -
+                case_name: rally_jobs
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs a group of Rally jobs used in
+                    OpenStack gating
+                run:
+                    name: rally_jobs
+                    args:
+                        optional:
+                            - 'gnocchi'
+
+            -
+                case_name: vmtp
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    VMTP is a small python application that will automatically
+                    perform ping connectivity, round trip time measurement
+                    (latency) and TCP/UDP throughput
+                dependencies:
+                    - POD_ARCH: '^(?!aarch64$)'
+                run:
+                    name: vmtp
+
+            -
+                case_name: shaker
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    Shaker wraps around popular system network testing tools
+                    like iperf, iperf3 and netperf (with help of flent). Shaker
+                    is able to deploy OpenStack instances and networks in
+                    different topologies.
+                dependencies:
+                    - POD_ARCH: '^(?!aarch64$)'
+                run:
+                    name: shaker
+
+    -
+        name: benchmarking_cntt
+        order: 4
+        description: >-
+            Run several OpenStack performance tools
+            https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
+        testcases:
+            -
+                case_name: rally_full_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs the full suite of scenarios of the
+                    OpenStack Rally suite using several threads and iterations.
+                run:
+                    name: rally_full
+                    args:
+                        tests:
+                            - 'authenticate'
+                            - 'glance'
+                            - 'cinder'
+                            - 'heat'
+                            - 'keystone'
+                            - 'neutron'
+                            - 'nova'
+                            - 'quotas'
+                            - 'swift'
+
+            -
+                case_name: rally_jobs_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs a group of Rally jobs used in
+                    OpenStack gating
+                run:
+                    name: rally_jobs
+                    args:
+                        tests:
+                            - 'neutron'
 
     -
         name: vnf
 
     -
         name: vnf
-        order: 4
+        order: 5
         description: >-
             Collection of VNF test cases.
         testcases:
         description: >-
             Collection of VNF test cases.
         testcases:
@@ -535,8 +897,6 @@ tiers:
                 blocking: false
                 description: >-
                     This test case deploys the Cloudify orchestrator.
                 blocking: false
                 description: >-
                     This test case deploys the Cloudify orchestrator.
-                dependencies:
-                    - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
                 run:
                     name: cloudify
             -
                 run:
                     name: cloudify
             -
@@ -562,8 +922,6 @@ tiers:
                     This test case deploys an OpenSource vIMS solution from
                     Clearwater using the OpenStack Heat orchestrator.
                     It also runs some signaling traffic.
                     This test case deploys an OpenSource vIMS solution from
                     Clearwater using the OpenStack Heat orchestrator.
                     It also runs some signaling traffic.
-                dependencies:
-                    - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
                 run:
                     name: heat_ims
 
                 run:
                     name: heat_ims
 
@@ -574,8 +932,6 @@ tiers:
                 blocking: false
                 description: >-
                     This test case is vRouter testing.
                 blocking: false
                 description: >-
                     This test case is vRouter testing.
-                dependencies:
-                    - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
                 run:
                     name: vyos_vrouter
 
                 run:
                     name: vyos_vrouter
 
@@ -587,7 +943,5 @@ tiers:
                 description: >-
                     vEPC validation with Juju as VNF manager and ABoT as test
                     executor.
                 description: >-
                     vEPC validation with Juju as VNF manager and ABoT as test
                     executor.
-                dependencies:
-                    - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
                 run:
                     name: juju_epc
                 run:
                     name: juju_epc
Test suites and cases to verify OPNFV Platform functionality