Merge "Add keystone-tempest-plugin"

[functest.git] / functest / ci / testcases.yaml

diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index 3e515bf..a49a79b 100644 (file)
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -145,33 +145,6 @@ tiers:
                             - /src/odl_test/csit/suites/integration/basic
                             - /src/odl_test/csit/suites/openstack/neutron
 
                             - /src/odl_test/csit/suites/integration/basic
                             - /src/odl_test/csit/suites/openstack/neutron
 

-            -
-                case_name: api_check
-                project_name: functest
-                criteria: 100
-                blocking: true
-                description: >-
-                    This test case verifies the retrieval of OpenStack clients:
-                    Keystone, Glance, Neutron and Nova and may perform some
-                    simple queries. When the config value of
-                    snaps.use_keystone is True, functest must have access to
-                    the cloud's private network.
-                run:
-                    name: api_check
-
-            -
-                case_name: snaps_health_check
-                project_name: functest
-                criteria: 100
-                blocking: true
-                description: >-
-                    This test case creates executes the SimpleHealthCheck
-                    Python test class which creates an, image, flavor, network,
-                    and Cirros VM instance and observes the console output to
-                    validate the single port obtains the correct IP address.
-                run:
-                    name: snaps_health_check
-

             -
                 case_name: tempest_smoke
                 project_name: functest
             -
                 case_name: tempest_smoke
                 project_name: functest


@@ -190,6 +163,21 @@ tiers:
                         option:
                             - '--concurrency=4'
 
                         option:
                             - '--concurrency=4'
 

+            -
+                case_name: tempest_horizon
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs the Tempest suite proposed by the
+                    Horizon project.
+                dependencies:
+                    - DASHBOARD_URL: '^(?!\s*$).+'
+                run:
+                    name: tempest_horizon
+                    args:
+                        mode: '^tempest_horizon.'
+

     -
         name: smoke
         order: 1
     -
         name: smoke
         order: 1


@@ -213,6 +201,37 @@ tiers:
                         option:
                             - '--concurrency=4'
 
                         option:
                             - '--concurrency=4'
 

+            -
+                case_name: tempest_cinder
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs the Tempest suite proposed by the
+                    Cinder project.
+                run:
+                    name: tempest_common
+                    args:
+                        mode:
+                            '(?!.*test_incremental_backup)(^cinder_tempest_plugin.)'
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: tempest_keystone
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs the Tempest suite proposed by the
+                    Keystone project.
+                run:
+                    name: tempest_common
+                    args:
+                        mode: 'keystone_tempest_plugin.'
+                        option:
+                            - '--concurrency=4'
+

             -
                 case_name: rally_sanity
                 project_name: functest
             -
                 case_name: rally_sanity
                 project_name: functest


@@ -226,18 +245,56 @@ tiers:
                     args:
                         optional:
                             - 'gnocchi'
                     args:
                         optional:
                             - 'gnocchi'

+                            - 'barbican'

 
             -
 
             -

-                case_name: refstack_defcore
+                case_name: refstack_compute

                 project_name: functest
                 criteria: 100
                 blocking: false
                 project_name: functest
                 criteria: 100
                 blocking: false

+                deny_skipping: true
+                tests_count: 222

                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                 description: >-
                     This test case runs a sub group of tests of the OpenStack

-                    Defcore testcases.
+                    Compute testcases.

                 run:
                 run:

-                    name: refstack_defcore
+                    name: refstack

                     args:
                     args:

+                        target: compute
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: refstack_object
+                project_name: functest
+                criteria: 100
+                blocking: false
+                deny_skipping: true
+                tests_count: 58
+                description: >-
+                    This test case runs a sub group of tests of the OpenStack
+                    Object testcases.
+                run:
+                    name: refstack
+                    args:
+                        target: object
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: refstack_platform
+                project_name: functest
+                criteria: 100
+                blocking: false
+                deny_skipping: true
+                tests_count: 278
+                description: >-
+                    This test case runs a sub group of tests of the OpenStack
+                    Platform testcases.
+                run:
+                    name: refstack
+                    args:
+                        target: platform

                         option:
                             - '--concurrency=4'
 
                         option:
                             - '--concurrency=4'
 


@@ -269,12 +326,29 @@ tiers:
                     the OpenStack deployment.
                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
                 run:
                     the OpenStack deployment.
                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
                 run:

-                    name: tempest_common
+                    name: tempest_scenario

                     args:
                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
                         option:
                             - '--concurrency=1'
 
                     args:
                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
                         option:
                             - '--concurrency=1'
 

+            -
+                case_name: tempest_slow
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
+                run:
+                    name: tempest_common
+                    args:
+                        mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)'
+                        option:
+                            - '--concurrency=2'
+

             -
                 case_name: patrole
                 project_name: functest
             -
                 case_name: patrole
                 project_name: functest


@@ -294,22 +368,11 @@ tiers:
                             - 'network.test_network_ip_availability_rbac'
                             - 'test_networks_multiprovider_rbac'
                             - 'test_auto_allocated_topology_rbac'
                             - 'network.test_network_ip_availability_rbac'
                             - 'test_networks_multiprovider_rbac'
                             - 'test_auto_allocated_topology_rbac'

-
-            -
-                case_name: snaps_smoke
-                project_name: functest
-                criteria: 100
-                blocking: false
-                description: >-
-                    This test case contains tests that setup and destroy
-                    environments with VMs with and without Floating IPs
-                    with a newly created user and project. Set the config
-                    value snaps.use_floating_ips (True|False) to toggle
-                    this functionality. When the config value of
-                    snaps.use_keystone is True, functest must have access to
-                    the cloud's private network.
-                run:
-                    name: snaps_smoke
+                            - 'test_create_network_provider_segmentation_id'
+                            - 'test_update_network_provider_physical_network'
+                            - 'test_update_network_provider_segmentation_id'
+                            - 'network.test_agents_rbac'
+                            - 'test_list_metadef_namespaces'

 
             -
                 case_name: neutron_trunk
 
             -
                 case_name: neutron_trunk


@@ -341,7 +404,7 @@ tiers:
                 run:
                     name: tempest_common
                     args:
                 run:
                     name: tempest_common
                     args:

-                        mode: '^networking_bgpvpn_tempest.tests.api\.'
+                        mode: '^neutron_tempest_plugin.bgpvpn.api\.'

                         neutron_extensions:
                             - bgpvpn
                         option:
                         neutron_extensions:
                             - bgpvpn
                         option:


@@ -360,7 +423,7 @@ tiers:
                     name: tempest_common
                     args:
                         mode:
                     name: tempest_common
                     args:
                         mode:

-                            '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$'
+                            '^(?:neutron_tempest_plugin\.sfc\.tests\.api).*$'

                         neutron_extensions:
                             - sfc
                         option:
                         neutron_extensions:
                             - sfc
                         option:


@@ -382,6 +445,24 @@ tiers:
                         services:
                             - barbican
 
                         services:
                             - barbican
 

+            -
+                case_name: octavia
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    It leverages on the tempest plugin containing tests used to
+                    verify the functionality of an octavia installation.
+                run:
+                    name: tempest_common
+                    args:
+                        mode:
+                            '^octavia_tempest_plugin.tests.api.v2'
+                        services:
+                            - octavia
+                        option:
+                            - '--concurrency=2'
+

     -
         name: benchmarking
         order: 2
     -
         name: benchmarking
         order: 2


@@ -402,6 +483,7 @@ tiers:
                     args:
                         optional:
                             - 'gnocchi'
                     args:
                         optional:
                             - 'gnocchi'

+                            - 'barbican'

 
             -
                 case_name: rally_jobs
 
             -
                 case_name: rally_jobs


@@ -441,98 +523,10 @@ tiers:
                     like iperf, iperf3 and netperf (with help of flent). Shaker
                     is able to deploy OpenStack instances and networks in
                     different topologies.
                     like iperf, iperf3 and netperf (with help of flent). Shaker
                     is able to deploy OpenStack instances and networks in
                     different topologies.

-                run:
-                    name: shaker
-
-    -
-        name: features
-        order: 3
-        description: >-
-            Test suites from feature projects
-            integrated in functest
-        testcases:
-            -
-                case_name: doctor-notification
-                project_name: doctor
-                enabled: false
-                criteria: 100
-                blocking: false
-                description: >-
-                    Test suite from Doctor project.

                 dependencies:
                 dependencies:

-                    - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)'
-                    - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$'
-                run:
-                    name: bashfeature
-                    args:
-                        cmd: 'doctor-test'
-
-            -
-                case_name: bgpvpn
-                project_name: sdnvpn
-                enabled: false
-                criteria: 100
-                blocking: false
-                description: >-
-                    Test suite from SDNVPN project.
-                dependencies:
-                    - DEPLOY_SCENARIO: 'bgpvpn'
-                run:
-                    name: bgpvpn
-
-            -
-                case_name: functest-odl-sfc
-                project_name: sfc
-                criteria: 100
-                blocking: false
-                description: >-
-                    Test suite for odl-sfc to test two chains with one SF and
-                    one chain with two SFs
-                dependencies:
-                    - DEPLOY_SCENARIO: 'odl.*sfc'
-                run:
-                    name: functest-odl-sfc
-
-            -
-                case_name: barometercollectd
-                project_name: barometer
-                enabled: false
-                criteria: 100
-                blocking: false
-                description: >-
-                    Test suite for the Barometer project. Separate tests verify
-                    the proper configuration and basic functionality of all the
-                    collectd plugins as described in the Project Release Plan
-                dependencies:
-                    - DEPLOY_SCENARIO: 'bar'
-                run:
-                    name: barometercollectd
-
-            -
-                case_name: vgpu
-                project_name: functest
-                criteria: 100
-                blocking: false
-                description: >-
-                    Test suite for the OpenStack vGPU feature
-                dependencies:
-                    - DEPLOY_SCENARIO: 'vgpu'
-                run:
-                    name: vgpu
-
-            -
-                case_name: stor4nfv_os
-                project_name: stor4nfv
-                enabled: false
-                criteria: 100
-                blocking: false
-                description: >-
-                    This tests are necessary demonstrate conformance
-                    of the OpenStack+Stor4NFV deployment.
-                dependencies:
-                    - DEPLOY_SCENARIO: 'stor4nfv'
+                    - POD_ARCH: '^(?!aarch64$)'

                 run:
                 run:

-                    name: stor4nfv_os
+                    name: shaker

 
     -
         name: vnf
 
     -
         name: vnf