Conform with OpenStack gates

It adds testcases and updates regexes to conform with OpenStack gates:
  - tempest-full
  - neutron-tempest-plugin-api

It should be noted that tempest-smoke is no longer run in gates and
two testcases are available for tempest-full as we can't manage
Tempest combine.

Criteria is set to 100% for tempest_full (it works in OpenStack
gates).

JIRA: FUNCTEST-1000

Change-Id: I77a0f533255483bdf61f0a5f60c9c437ce843591
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

diff --git a/docker/components/testcases.yaml b/docker/components/testcases.yaml
index babe3e6..468cd8b 100644 (file)
--- a/docker/components/testcases.yaml
+++ b/docker/components/testcases.yaml
@@ -9,17 +9,38 @@ tiers:
             -
                 case_name: tempest_full
                 project_name: functest
-                criteria: 80
+                criteria: 100
+                blocking: false
+                description: >-
+                    The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
+                run:
+                    module: 'functest.opnfv_tests.openstack.tempest.tempest'
+                    class: 'TempestCommon'
+                    args:
+                        mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: tempest_scenario
+                project_name: functest
+                criteria: 100
                 blocking: false
                 description: >-
                     The list of test cases is generated by
                     Tempest automatically and depends on the parameters of
-                    the OpenStack deplopyment.
+                    the OpenStack deployment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
                 run:
                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
                     class: 'TempestCommon'
                     args:
-                        mode: '^(tempest|neutron_tempest_plugin)\.'
+                        mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
+                        option:
+                            - '--serial'
 
             -
                 case_name: rally_full

diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml
index 20112a6..f918bbe 100644 (file)
--- a/docker/smoke/testcases.yaml
+++ b/docker/smoke/testcases.yaml
@@ -16,12 +16,32 @@ tiers:
                     Tempest suite. The list of test cases is generated by
                     Tempest automatically and depends on the parameters of
                     the OpenStack deplopyment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
                 run:
                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
                     class: 'TempestCommon'
                     args:
-                        mode:
-                            '^(tempest|neutron_tempest_plugin)\.(api|scenario).*\[.*\bsmoke\b.*\]$'
+                        mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: neutron-tempest-plugin-api
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs the Tempest suite proposed by the
+                    Neutron project. The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                run:
+                    module: 'functest.opnfv_tests.openstack.tempest.tempest'
+                    class: 'TempestCommon'
+                    args:
+                        mode: '^neutron_tempest_plugin\.api'
+                        option:
+                            - '--concurrency=4'
 
             -
                 case_name: rally_sanity

diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index 511935e..188c708 100644 (file)
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -207,12 +207,32 @@ tiers:
                     Tempest suite. The list of test cases is generated by
                     Tempest automatically and depends on the parameters of
                     the OpenStack deplopyment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
                 run:
                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
                     class: 'TempestCommon'
                     args:
-                        mode:
-                            '^(tempest|neutron_tempest_plugin)\.(api|scenario).*\[.*\bsmoke\b.*\]$'
+                        mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: neutron-tempest-plugin-api
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs the Tempest suite proposed by the
+                    Neutron project. The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                run:
+                    module: 'functest.opnfv_tests.openstack.tempest.tempest'
+                    class: 'TempestCommon'
+                    args:
+                        mode: '^neutron_tempest_plugin\.api'
+                        option:
+                            - '--concurrency=4'
 
             -
                 case_name: rally_sanity
@@ -434,17 +454,38 @@ tiers:
             -
                 case_name: tempest_full
                 project_name: functest
-                criteria: 80
+                criteria: 100
+                blocking: false
+                description: >-
+                    The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
+                run:
+                    module: 'functest.opnfv_tests.openstack.tempest.tempest'
+                    class: 'TempestCommon'
+                    args:
+                        mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: tempest_scenario
+                project_name: functest
+                criteria: 100
                 blocking: false
                 description: >-
                     The list of test cases is generated by
                     Tempest automatically and depends on the parameters of
-                    the OpenStack deplopyment.
+                    the OpenStack deployment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
                 run:
                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
                     class: 'TempestCommon'
                     args:
-                        mode: '^(tempest|neutron_tempest_plugin)\.'
+                        mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
+                        option:
+                            - '--serial'
 
             -
                 case_name: rally_full