Support running on openstack which enabled https

[functest.git] / functest / ci / testcases.yaml

diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index b1d824f..1a1d0f4 100755 (executable)
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -7,27 +7,16 @@ tiers:
             First tier to be executed to verify the basic
             operations in the VIM.
         testcases:
             First tier to be executed to verify the basic
             operations in the VIM.
         testcases:

-            -
-                name: healthcheck
-                criteria: 'status == "PASS"'
-                blocking: true
-                description: >-
-                    This test case verifies the basic OpenStack services like
-                    Keystone, Glance, Cinder, Neutron and Nova.
-
-                dependencies:
-                    installer: ''
-                    scenario: '^((?!lxd).)*$'

             -
                 name: snaps_health_check
                 criteria: 'status == "PASS"'
             -
                 name: snaps_health_check
                 criteria: 'status == "PASS"'

-                blocking: false
+                blocking: true
+                clean_flag: false

                 description: >-
                     This test case creates executes the SimpleHealthCheck
                     Python test class which creates an, image, flavor, network,
                     and Cirros VM instance and observes the console output to
                     validate the single port obtains the correct IP address.
                 description: >-
                     This test case creates executes the SimpleHealthCheck
                     Python test class which creates an, image, flavor, network,
                     and Cirros VM instance and observes the console output to
                     validate the single port obtains the correct IP address.

-

                 dependencies:
                     installer: ''
                     scenario: '^((?!lxd).)*$'
                 dependencies:
                     installer: ''
                     scenario: '^((?!lxd).)*$'


@@ -38,6 +27,7 @@ tiers:
                 name: connection_check
                 criteria: 'status == "PASS"'
                 blocking: true
                 name: connection_check
                 criteria: 'status == "PASS"'
                 blocking: true

+                clean_flag: false

                 description: >-
                     This test case verifies the retrieval of OpenStack clients:
                     Keystone, Glance, Neutron and Nova and may perform some
                 description: >-
                     This test case verifies the retrieval of OpenStack clients:
                     Keystone, Glance, Neutron and Nova and may perform some


@@ -56,6 +46,7 @@ tiers:
                 name: api_check
                 criteria: 'status == "PASS"'
                 blocking: true
                 name: api_check
                 criteria: 'status == "PASS"'
                 blocking: true

+                clean_flag: false

                 description: >-
                     This test case verifies the retrieval of OpenStack clients:
                     Keystone, Glance, Neutron and Nova and may perform some
                 description: >-
                     This test case verifies the retrieval of OpenStack clients:
                     Keystone, Glance, Neutron and Nova and may perform some


@@ -80,13 +71,14 @@ tiers:
                 name: vping_ssh
                 criteria: 'status == "PASS"'
                 blocking: true
                 name: vping_ssh
                 criteria: 'status == "PASS"'
                 blocking: true

+                clean_flag: true

                 description: >-
                     This test case verifies: 1) SSH to an instance using floating
                     IPs over the public network. 2) Connectivity between 2 instances
                     over a private network.
                 dependencies:
                     installer: ''
                 description: >-
                     This test case verifies: 1) SSH to an instance using floating
                     IPs over the public network. 2) Connectivity between 2 instances
                     over a private network.
                 dependencies:
                     installer: ''

-                    scenario: '^((?!bgpvpn|odl_l3).)*$'
+                    scenario: '^((?!odl_l3).)*$'

                 run:
                     module: 'functest.opnfv_tests.openstack.vping.vping_ssh'
                     class: 'VPingSSH'
                 run:
                     module: 'functest.opnfv_tests.openstack.vping.vping_ssh'
                     class: 'VPingSSH'


@@ -95,6 +87,7 @@ tiers:
                 name: vping_userdata
                 criteria: 'status == "PASS"'
                 blocking: true
                 name: vping_userdata
                 criteria: 'status == "PASS"'
                 blocking: true

+                clean_flag: true

                 description: >-
                     This test case verifies:  1) Boot a VM with given userdata.
                     2) Connectivity between 2 instances over a private network.
                 description: >-
                     This test case verifies:  1) Boot a VM with given userdata.
                     2) Connectivity between 2 instances over a private network.


@@ -109,6 +102,7 @@ tiers:
                 name: tempest_smoke_serial
                 criteria: 'success_rate == 100%'
                 blocking: false
                 name: tempest_smoke_serial
                 criteria: 'success_rate == 100%'
                 blocking: false

+                clean_flag: false

                 description: >-
                     This test case runs the smoke subset of the OpenStack
                     Tempest suite. The list of test cases is generated by
                 description: >-
                     This test case runs the smoke subset of the OpenStack
                     Tempest suite. The list of test cases is generated by


@@ -125,12 +119,13 @@ tiers:
                 name: rally_sanity
                 criteria: 'success_rate == 100%'
                 blocking: false
                 name: rally_sanity
                 criteria: 'success_rate == 100%'
                 blocking: false

+                clean_flag: false

                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Rally suite in smoke mode.
                 dependencies:
                     installer: ''
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Rally suite in smoke mode.
                 dependencies:
                     installer: ''

-                    scenario: '^((?!bgpvpn).)*$'
+                    scenario: ''

                 run:
                     module: 'functest.opnfv_tests.openstack.rally.rally'
                     class: 'RallySanity'
                 run:
                     module: 'functest.opnfv_tests.openstack.rally.rally'
                     class: 'RallySanity'


@@ -139,6 +134,7 @@ tiers:
                 name: odl
                 criteria: 'success_rate == 100%'
                 blocking: true
                 name: odl
                 criteria: 'success_rate == 100%'
                 blocking: true

+                clean_flag: false

                 description: >-
                     Test Suite for the OpenDaylight SDN Controller. It
                     integrates some test suites from upstream using
                 description: >-
                     Test Suite for the OpenDaylight SDN Controller. It
                     integrates some test suites from upstream using


@@ -158,6 +154,7 @@ tiers:
                 name: odl_netvirt
                 criteria: 'success_rate == 100%'
                 blocking: true
                 name: odl_netvirt
                 criteria: 'success_rate == 100%'
                 blocking: true

+                clean_flag: false

                 description: >-
                     Test Suite for the OpenDaylight SDN Controller when
                     the NetVirt features are installed. It integrates
                 description: >-
                     Test Suite for the OpenDaylight SDN Controller when
                     the NetVirt features are installed. It integrates


@@ -179,6 +176,7 @@ tiers:
                 name: onos
                 criteria: 'status == "PASS"'
                 blocking: true
                 name: onos
                 criteria: 'status == "PASS"'
                 blocking: true

+                clean_flag: true

                 description: >-
                     Test Suite for the ONOS SDN Controller. It integrates
                     some test suites from upstream using TestON as the test
                 description: >-
                     Test Suite for the ONOS SDN Controller. It integrates
                     some test suites from upstream using TestON as the test


@@ -186,12 +184,15 @@ tiers:
                 dependencies:
                     installer: ''
                     scenario: 'onos'
                 dependencies:
                     installer: ''
                     scenario: 'onos'

-
+                run:
+                    module: 'functest.opnfv_tests.sdn.onos.onos'
+                    class: 'Onos'

 
             -
                 name: snaps_smoke
                 criteria: 'status == "PASS"'
                 blocking: false
 
             -
                 name: snaps_smoke
                 criteria: 'status == "PASS"'
                 blocking: false

+                clean_flag: false

                 description: >-
                     This test case contains tests that setup and destroy
                     environments with VMs with and without Floating IPs
                 description: >-
                     This test case contains tests that setup and destroy
                     environments with VMs with and without Floating IPs


@@ -220,6 +221,7 @@ tiers:
                 name: promise
                 criteria: 'success_rate == 100%'
                 blocking: false
                 name: promise
                 criteria: 'success_rate == 100%'
                 blocking: false

+                clean_flag: true

                 description: >-
                     Test suite from Promise project.
                 dependencies:
                 description: >-
                     Test suite from Promise project.
                 dependencies:


@@ -233,6 +235,7 @@ tiers:
                 name: doctor
                 criteria: 'status == "PASS"'
                 blocking: false
                 name: doctor
                 criteria: 'status == "PASS"'
                 blocking: false

+                clean_flag: true

                 description: >-
                     Test suite from Doctor project.
                 dependencies:
                 description: >-
                     Test suite from Doctor project.
                 dependencies:


@@ -246,6 +249,7 @@ tiers:
                 name: bgpvpn
                 criteria: 'status == "PASS"'
                 blocking: false
                 name: bgpvpn
                 criteria: 'status == "PASS"'
                 blocking: false

+                clean_flag: true

                 description: >-
                     Test suite from SDNVPN project.
                 dependencies:
                 description: >-
                     Test suite from SDNVPN project.
                 dependencies:


@@ -259,6 +263,7 @@ tiers:
                 name: security_scan
                 criteria: 'status == "PASS"'
                 blocking: false
                 name: security_scan
                 criteria: 'status == "PASS"'
                 blocking: false

+                clean_flag: true

                 description: >-
                     Simple Security Scan
                 dependencies:
                 description: >-
                     Simple Security Scan
                 dependencies:


@@ -271,6 +276,7 @@ tiers:
 #                name: copper
 #                criteria: 'status == "PASS"'
 #                blocking: false
 #                name: copper
 #                criteria: 'status == "PASS"'
 #                blocking: false

+#                clean_flag: true

 #                description: >-
 #                    Test suite for policy management based on OpenStack Congress
 #                dependencies:
 #                description: >-
 #                    Test suite for policy management based on OpenStack Congress
 #                dependencies:


@@ -283,6 +289,7 @@ tiers:
                 name: multisite
                 criteria: 'success_rate == 100%'
                 blocking: false
                 name: multisite
                 criteria: 'success_rate == 100%'
                 blocking: false

+                clean_flag: false

                 description: >-
                     Test suite from kingbird
                 dependencies:
                 description: >-
                     Test suite from kingbird
                 dependencies:


@@ -295,6 +302,7 @@ tiers:
                 name: odl-sfc
                 criteria: 'status == "PASS"'
                 blocking: false
                 name: odl-sfc
                 criteria: 'status == "PASS"'
                 blocking: false

+                clean_flag: true

                 description: >-
                     Test suite for odl-sfc to test two chains and two SFs
                 dependencies:
                 description: >-
                     Test suite for odl-sfc to test two chains and two SFs
                 dependencies:


@@ -307,15 +315,20 @@ tiers:
                 name: onos_sfc
                 criteria: 'status == "PASS"'
                 blocking: true
                 name: onos_sfc
                 criteria: 'status == "PASS"'
                 blocking: true

+                clean_flag: true

                 description: >-
                     Test Suite for onos-sfc to test sfc function.
                 dependencies:
                     installer: ''
                     scenario: 'onos-sfc'
                 description: >-
                     Test Suite for onos-sfc to test sfc function.
                 dependencies:
                     installer: ''
                     scenario: 'onos-sfc'

+                run:
+                    module: 'functest.opnfv_tests.sdn.onos.onos'
+                    class: 'OnosSfc'

             -
                 name: parser
                 criteria: 'ret == 0'
                 blocking: false
             -
                 name: parser
                 criteria: 'ret == 0'
                 blocking: false

+                clean_flag: true

                 description: >-
                     Test suite from Parser project.
                 dependencies:
                 description: >-
                     Test suite from Parser project.
                 dependencies:


@@ -324,10 +337,24 @@ tiers:
                 run:
                     module: 'functest.opnfv_tests.vnf.rnc.parser'
                     class: 'Parser'
                 run:
                     module: 'functest.opnfv_tests.vnf.rnc.parser'
                     class: 'Parser'

+            -
+                name: domino
+                criteria: 'status == "PASS"'
+                blocking: false
+                clean_flag: true
+                description: >-
+                    Test suite from Domino project.
+                dependencies:
+                    installer: ''
+                    scenario: ''
+                run:
+                    module: 'functest.opnfv_tests.features.domino'
+                    class: 'Domino'

             -
                 name: orchestra
                 criteria: 'ret == 0'
                 blocking: false
             -
                 name: orchestra
                 criteria: 'ret == 0'
                 blocking: false

+                clean_flag: true

                 description: >-
                     Test OpenBaton (Orchestra) stack
                 dependencies:
                 description: >-
                     Test OpenBaton (Orchestra) stack
                 dependencies:


@@ -340,6 +367,7 @@ tiers:
                 name: netready
                 criteria: 'status == "PASS"'
                 blocking: false
                 name: netready
                 criteria: 'status == "PASS"'
                 blocking: false

+                clean_flag: true

                 description: >-
                     Test suite from Netready project.
                 dependencies:
                 description: >-
                     Test suite from Netready project.
                 dependencies:


@@ -348,41 +376,89 @@ tiers:
                 run:
                      module: 'functest.opnfv_tests.features.netready'
                      class: 'GluonVping'
                 run:
                      module: 'functest.opnfv_tests.features.netready'
                      class: 'GluonVping'

+            -
+                name: barometer
+                criteria: 'status == "PASS"'
+                blocking: false
+                clean_flag: true
+                description: >-
+                    Test suite for the Barometer project. Separate tests verify the
+                    proper configuration and functionality of the following
+                    collectd plugins Ceilometer, Hugepages, Memory RAS (mcelog),
+                    and OVS Events
+                dependencies:
+                    installer: 'fuel'
+                    scenario: 'kvm_ovs_dpdk_bar'
+                run:
+                     module: 'functest.opnfv_tests.features.barometer'
+                     class: 'BarometerCollectd'

     -
         name: components
         order: 3
     -
         name: components
         order: 3

-        ci_loop: 'weekly'
+        ci_loop: 'daily'

         description : >-
             Extensive testing of OpenStack API.
         testcases:
         description : >-
             Extensive testing of OpenStack API.
         testcases:

+#            -
+#                name: tempest_full_parallel
+#                criteria: 'success_rate >= 80%'
+#                blocking: false
+#                clean_flag: false
+#                description: >-
+#                    The list of test cases is generated by
+#                    Tempest automatically and depends on the parameters of
+#                    the OpenStack deplopyment.
+#                dependencies:
+#                    installer: '^((?!netvirt).)*$'
+#                    scenario: ''
+#                run:
+#                    module: 'functest.opnfv_tests.openstack.tempest.tempest'
+#                    class: 'TempestFullParallel'

             -
             -

-                name: tempest_full_parallel
-                criteria: 'success_rate >= 80%'
+                name: tempest_defcore
+                criteria: 'success_rate == 100%'

                 blocking: false
                 blocking: false

+                clean_flag: false

                 description: >-
                 description: >-

-                    The list of test cases is generated by
-                    Tempest automatically and depends on the parameters of
-                    the OpenStack deplopyment.
+                    This is the set of Tempest test cases created by OpenStack
+                    Interop Working Group for certification purposes.

                 dependencies:
                 dependencies:

-                    installer: '^((?!netvirt).)*$'
-                    scenario: ''
+                    installer: ''
+                    scenario: 'nosdn-nofeature-ha'

                 run:
                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
                 run:
                     module: 'functest.opnfv_tests.openstack.tempest.tempest'

-                    class: 'TempestFullParallel'
-
+                    class: 'TempestDefcore'

             -
             -

-                name: rally_full
-                criteria: 'success_rate >= 90%'
+                name: tempest_custom
+                criteria: 'success_rate == 100%'

                 blocking: false
                 blocking: false

+                clean_flag: false

                 description: >-
                 description: >-

-                    This test case runs the full suite of scenarios of the OpenStack
-                    Rally suite using several threads and iterations.
+                    The test case allows running a customized list of tempest
+                    test cases defined in a file under
+                    <dir_functest_repo>/functest/opnfv_tests/openstack/
+                      /tempest/custom_tests/test_list.txt
+                    The file is empty and can be customized with the desired tests.

                 dependencies:
                 dependencies:

-                    installer: '^((?!netvirt).)*$'
-                    scenario: ''
+                    installer: 'unknown'
+                    scenario: 'unknown'

                 run:
                 run:

-                    module: 'functest.opnfv_tests.openstack.rally.rally'
-                    class: 'RallyFull'
+                    module: 'functest.opnfv_tests.openstack.tempest.tempest'
+                    class: 'TempestCustom'
+#            -
+#                name: rally_full
+#                criteria: 'success_rate >= 90%'
+#                blocking: false
+#                clean_flag: false
+#                description: >-
+#                    This test case runs the full suite of scenarios of the OpenStack
+#                    Rally suite using several threads and iterations.
+#                dependencies:
+#                    installer: '^((?!netvirt).)*$'
+#                    scenario: ''
+#                run:
+#                    module: 'functest.opnfv_tests.openstack.rally.rally'
+#                    class: 'RallyFull'

 
     -
         name: vnf
 
     -
         name: vnf


@@ -395,6 +471,7 @@ tiers:
                 name: cloudify_ims
                 criteria: 'status == "PASS"'
                 blocking: false
                 name: cloudify_ims
                 criteria: 'status == "PASS"'
                 blocking: false

+                clean_flag: true

                 description: >-
                     This test case deploys an OpenSource vIMS solution from Clearwater
                     using the Cloudify orchestrator. It also runs some signaling traffic.
                 description: >-
                     This test case deploys an OpenSource vIMS solution from Clearwater
                     using the Cloudify orchestrator. It also runs some signaling traffic.


@@ -408,6 +485,7 @@ tiers:
                 name: aaa
                 criteria: 'ret == 0'
                 blocking: false
                 name: aaa
                 criteria: 'ret == 0'
                 blocking: false

+                clean_flag: true

                 description: >-
                     Test suite from Parser project.
                 dependencies:
                 description: >-
                     Test suite from Parser project.
                 dependencies:


@@ -421,6 +499,7 @@ tiers:
                 name: juju_epc
                 criteria: 'ret == 0'
                 blocking: false
                 name: juju_epc
                 criteria: 'ret == 0'
                 blocking: false

+                clean_flag: true

                 description: >-
                     Test suite from OAI project, vEPC deployed with Juju.
                 dependencies:
                 description: >-
                     Test suite from OAI project, vEPC deployed with Juju.
                 dependencies:


@@ -434,11 +513,12 @@ tiers:
                 name: orchestra_ims
                 criteria: 'ret == 0'
                 blocking: false
                 name: orchestra_ims
                 criteria: 'ret == 0'
                 blocking: false

+                clean_flag: true

                 description: >-
                     VNF deployment with OpenBaton (Orchestra)
                 dependencies:
                 description: >-
                     VNF deployment with OpenBaton (Orchestra)
                 dependencies:

-                    installer: 'unknown'
-                    scenario: 'unknown'
+                    installer: ''
+                    scenario: ''

                 run:
                     module: 'functest.opnfv_tests.vnf.ims.orchestra_ims'
                     class: 'ImsVnf'
                 run:
                     module: 'functest.opnfv_tests.vnf.ims.orchestra_ims'
                     class: 'ImsVnf'


@@ -447,6 +527,7 @@ tiers:
                 name: opera_ims
                 criteria: 'ret == 0'
                 blocking: false
                 name: opera_ims
                 criteria: 'ret == 0'
                 blocking: false

+                clean_flag: true

                 description: >-
                     Evolution of vIMS
                 dependencies:
                 description: >-
                     Evolution of vIMS
                 dependencies:


@@ -455,3 +536,17 @@ tiers:
                 run:
                     module: 'functest.opnfv_tests.vnf.ims.opera_ims'
                     class: 'ImsVnf'
                 run:
                     module: 'functest.opnfv_tests.vnf.ims.opera_ims'
                     class: 'ImsVnf'

+
+            -
+                name: vyos_vrouter
+                criteria: 'status == "PASS"'
+                blocking: false
+                clean_flag: true
+                description: >-
+                    This test case is vRouter testing.
+                dependencies:
+                    installer: 'fuel'
+                    scenario: 'nosdn-nofeature'
+                run:
+                    module: 'functest.opnfv_tests.vnf.router.vyos_vrouter'
+                    class: 'VrouterVnf'