From f11a73a65b3da7e3c1dd9ee05fc036fed2b24123 Mon Sep 17 00:00:00 2001 From: =?utf8?q?C=C3=A9dric=20Ollivier?= Date: Thu, 3 Jul 2025 15:49:36 +0200 Subject: [PATCH] Fix rally unit tests MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Change-Id: I19a4f148a8c5e0a3fe4782f322e3e50deb22bf1c Signed-off-by: Cédric Ollivier --- docs/testing/developer/devguide/index.rst | 13 +-- docs/testing/user/configguide/configguide.rst | 4 +- docs/testing/user/userguide/index.rst | 1 - docs/testing/user/userguide/reporting.rst | 98 ----------------------- docs/testing/user/userguide/test_overview.rst | 17 ---- docs/testing/user/userguide/troubleshooting.rst | 17 ---- functest/core/tenantnetwork.py | 2 +- functest/tests/unit/openstack/rally/test_rally.py | 2 +- functest/utils/env.py | 5 -- tox.ini | 2 +- upper-constraints.txt | 1 + 11 files changed, 6 insertions(+), 156 deletions(-) delete mode 100644 docs/testing/user/userguide/reporting.rst diff --git a/docs/testing/developer/devguide/index.rst b/docs/testing/developer/devguide/index.rst index da5485ef0..78b3eb1e4 100644 --- a/docs/testing/developer/devguide/index.rst +++ b/docs/testing/developer/devguide/index.rst @@ -82,7 +82,6 @@ The internal test cases in Euphrates are: * connection_check * vping_ssh * vping_userdata - * odl * rally_full * rally_sanity * tempest_smoke @@ -93,14 +92,13 @@ By internal, we mean that this particular test cases have been developed and/or integrated by functest contributors and the associated code is hosted in the Functest repository. An internal case can be fully developed or a simple integration of -upstream suites (e.g. Tempest/Rally developed in OpenStack, or odl suites are +upstream suites (e.g. Tempest/Rally developed in OpenStack are just integrated in Functest). The structure of this repository is detailed in `[1]`_. The main internal test cases are in the opnfv_tests subfolder of the repository, the internal test cases can be grouped by domain: - * sdn: odl, odl_fds * openstack: connection_check, vping_ssh, vping_userdata, tempest_*, rally_* * vnf: cloudify_ims @@ -201,13 +199,6 @@ A web page is automatically generated every day to display the status based on jinja2 templates `[3]`_. -Dashboard -========= - -Additional dashboarding is managed at the testing group level, see `[5]`_ for -details. - - ========== References ========== @@ -220,6 +211,4 @@ _`[3]`: https://github.com/opnfv/releng-testresults/tree/master/reporting _`[4]`: https://wiki.opnfv.org/display/functest/2017+Beijing?preview=%2F11699623%2F11700523%2FTestAPI+-+test+results+collection+service.pptx -_`[5]`: https://lfanalytics.io/projects/lfn%2Fopnfv/dashboard - IRC support chan: #opnfv-functest diff --git a/docs/testing/user/configguide/configguide.rst b/docs/testing/user/configguide/configguide.rst index d0c1b6f45..98d487cfb 100644 --- a/docs/testing/user/configguide/configguide.rst +++ b/docs/testing/user/configguide/configguide.rst @@ -31,7 +31,6 @@ cat env:: EXTERNAL_NETWORK=XXX # if not first network with router:external=True DASHBOARD_URL=XXX # else tempest_horizon will be skipped NEW_USER_ROLE=XXX # if not member - SDN_CONTROLLER_IP=XXX # if odl scenario VOLUME_DEVICE_NAME=XXX # if not vdb FLAVOR_EXTRA_SPECS=hw:mem_page_size:large # if fdio scenarios @@ -91,7 +90,6 @@ Results shall be displayed as follows:: | cinder_test | functest | healthcheck | 01:05 | PASS | | tempest_smoke | functest | healthcheck | 05:39 | PASS | | tempest_horizon | functest | healthcheck | 01:05 | PASS | - | odl | functest | healthcheck | 00:00 | SKIP | +--------------------------+------------------+---------------------+------------------+----------------+ NOTE: the duration is a reference and it might vary depending on your SUT. @@ -353,7 +351,7 @@ the deployment. The format for the DEPLOY_SCENARIO env variable can be described as follows: * vim: (os|k8s) = OpenStack or Kubernetes - * controller is one of ( nosdn | odl ) + * controller: nosdn * nfv_feature is one or more of ( ovs | kvm | sfc | bgpvpn | nofeature ) * ha_mode (high availability) is one of ( ha | noha ) diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst index 1e73cd622..1da23a634 100644 --- a/docs/testing/user/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst @@ -13,7 +13,6 @@ Functest User Guide test_overview.rst test_details.rst test_results.rst - reporting.rst troubleshooting.rst diff --git a/docs/testing/user/userguide/reporting.rst b/docs/testing/user/userguide/reporting.rst deleted file mode 100644 index 8fad55d33..000000000 --- a/docs/testing/user/userguide/reporting.rst +++ /dev/null @@ -1,98 +0,0 @@ -.. http://creativecommons.org/licenses/by/4.0 - -Test reporting -============== - -An automatic reporting page has been created in order to provide a consistent -view of the Functest tests on the different scenarios. - -In this page, each scenario is evaluated according to test criteria. - -The results are collected from the centralized database every day and, per -scenario. A score is calculated based on the results from the last 10 days. -This score is the addition of single test scores. Each test case has a success -criteria reflected in the criteria field from the results. - -As an illustration, let's consider the scenario -os-odl_l2-nofeature-ha scenario, the scenario scoring is the addition of the -scores of all the runnable tests from the categories (tiers, healthcheck, smoke -and features) corresponding to this scenario. - - +---------------------+---------+---------+---------+---------+ - | Test | Apex | Compass | Fuel | Joid | - +=====================+=========+=========+=========+=========+ - | vPing_ssh | X | X | X | X | - +---------------------+---------+---------+---------+---------+ - | vPing_userdata | X | X | X | X | - +---------------------+---------+---------+---------+---------+ - | tempest_smoke | X | X | X | X | - +---------------------+---------+---------+---------+---------+ - | rally_sanity | X | X | X | X | - +---------------------+---------+---------+---------+---------+ - | odl | X | X | X | X | - +---------------------+---------+---------+---------+---------+ - | promise | | | X | X | - +---------------------+---------+---------+---------+---------+ - | doctor | X | | X | | - +---------------------+---------+---------+---------+---------+ - | security_scan | X | | | | - +---------------------+---------+---------+---------+---------+ - | parser | | | X | | - +---------------------+---------+---------+---------+---------+ - | copper | X | | | X | - +---------------------+---------+---------+---------+---------+ - - src: os-odl_l2-nofeature-ha Colorado (see release note for the last matrix - version) - -All the testcases (X) listed in the table are runnable on os-odl_l2-nofeature -scenarios. -Please note that other test cases (e.g. sfc_odl, bgpvpn) need ODL configuration -add-ons and, as a consequence, specific scenario. -There are not considered as runnable on the generic odl_l2 scenario. - - -If no result is available or if all the results are failed, the test case get 0 -point. -If it was successful at least once but not anymore during the 4 runs, the case -get 1 point (it worked once). -If at least 3 of the last 4 runs were successful, the case get 2 points. -If the last 4 runs of the test are successful, the test get 3 points. - -In the example above, the target score for fuel/os-odl_l2-nofeature-ha is -3 x 8 = 24 points and for compass it is 3 x 5 = 15 points . - -The scenario is validated per installer when we got 3 points for all individual -test cases (e.g 24/24 for fuel, 15/15 for compass). - -Please note that complex or long duration tests are not considered yet for the -scoring. In fact the success criteria are not always easy to define and may -require specific hardware configuration. - -Please also note that all the test cases have the same "weight" for the score -calculation whatever the complexity of the test case. Concretely a vping has -the same weight than the 200 tempest tests. -Moreover some installers support more features than others. The more cases your -scenario is dealing with, the most difficult to rich a good scoring. - -Therefore the scoring provides 3 types of indicators: - - * the richness of the scenario: if the target scoring is high, it means that - the scenario includes lots of features - * the maturity: if the percentage (scoring/target scoring * 100) is high, it - means that all the tests are PASS - * the stability: as the number of iteration is included in the calculation, - the percentage can be high only if the scenario is run regularly (at least - more than 4 iterations over the last 10 days in CI) - -In any case, the scoring is used to give feedback to the other projects and -does not represent an absolute value of the scenario. - -See `reporting page`_ for details. For the status, click on the version, -Functest then the Status menu. - -.. _`reporting page`: http://testresults.opnfv.org/reporting/ - -.. figure:: ../../../images/functest-reporting-status.png - :align: center - :alt: Functest reporting portal Fuel status page diff --git a/docs/testing/user/userguide/test_overview.rst b/docs/testing/user/userguide/test_overview.rst index bc3e79dcb..737920f5d 100644 --- a/docs/testing/user/userguide/test_overview.rst +++ b/docs/testing/user/userguide/test_overview.rst @@ -80,13 +80,6 @@ validate the scenario for the release. | | | | benchmarking OpenStack modules | | | | | See the Rally documents `[3]`_ | +-------------+---------------+------------+----------------------------------+ -| Controllers | smoke | odl | Opendaylight Test suite | -| | | | Limited test suite to check the | -| | | | basic neutron (Layer 2) | -| | | | operations mainly based on | -| | | | upstream testcases. See below | -| | | | for details | -+-------------+---------------+------------+----------------------------------+ | VNF | vnf | cloudify | Example of a real VNF deployment | | | | \_ims | to show the NFV capabilities of | | | | | the platform. The IP Multimedia | @@ -97,16 +90,6 @@ validate the scenario for the release. | | +------------+----------------------------------+ | | | vyos | vRouter testing | | | | \_vrouter | | -| | +------------+----------------------------------+ -| | | juju_epc | Validates deployment of a complex| -| | | | mobility VNF on OPNFV Platform. | -| | | | Uses Juju for deploying the OAI | -| | | | EPC and ABot for defining test | -| | | | scenarios using high-level DSL. | -| | | | VNF tests reference 3GPP | -| | | | Technical Specs and are executed | -| | | | through protocol drivers provided| -| | | | by ABot. | +-------------+---------------+------------+----------------------------------+ | Kubernetes | healthcheck | k8s_smoke | Test a running Kubernetes | | | | | cluster and ensure it satisfies | diff --git a/docs/testing/user/userguide/troubleshooting.rst b/docs/testing/user/userguide/troubleshooting.rst index d857ed4c4..c2c971145 100644 --- a/docs/testing/user/userguide/troubleshooting.rst +++ b/docs/testing/user/userguide/troubleshooting.rst @@ -311,23 +311,6 @@ To check any possible problems with Rally, the logs are stored under .. _`[3]`: https://rally.readthedocs.io/en/latest/index.html -Controllers ------------ - -Opendaylight -^^^^^^^^^^^^ - -If the Basic Restconf test suite fails, check that the ODL controller is -reachable and its Restconf module has been installed. - -If the Neutron Reachability test fails, verify that the modules -implementing Neutron requirements have been properly installed. - -If any of the other test cases fails, check that Neutron and ODL have -been correctly configured to work together. Check Neutron configuration -files, accounts, IP addresses etc.). - - Features -------- diff --git a/functest/core/tenantnetwork.py b/functest/core/tenantnetwork.py index 3670dbe8a..ecaf59f93 100644 --- a/functest/core/tenantnetwork.py +++ b/functest/core/tenantnetwork.py @@ -25,7 +25,7 @@ import uuid import os_client_config import shade -from tempest.lib.common.utils import data_utils +from tempest.lib.common.utils import data_utils # pylint: disable=import-error from xtesting.core import testcase from functest.utils import config diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py index f3c2e7cf6..2150d62fa 100644 --- a/functest/tests/unit/openstack/rally/test_rally.py +++ b/functest/tests/unit/openstack/rally/test_rally.py @@ -85,7 +85,7 @@ class OSRallyTesting(unittest.TestCase): mock.call().decode("utf-8"), mock.call(['rally', 'deployment', 'check']), mock.call().decode("utf-8")] - mock_exec.assert_has_calls(calls) + mock_exec.assert_has_calls(calls, any_order=True) @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists') @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs') diff --git a/functest/utils/env.py b/functest/utils/env.py index 2e312726c..4907e52ac 100644 --- a/functest/utils/env.py +++ b/functest/utils/env.py @@ -20,11 +20,6 @@ INPUTS = { 'DEBUG': env.INPUTS['DEBUG'], 'DEPLOY_SCENARIO': env.INPUTS['DEPLOY_SCENARIO'], 'INSTALLER_TYPE': env.INPUTS['INSTALLER_TYPE'], - 'SDN_CONTROLLER_IP': None, - 'SDN_CONTROLLER_USER': 'admin', - 'SDN_CONTROLLER_PASSWORD': 'admin', - 'SDN_CONTROLLER_WEBPORT': '8080', - 'SDN_CONTROLLER_RESTCONFPORT': '8181', 'BUILD_TAG': env.INPUTS['BUILD_TAG'], 'NODE_NAME': env.INPUTS['NODE_NAME'], 'POD_ARCH': None, diff --git a/tox.ini b/tox.ini index 02a89ee46..78d2e254d 100644 --- a/tox.ini +++ b/tox.ini @@ -88,7 +88,7 @@ commands = nosetests --with-coverage --cover-tests \ [testenv:perm] basepython = python3.9 -whitelist_externals = bash +allowlist_externals = bash path=. -not -path './.tox/*' -not -path './.git/*' -not -path './docs/com/pres/reveal.js/*' -not -path './elements/functest/install.d/*' commands = bash -c "\ diff --git a/upper-constraints.txt b/upper-constraints.txt index 76dabeab3..9e05a79c1 100644 --- a/upper-constraints.txt +++ b/upper-constraints.txt @@ -13,6 +13,7 @@ ruamel.yaml.jinja2==0.2.2 git+https://opendev.org/openstack/rally-openstack.git@2.3.0#egg=rally-openstack git+https://github.com/xrally/xrally-kubernetes.git@2ffa85af2bff3438b6b23034b6ec6ee1de481090#egg=xrally-kubernetes pylint===2.11.1 +astroid==2.8.0 flake8===4.0.1 nose===1.3.7 ruamel.yaml===0.17.17 -- 2.16.6