From 8ecfd4c886507fe602398a8623e6044d40ea8090 Mon Sep 17 00:00:00 2001 From: Gwenael Lambrouin Date: Fri, 6 Jan 2023 17:53:23 +0100 Subject: [PATCH] Cleanup requirements & tox config, update pylint - remove useless Python packages - make the purpose of each requirements.txt file explicit - don't overlap between requirements.txt files - fix requirements so that the "new" pip dependency resolver (enabled by default since pip 20.3) work. Consequently, it is not necessary to pin pip version to 20.2.4 and we can drop tox-pip-version - remove unused tox environments - skip the installation of nfvbench in the tox environments where this is not necessary - update pylint 2.10.2 => 2.15.9 (that was necessary to fix some requirement conflicts): - disable some of new checkers: consider-using-f-string,consider-using-generator,unnecessary-lambda-assignment,implicit-str-concat,global-variable-not-assigned - remove many obsolete checkers from the disable list - apply a simple refactor suggestion in nfvbench code - upgrade to Python 3.8 some remaining references to Python 3.6 Change-Id: I55c0fb144b9199681962f396582590aba89ed02c Signed-off-by: Gwenael Lambrouin --- .gitlab-ci.yml | 8 ++++---- docs/developer/testing-nfvbench.rst | 16 +++++++++------- docs/user/xtesting.rst | 4 ++-- nfvbench/traffic_client.py | 2 +- pylint.rc | 38 +------------------------------------ requirements-dev.txt | 17 ++++++++++------- requirements.txt | 14 +++++++------- setup.cfg | 4 ++-- test-requirements.txt | 30 ++++++++++++++--------------- tox.ini | 13 ++++--------- 10 files changed, 55 insertions(+), 91 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ab02fe..32e3299 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,9 +13,9 @@ variables: .tox-defaults: &tox-defaults stage: test - image: python:3.6 + image: python:3.8 before_script: - - pip install tox tox-pip-version + - pip install tox==3.21.4 cache: paths: - .cache/pip @@ -29,10 +29,10 @@ pep8: script: tox -e pep8 -tox-py36: +tox-py38: <<: *tox-defaults script: - tox -e py36 + tox -e py38 verify-image: stage: build diff --git a/docs/developer/testing-nfvbench.rst b/docs/developer/testing-nfvbench.rst index 799f8a6..fd6c6f7 100644 --- a/docs/developer/testing-nfvbench.rst +++ b/docs/developer/testing-nfvbench.rst @@ -20,10 +20,9 @@ format. What tox should do is specified in a ``tox.ini`` file located at the project root. -tox is used in continuous integration (jenkins-ci today, gitlab-ci in the near -future): all the actions performed by tox must succeed before a patchset can be -merged. As a developer, it is also useful to run tox locally to detect and fix -the issues before pushing the code for review. +tox is used in continuous integration: all the actions performed by tox must +succeed before a patchset can be merged. As a developer, it is also useful to +run tox locally to detect and fix the issues before pushing the code for review. .. _tox: https://tox.wiki/en/latest/ @@ -35,7 +34,7 @@ Using tox on a developer's machine Requirement: |python-version| ----------------------------- -.. |python-version| replace:: Python 3.6 +.. |python-version| replace:: Python 3.8 The current version of Python used by NFVbench is |python-version|. In particular, this means that |python-version| is used: @@ -58,7 +57,10 @@ tox installation Install tox with:: - $ pip install tox tox-pip-version + $ pip install tox==3.21.4 + +.. note:: tox 3.21.4 is the version that comes with Ubuntu 22.04 and that can be + found on gerrit.opnfv.org build servers. Running tox @@ -70,7 +72,7 @@ In nfvbench root directory, simply run tox with:: If all goes well, tox shows a green summary such as:: - py36: commands succeeded + py38: commands succeeded pep8: commands succeeded lint: commands succeeded docs: commands succeeded diff --git a/docs/user/xtesting.rst b/docs/user/xtesting.rst index 0adfdda..96d1d19 100644 --- a/docs/user/xtesting.rst +++ b/docs/user/xtesting.rst @@ -179,9 +179,9 @@ To replace existing testcases.yaml file, using Xtesting CI add the volume mappin - /usr/src/kernels:/usr/src/kernels -v /dev:/dev - /home/opnfv/nfvbench/config:/etc/nfvbench - /home/opnfv/nfvbench/results:/var/lib/xtesting/results - - /home/opnfv/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml + - /home/opnfv/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.8/dist-packages/xtesting/ci/testcases.yaml -* ``/home/opnfv/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml`` : volume mapping to pass testcases.yaml file between the host and the docker space. Host path required testcases.yaml file inside. +* ``/home/opnfv/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.8/dist-packages/xtesting/ci/testcases.yaml`` : volume mapping to pass testcases.yaml file between the host and the docker space. Host path required testcases.yaml file inside. To replace existing testcases.yaml file, using NFVbench container: diff --git a/nfvbench/traffic_client.py b/nfvbench/traffic_client.py index c349289..47af265 100755 --- a/nfvbench/traffic_client.py +++ b/nfvbench/traffic_client.py @@ -628,7 +628,7 @@ class GeneratorConfig(object): self.intf_speed = gen_config.intf_speed else: self.intf_speed = "auto" - if self.intf_speed == "auto" or self.intf_speed == "0": + if self.intf_speed in ("auto", "0"): # interface speed is discovered/provided by the traffic generator self.intf_speed = 0 else: diff --git a/pylint.rc b/pylint.rc index c54981f..f31b894 100644 --- a/pylint.rc +++ b/pylint.rc @@ -50,7 +50,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=unused-argument,global-statement,too-many-statements,too-many-arguments,too-many-branches,catching-non-exception,relative-import,too-many-locals,invalid-name,broad-except,print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,import-star-module-level,raw-checker-failed,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call,missing-docstring,redefined-builtin,no-name-in-module,no-self-use,no-member,arguments-differ,attribute-defined-outside-init,non-iterator-returned,eval-used,unexpected-keyword-arg,pointless-string-statement,no-value-for-parameter,useless-object-inheritance,import-outside-toplevel,wrong-import-order +disable=unused-argument,global-statement,too-many-statements,too-many-arguments,too-many-branches,catching-non-exception,too-many-locals,invalid-name,broad-except,raw-checker-failed,locally-disabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,missing-docstring,redefined-builtin,no-name-in-module,no-member,arguments-differ,attribute-defined-outside-init,non-iterator-returned,eval-used,unexpected-keyword-arg,pointless-string-statement,no-value-for-parameter,useless-object-inheritance,import-outside-toplevel,wrong-import-order,consider-using-f-string,consider-using-generator,unnecessary-lambda-assignment,implicit-str-concat,global-variable-not-assigned # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -92,36 +92,21 @@ max-nested-blocks=5 [BASIC] -# Naming hint for argument names -argument-name-hint=(([a-z][a-z0-9_]{2,50})|(_[a-z0-9_]*))$ - # Regular expression matching correct argument names argument-rgx=(([a-z][a-z0-9_]{2,50})|(_[a-z0-9_]*))$ -# Naming hint for attribute names -attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - # Regular expression matching correct attribute names attr-rgx=(([a-z][a-z0-9_]{2,50})|(_[a-z0-9_]*))$ # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - # Regular expression matching correct class attribute names class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,50}|(__.*__))$ -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - # Regular expression matching correct class names class-rgx=[A-Z_][a-zA-Z0-9]+$ -# Naming hint for constant names -const-name-hint=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$ - # Regular expression matching correct constant names const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$ @@ -129,9 +114,6 @@ const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$ # ones are exempt. docstring-min-length=-1 -# Naming hint for function names -function-name-hint=(([a-z][a-zA-Z0-9_]{2,50})|(_[a-z0-9_]*))$ - # Regular expression matching correct function names function-rgx=(([a-z][a-zA-Z0-9_]{2,50})|(_[a-z0-9_]*))$ @@ -141,21 +123,12 @@ good-names=i,j,k,ex,Run,_ # Include a hint for the correct naming format with invalid-name include-naming-hint=no -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - # Regular expression matching correct inline iteration names inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ -# Naming hint for method names -method-name-hint=(([a-z][a-z0-9_]{2,50})|(_[a-z0-9_]*))$ - # Regular expression matching correct method names method-rgx=(([a-z][a-z0-9_]{2,50})|(_[a-z0-9_]*))$ -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - # Regular expression matching correct module names module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ @@ -171,9 +144,6 @@ no-docstring-rgx=^_ # to this list to register other decorators that produce valid properties. property-classes=abc.abstractproperty -# Naming hint for variable names -variable-name-hint=(([a-z][a-z0-9_]{2,50})|(_[a-z0-9_]*))$ - # Regular expression matching correct variable names variable-rgx=(([a-z][a-z0-9_]{2,50})|(_[a-z0-9_]*))$ @@ -199,12 +169,6 @@ max-line-length=100 # Maximum number of lines in a module max-module-lines=1600 -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - # Allow the body of a class to be on the same line as the declaration if body # contains single statement. single-line-class-stmt=no diff --git a/requirements-dev.txt b/requirements-dev.txt index 2d028c0..5522a05 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,11 @@ +# Python requirements to setup a development environment to be able to do +# anything directly or with tox: run unit tests, run code quality checks, build +# the docs, ... + +-c https://opendev.org/openstack/requirements/raw/branch/stable/yoga/upper-constraints.txt -r requirements.txt -docutils==0.14.0 -flake8>=3.3.0 -pylint==2.10.2 -sphinx>=1.4.0 -sphinx_rtd_theme>=0.2.4 -tox>=2.3.0 -mock>=2.0.0 +-r test-requirements.txt +-r docs/requirements.txt + +# Install tox with the same version as on OPNFV build servers +tox==3.21.4 diff --git a/requirements.txt b/requirements.txt index 717ba07..488aa5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +# Python dependencies required to run nfvbench and behave_tests +# # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. @@ -6,23 +8,21 @@ pbr!=2.1.0 # Apache-2.0 attrdict>=2.0.0 bitmath>=1.3.1.1 -paramiko # LGPLv2.1+ -PrettyTable # BSD pytz # MIT -six # MIT python-glanceclient # Apache-2.0 python-neutronclient # Apache-2.0 python-novaclient # Apache-2.0 python-openstackclient # Apache-2.0 python-keystoneclient!=2.1.0 # Apache-2.0 PyYAML # MIT -pyzmq # LGPL+BSD -requests!=2.20.0,!=2.24.0 # Apache-2.0 tabulate # MIT Flask!=0.11 # BSD fluent-logger>=0.5.3 netaddr # BSD -xtesting>=0.92.0 hdrhistogram>=0.8.0 -behave>=1.2.6 + +# Extra requirements for behave_tests: +requests!=2.20.0,!=2.24.0 # Apache-2.0 retry>=0.9.2 +xtesting>=0.92.0 +behave>=1.2.6 diff --git a/setup.cfg b/setup.cfg index d819ecb..d570851 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,8 +29,8 @@ classifier = Operating System :: POSIX :: Linux Operating System :: MacOS Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.8 [files] packages = diff --git a/test-requirements.txt b/test-requirements.txt index da52c9d..4fa57c3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,21 +1,21 @@ +# Python dependencies required by tox to run unit tests and code quality checks +# +# (dependencies required to build and check the docs can be found in +# docs/requirements.txt) +# # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking +# Requirements for nfvbench unit tests: +pytest # MIT +scapy>=2.3.1 +# Extra requirements for behave_tests unit tests: +pytest-subtests -coverage!=4.4 # Apache-2.0 -discover +# Requirements for pep8 test: +hacking==5.0.0 flake8>=3.3.0 -python-subunit # Apache-2.0/BSD -scapy>=2.3.1 -sphinx!=1.6.6,!=1.6.7,!=2.1.0,!=3.0.0,!=3.4.2 # BSD -sphinx-rtd-theme # MIT -oslosphinx # Apache-2.0 -oslotest<4;python_version<'3.6' # Apache-2.0 -oslotest;python_version>='3.6' # Apache-2.0 -testrepository # Apache-2.0/BSD -testscenarios # Apache-2.0/BSD -testtools # MIT -pytest # MIT -pylint==2.10.2 + +# Requirements for linter test: +pylint==2.15.9 diff --git a/tox.ini b/tox.ini index a5bd504..9b4d218 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,6 @@ envlist = py38,pep8,lint,docs,docs-linkcheck skipsdist = True [testenv] -pip_version = pip==20.2.4 usedevelop = True install_command = pip install -U {opts} {packages} setenv = @@ -17,20 +16,14 @@ commands = py.test -q --basetemp={envtmpdir} {posargs} test/ [testenv:pep8] basepython = python3.8 +skip_install = True commands = flake8 {toxinidir} [testenv:lint] basepython = python3.8 +skip_install = True commands = pylint --rcfile pylint.rc nfvbench test -[testenv:venv] -basepython = python3.8 -commands = {posargs} - -[testenv:cover] -basepython = python3.8 -commands = python setup.py testr --coverage --testr-args='{posargs}' - [flake8] # H803 skipped on purpose per list discussion. # E123, E125 skipped as they are invalid PEP-8. @@ -75,6 +68,7 @@ basepython = python3.8 deps = -chttps://opendev.org/openstack/requirements/raw/branch/stable/yoga/upper-constraints.txt -rdocs/requirements.txt +skip_install = True commands = sphinx-build -b html -n -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/html echo "Generated docs available in {toxinidir}/docs/_build/html" @@ -85,5 +79,6 @@ basepython = python3.8 deps = -chttps://opendev.org/openstack/requirements/raw/branch/stable/yoga/upper-constraints.txt -rdocs/requirements.txt +skip_install = True passenv = http_proxy https_proxy no_proxy commands = sphinx-build -b linkcheck -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/linkcheck -- 2.16.6