From 9dc587f166af106f7e30dbe00ddf152e9be6f1ea Mon Sep 17 00:00:00 2001 From: Rodolfo Alonso Hernandez Date: Thu, 25 Jan 2018 11:12:57 +0000 Subject: [PATCH] Module to manage pip packages This new module provides methods to manage Python PIP packages from a URL, from a local directory or from a build PIP package. The implemented commands are: - Install package. - Remove package. - List all installed packages in the system. JIRA: YARDSTICK-910 Change-Id: I8f7d1b77c0c384b801cc6f5e67d8b45ce7c6bfdf Signed-off-by: Rodolfo Alonso Hernandez --- requirements.txt | 2 + test-requirements.txt | 1 + tox.ini | 2 + yardstick/common/packages.py | 87 +++++++++++++++++++ yardstick/common/privsep.py | 23 +++++ yardstick/common/utils.py | 8 +- yardstick/tests/functional/base.py | 46 ++++++++++ .../common/fake_directory_package/README.md | 2 + .../common/fake_directory_package/setup.py | 29 +++++++ .../yardstick_new_plugin_2/__init__.py | 0 .../yardstick_new_plugin_2/benchmark/__init__.py | 0 .../benchmark/scenarios/__init__.py | 0 .../benchmark/scenarios/dummy2/__init__.py | 0 .../benchmark/scenarios/dummy2/dummy2.py | 40 +++++++++ .../yardstick_new_plugin-1.0.0.tar.gz | Bin 0 -> 1650 bytes yardstick/tests/functional/common/test_packages.py | 94 +++++++++++++++++++++ yardstick/tests/unit/common/test_packages.py | 88 +++++++++++++++++++ 17 files changed, 418 insertions(+), 4 deletions(-) create mode 100644 yardstick/common/packages.py create mode 100644 yardstick/common/privsep.py create mode 100644 yardstick/tests/functional/base.py create mode 100644 yardstick/tests/functional/common/fake_directory_package/README.md create mode 100644 yardstick/tests/functional/common/fake_directory_package/setup.py create mode 100644 yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/__init__.py create mode 100644 yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/__init__.py create mode 100644 yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/__init__.py create mode 100644 yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/__init__.py create mode 100644 yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/dummy2.py create mode 100644 yardstick/tests/functional/common/fake_pip_package/yardstick_new_plugin-1.0.0.tar.gz create mode 100644 yardstick/tests/functional/common/test_packages.py create mode 100644 yardstick/tests/unit/common/test_packages.py diff --git a/requirements.txt b/requirements.txt index 88c0e659a..9e79329b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,11 +37,13 @@ os-client-config==1.28.0 # OSI Approved Apache Software License osc-lib==1.7.0 # OSI Approved Apache Software License oslo.config==4.11.1 # OSI Approved Apache Software License oslo.i18n==3.17.0 # OSI Approved Apache Software License +oslo.privsep===1.22.1 # OSI Approved Apache Software License oslo.serialization==2.20.1 # OSI Approved Apache Software License oslo.utils==3.28.0 # OSI Approved Apache Software License paramiko==2.2.1 # LGPL; OSI Approved GNU Library or Lesser General Public License (LGPL) pbr==3.1.1 # OSI Approved Apache Software License; Apache License, Version 2.0 pika==0.10.0 # BSD; OSI Approved BSD License +pip==9.0.1;python_version=='2.7' # MIT positional==1.1.2 # OSI Approved Apache Software License pycrypto==2.6.1 # Public Domain pyparsing==2.2.0 # MIT License; OSI Approved MIT License diff --git a/test-requirements.txt b/test-requirements.txt index f933df29a..5fed9458f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,6 +4,7 @@ coverage==4.4.2 # Apache 2.0; OSI Approved Apache Software License; http://www.apache.org/licenses/LICENSE-2.0; http://www.apache.org/licenses/LICENSE-2.0 fixtures==3.0.0 # OSI Approved BSD License; OSI Approved Apache Software License +oslotest===2.17.1 # OSI Approved Apache Software License packaging==16.8.0 # BSD or Apache License, Version 2.0 pyflakes==1.0.0 # MIT; OSI Approved MIT License pylint==1.8.1 # GPLv2 diff --git a/tox.ini b/tox.ini index 822ffdab4..313f1eca2 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,8 @@ envlist = py{27,3},pep8,functional{,-py3},coverage [testenv] usedevelop=True passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY BRANCH +setenv = + VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt diff --git a/yardstick/common/packages.py b/yardstick/common/packages.py new file mode 100644 index 000000000..f20217fdc --- /dev/null +++ b/yardstick/common/packages.py @@ -0,0 +1,87 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import re + +import pip +from pip import exceptions as pip_exceptions +from pip.operations import freeze + +from yardstick.common import privsep + + +LOG = logging.getLogger(__name__) + +ACTION_INSTALL = 'install' +ACTION_UNINSTALL = 'uninstall' + + +@privsep.yardstick_root.entrypoint +def _pip_main(package, action, target=None): + if action == ACTION_UNINSTALL: + cmd = [action, package, '-y'] + elif action == ACTION_INSTALL: + cmd = [action, package, '--upgrade'] + if target: + cmd.append('--target=%s' % target) + return pip.main(cmd) + + +def _pip_execute_action(package, action=ACTION_INSTALL, target=None): + """Execute an action with a PIP package. + + According to [1], a package could be a URL, a local directory, a local dist + file or a requirements file. + + [1] https://pip.pypa.io/en/stable/reference/pip_install/#argument-handling + """ + try: + status = _pip_main(package, action, target) + except pip_exceptions.PipError: + status = 1 + + if not status: + LOG.info('Action "%s" executed, package %s', package, action) + else: + LOG.info('Error executing action "%s", package %s', package, action) + return status + + +def pip_remove(package): + """Remove an installed PIP package""" + return _pip_execute_action(package, action=ACTION_UNINSTALL) + + +def pip_install(package, target=None): + """Install a PIP package""" + return _pip_execute_action(package, action=ACTION_INSTALL, target=target) + + +def pip_list(pkg_name=None): + """Dict of installed PIP packages with version. + + If 'pkg_name' is not None, will return only those packages matching the + name.""" + pip_regex = re.compile(r"(?P.*)==(?P[\w\.]+)") + git_regex = re.compile(r".*@(?P[\w]+)#egg=(?P[\w]+)") + + pkg_dict = {} + for _pkg in freeze.freeze(local_only=True): + match = pip_regex.match(_pkg) or git_regex.match(_pkg) + if match and (not pkg_name or ( + pkg_name and match.group('name').find(pkg_name) != -1)): + pkg_dict[match.group('name')] = match.group('version') + + return pkg_dict diff --git a/yardstick/common/privsep.py b/yardstick/common/privsep.py new file mode 100644 index 000000000..4ae510489 --- /dev/null +++ b/yardstick/common/privsep.py @@ -0,0 +1,23 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_privsep import capabilities as c +from oslo_privsep import priv_context + +yardstick_root = priv_context.PrivContext( + "yardstick", + cfg_section="yardstick_privileged", + pypath=__name__ + ".yardstick_root", + capabilities=[c.CAP_SYS_ADMIN] +) diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py index 8604e900f..8d6d31efb 100644 --- a/yardstick/common/utils.py +++ b/yardstick/common/utils.py @@ -30,6 +30,7 @@ import six from flask import jsonify from six.moves import configparser from oslo_serialization import jsonutils +from oslo_utils import encodeutils import yardstick @@ -105,13 +106,12 @@ def remove_file(path): raise -def execute_command(cmd): +def execute_command(cmd, **kwargs): exec_msg = "Executing command: '%s'" % cmd logger.debug(exec_msg) - output = subprocess.check_output(cmd.split()).split(os.linesep) - - return output + output = subprocess.check_output(cmd.split(), **kwargs) + return encodeutils.safe_decode(output, incoming='utf-8').split(os.linesep) def source_env(env_file): diff --git a/yardstick/tests/functional/base.py b/yardstick/tests/functional/base.py new file mode 100644 index 000000000..51be013a1 --- /dev/null +++ b/yardstick/tests/functional/base.py @@ -0,0 +1,46 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import six + +from oslo_config import cfg +from oslotest import base + + +CONF = cfg.CONF + + +@six.add_metaclass(abc.ABCMeta) +class BaseFunctionalTestCase(base.BaseTestCase): + """Base class for functional tests.""" + + def setUp(self): + super(BaseFunctionalTestCase, self).setUp() + + def config(self, **kw): + """Override some configuration values. + + The keyword arguments are the names of configuration options to + override and their values. + + If a group argument is supplied, the overrides are applied to + the specified configuration option group. + + All overrides are automatically cleared at the end of the current + test by the fixtures cleanup process. + """ + group = kw.pop('group', None) + for k, v in kw.items(): + CONF.set_override(k, v, group) diff --git a/yardstick/tests/functional/common/fake_directory_package/README.md b/yardstick/tests/functional/common/fake_directory_package/README.md new file mode 100644 index 000000000..689e47039 --- /dev/null +++ b/yardstick/tests/functional/common/fake_directory_package/README.md @@ -0,0 +1,2 @@ +# yardstick_new_plugin +Yardstick plugin diff --git a/yardstick/tests/functional/common/fake_directory_package/setup.py b/yardstick/tests/functional/common/fake_directory_package/setup.py new file mode 100644 index 000000000..cf938ef4f --- /dev/null +++ b/yardstick/tests/functional/common/fake_directory_package/setup.py @@ -0,0 +1,29 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup, find_packages + +setup( + name='yardstick_new_plugin_2', + version='1.0.0', + packages=find_packages(), + include_package_data=True, + url='https://www.opnfv.org', + entry_points={ + 'yardstick.scenarios': [ + 'Dummy2 = yardstick_new_plugin.benchmark.scenarios.dummy2.dummy2:' + 'Dummy2', + ] + }, +) diff --git a/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/__init__.py b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/__init__.py b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/__init__.py b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/__init__.py b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/dummy2.py b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/dummy2.py new file mode 100644 index 000000000..a2211ec51 --- /dev/null +++ b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/dummy2.py @@ -0,0 +1,40 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from yardstick.benchmark.scenarios import base + + +LOG = logging.getLogger(__name__) + + +class Dummy2(base.Scenario): + """Execute Dummy (v2!) echo""" + __scenario_type__ = "Dummy2" + + def __init__(self, scenario_cfg, context_cfg): + self.scenario_cfg = scenario_cfg + self.context_cfg = context_cfg + self.setup_done = False + + def setup(self): + self.setup_done = True + + def run(self, result): + if not self.setup_done: + self.setup() + + result["hello"] = "yardstick" + LOG.info("Dummy (v2!) echo hello yardstick!") diff --git a/yardstick/tests/functional/common/fake_pip_package/yardstick_new_plugin-1.0.0.tar.gz b/yardstick/tests/functional/common/fake_pip_package/yardstick_new_plugin-1.0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..e5379a78a48d6290b26b9fa01dc87637c26aa00a GIT binary patch literal 1650 zcmV-&295b2iwFphXK-2q|72-%bT4^fa%6LKX=7_&Ze@30aBOvFX>KhsE-)@IE_7jX z0PS1dPvbZc?{oeNZ}x#!q9I9>^u(zTJz#gMW!n{nTdfd^OyUNMG|4#*r8?dJe&c-f zYs(f`=)vK6abwSn$NqWzF?Oez`6;K~wHuL%n}k_Fqo$$hn!Z@A@Zvo64((ROu_B>D~CRE7C|1G-))7 z_ns4f(=s5D_`hWQd7QW*xgnv(Z~12Y9ml#Gf74#UAGny16J zz?ym_!i>i0l5oW714;0PIL|$%FqBg=&p0Hh$7sT#?&d)pad7eG=`>l- zG0*%OVm^x5c%CEI{3r4Mb+htg_y2I%U-JK!ZaC8aw?fh#r2jw4|5vwIzp(xrEoWu@ zZ(DNx-wvJ6P}XLf#|z~!-!`HSS~~`2`^~wCR&F$lWWSJCy^Z!R(3OqdiYg(o) zHR-)hJN{aU~w*U7KJ2D5hH(PltEiBrwAutkcTh9%n(md%n%M3fifNogC~q~8U?#3jYoWf z838RmO*x~N8DHwH$~y{%P-M zhz2L<1&M=tts1d1|Z z@pv9US{F4!rN|{tiJ~aSgO8|F$!h`O$MBP|y6Z-GOkB4CJYI;Qc}0AwF66XfsjacE z2)tEQJ-Q_x^lZU}>Nn=Q1|r^7T$Sgk>sDsCe42oEs9G)bs!Gn`lM$-cw0cUy5kOpt zcq0JZ6W36;Tzr(33lf~#A|ohX{9I;S&;fi4Un}~=<-F_15dp3bI28R=*_fB*1}<0u z&8#Xd7^-%!S9!{^1&u6=$_B$=%0j-Zmz3Qr(4T?2sGg2;4_~Qeu4CDRdPA9&6xE%p zS0oH$^+G^08$fgYFitcPgRZNq2%}tZKIq=54H<)c_Vs^LOz1)TKeHw7|E!jb|GaSg z?;i<#2>!Y_Cz_6BOZ@+9{2%bb=hyK6;r)-%Hrs89|2F7e?|&BG2Fv$9Z)N?TJq>+s z|JUu={f}iZvEH%;mP+n{q2 z1g~`t@wqcq1{=aR?`yUzq zm+}AC6#viT{rRN9<2alJ_w$@xfI0GA0N@&`NtNQ?X7xOzD}wtSb=^dy?&b_P#i;72 z`MgwSR;oHnitCM{ltx~d`J~DhOBwEnAa_xmv9P1AJ)V}@7hgAb{+>mich[\w\.-]+) \((?P[\w\d_\.\-]+),*.*\)") + pkg_dict = {} + pkgs = utils.execute_command('pip list', + env={'PYTHONPATH': self.TMP_FOLDER}) + for line in pkgs: + match = pip_list_regex.match(line) + if match and match.group('name'): + pkg_dict[match.group('name')] = match.group('version') + return pkg_dict + + def test_install_from_folder(self): + dirname = path.dirname(__file__) + package_dir = dirname + '/fake_directory_package' + package_name = 'yardstick-new-plugin-2' + self.addCleanup(self._remove_package, package_name) + self._remove_package(package_name) + self.assertFalse(package_name in self._list_packages()) + + self.assertEqual(0, packages.pip_install(package_dir, self.TMP_FOLDER)) + self.assertTrue(package_name in self._list_packages()) + + def test_install_from_pip_package(self): + dirname = path.dirname(__file__) + package_path = (dirname + + '/fake_pip_package/yardstick_new_plugin-1.0.0.tar.gz') + package_name = 'yardstick-new-plugin' + self.addCleanup(self._remove_package, package_name) + self._remove_package(package_name) + self.assertFalse(package_name in self._list_packages()) + + self.assertEqual(0, packages.pip_install(package_path, self.TMP_FOLDER)) + self.assertTrue(package_name in self._list_packages()) + + # NOTE(ralonsoh): an stable test plugin project is needed in OPNFV git + # server to execute this test. + # def test_install_from_url(self): + + def test_pip_freeze(self): + # NOTE (ralonsoh): from requirements.txt file. The best way to test + # this function is to parse requirements.txt and test-requirements.txt + # and check all packages. + pkgs_ref = {'Babel': '2.3.4', + 'SQLAlchemy': '1.1.12', + 'influxdb': '4.1.1', + 'netifaces': '0.10.6', + 'unicodecsv': '0.14.1'} + pkgs = packages.pip_list() + for name, version in (pkgs_ref.items()): + self.assertEqual(version, pkgs[name]) diff --git a/yardstick/tests/unit/common/test_packages.py b/yardstick/tests/unit/common/test_packages.py new file mode 100644 index 000000000..ba59a3015 --- /dev/null +++ b/yardstick/tests/unit/common/test_packages.py @@ -0,0 +1,88 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from pip import exceptions as pip_exceptions +from pip.operations import freeze +import unittest + +from yardstick.common import packages + + +class PipExecuteActionTestCase(unittest.TestCase): + + def setUp(self): + self._mock_pip_main = mock.patch.object(packages, '_pip_main') + self.mock_pip_main = self._mock_pip_main.start() + self.mock_pip_main.return_value = 0 + self._mock_freeze = mock.patch.object(freeze, 'freeze') + self.mock_freeze = self._mock_freeze.start() + self.addCleanup(self._cleanup) + + def _cleanup(self): + self._mock_pip_main.stop() + self._mock_freeze.stop() + + def test_pip_execute_action(self): + self.assertEqual(0, packages._pip_execute_action('test_package')) + + def test_remove(self): + self.assertEqual(0, packages._pip_execute_action('test_package', + action='uninstall')) + + def test_install(self): + self.assertEqual(0, packages._pip_execute_action( + 'test_package', action='install', target='temp_dir')) + + def test_pip_execute_action_error(self): + self.mock_pip_main.return_value = 1 + self.assertEqual(1, packages._pip_execute_action('test_package')) + + def test_pip_execute_action_exception(self): + self.mock_pip_main.side_effect = pip_exceptions.PipError + self.assertEqual(1, packages._pip_execute_action('test_package')) + + def test_pip_list(self): + pkg_input = [ + 'XStatic-Rickshaw==1.5.0.0', + 'xvfbwrapper==0.2.9', + '-e git+https://git.opnfv.org/yardstick@50773a24afc02c9652b662ecca' + '2fc5621ea6097a#egg=yardstick', + 'zope.interface==4.4.3' + ] + pkg_dict = { + 'XStatic-Rickshaw': '1.5.0.0', + 'xvfbwrapper': '0.2.9', + 'yardstick': '50773a24afc02c9652b662ecca2fc5621ea6097a', + 'zope.interface': '4.4.3' + } + self.mock_freeze.return_value = pkg_input + + pkg_output = packages.pip_list() + for pkg_name, pkg_version in pkg_output.items(): + self.assertEqual(pkg_dict.get(pkg_name), pkg_version) + + def test_pip_list_single_package(self): + pkg_input = [ + 'XStatic-Rickshaw==1.5.0.0', + 'xvfbwrapper==0.2.9', + '-e git+https://git.opnfv.org/yardstick@50773a24afc02c9652b662ecca' + '2fc5621ea6097a#egg=yardstick', + 'zope.interface==4.4.3' + ] + self.mock_freeze.return_value = pkg_input + + pkg_output = packages.pip_list(pkg_name='xvfbwrapper') + self.assertEqual(1, len(pkg_output)) + self.assertEqual(pkg_output.get('xvfbwrapper'), '0.2.9') -- 2.16.6