X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=yardstick%2Fcommon%2Futils.py;h=357f66be86c089550c88a9e6a66544b7e1fc3c5f;hb=c5fe46b232edb3474dab8d99797dcbd8b982b447;hp=6ac99a5a9d32ac0d7bc323c04189f8b8816758f2;hpb=ec578c17540cfc57bb1c522046d369c5fdaf584b;p=yardstick.git diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py index 6ac99a5a9..357f66be8 100644 --- a/yardstick/common/utils.py +++ b/yardstick/common/utils.py @@ -13,28 +13,25 @@ # License for the specific language governing permissions and limitations # under the License. -# yardstick comment: this is a modified copy of rally/rally/common/utils.py - -from __future__ import absolute_import -from __future__ import print_function - +import collections +from contextlib import closing import datetime import errno +import importlib +import ipaddress import logging import os +import random +import re +import socket import subprocess import sys -import collections -import socket -import random -import ipaddress -from contextlib import closing import six from flask import jsonify from six.moves import configparser -from oslo_utils import importutils from oslo_serialization import jsonutils +from oslo_utils import encodeutils import yardstick @@ -69,28 +66,46 @@ def itersubclasses(cls, _seen=None): yield sub -def import_modules_from_package(package): - """Import modules from package and append into sys.modules +def import_modules_from_package(package, raise_exception=False): + """Import modules given a package name :param: package - Full package name. For example: rally.deploy.engines """ yardstick_root = os.path.dirname(os.path.dirname(yardstick.__file__)) - path = os.path.join(yardstick_root, *package.split(".")) - for root, dirs, files in os.walk(path): - matches = (filename for filename in files if filename.endswith(".py") and - not filename.startswith("__")) - new_package = os.path.relpath(root, yardstick_root).replace(os.sep, ".") + path = os.path.join(yardstick_root, *package.split('.')) + for root, _, files in os.walk(path): + matches = (filename for filename in files if filename.endswith('.py') + and not filename.startswith('__')) + new_package = os.path.relpath(root, yardstick_root).replace(os.sep, + '.') module_names = set( - ("{}.{}".format(new_package, filename.rsplit(".py", 1)[0]) for filename in matches)) - # find modules which haven't already been imported + '{}.{}'.format(new_package, filename.rsplit('.py', 1)[0]) + for filename in matches) + # Find modules which haven't already been imported missing_modules = module_names.difference(sys.modules) - logger.debug("importing %s", missing_modules) - # we have already checked for already imported modules, so we don't need to check again + logger.debug('Importing modules: %s', missing_modules) for module_name in missing_modules: try: - sys.modules[module_name] = importutils.import_module(module_name) - except (ImportError, SyntaxError): - logger.exception("unable to import %s", module_name) + importlib.import_module(module_name) + except (ImportError, SyntaxError) as exc: + if raise_exception: + raise exc + logger.exception('Unable to import module %s', module_name) + + +NON_NONE_DEFAULT = object() + + +def get_key_with_default(data, key, default=NON_NONE_DEFAULT): + value = data.get(key, default) + if value is NON_NONE_DEFAULT: + raise KeyError(key) + return value + + +def make_dict_from_map(data, key_map): + return {dest_key: get_key_with_default(data, src_key, default) + for dest_key, (src_key, default) in key_map.items()} def makedirs(d): @@ -109,13 +124,12 @@ def remove_file(path): raise -def execute_command(cmd): +def execute_command(cmd, **kwargs): exec_msg = "Executing command: '%s'" % cmd logger.debug(exec_msg) - output = subprocess.check_output(cmd.split()).split(os.linesep) - - return output + output = subprocess.check_output(cmd.split(), **kwargs) + return encodeutils.safe_decode(output, incoming='utf-8').split(os.linesep) def source_env(env_file): @@ -251,10 +265,10 @@ def set_dict_value(dic, keys, value): def get_free_port(ip): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: - while True: + port = random.randint(5000, 10000) + while s.connect_ex((ip, port)) == 0: port = random.randint(5000, 10000) - if s.connect_ex((ip, port)) != 0: - return port + return port def mac_address_to_hex_list(mac): @@ -350,10 +364,13 @@ def config_to_dict(config): def validate_non_string_sequence(value, default=None, raise_exc=None): - if isinstance(value, collections.Sequence) and not isinstance(value, str): + # NOTE(ralonsoh): refactor this function to check if raise_exc is an + # Exception. Remove duplicate code, this function is duplicated in this + # repository. + if isinstance(value, collections.Sequence) and not isinstance(value, six.string_types): return value if raise_exc: - raise raise_exc + raise raise_exc # pylint: disable=raising-bad-type return default @@ -365,6 +382,13 @@ def join_non_strings(separator, *non_strings): return str(separator).join(str(non_string) for non_string in non_strings) +def safe_decode_utf8(s): + """Safe decode a str from UTF""" + if six.PY3 and isinstance(s, bytes): + return s.decode('utf-8', 'surrogateescape') + return s + + class ErrorClass(object): def __init__(self, *args, **kwargs): @@ -389,3 +413,45 @@ class Timer(object): def __getattr__(self, item): return getattr(self.delta, item) + + +def read_meminfo(ssh_client): + """Read "/proc/meminfo" file and parse all keys and values""" + + cpuinfo = six.BytesIO() + ssh_client.get_file_obj('/proc/meminfo', cpuinfo) + lines = cpuinfo.getvalue().decode('utf-8') + matches = re.findall(r"([\w\(\)]+):\s+(\d+)( kB)*", lines) + output = {} + for match in matches: + output[match[0]] = match[1] + + return output + + +def find_relative_file(path, task_path): + """ + Find file in one of places: in abs of path or relative to a directory path, + in this order. + + :param path: + :param task_path: + :return str: full path to file + """ + # fixme: create schema to validate all fields have been provided + for lookup in [os.path.abspath(path), os.path.join(task_path, path)]: + try: + with open(lookup): + return lookup + except IOError: + pass + raise IOError(errno.ENOENT, 'Unable to find {} file'.format(path)) + + +def open_relative_file(path, task_path): + try: + return open(path) + except IOError as e: + if e.errno == errno.ENOENT: + return open(os.path.join(task_path, path)) + raise