X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=yardstick%2Fcommon%2Futils.py;h=a77a4cad61ca2130fefe06bf4b617ea51a581ab4;hb=14a0f0b13e37cf0a65603915b7c75b86fd4a4383;hp=969bb4b388607b5b07a40f26222ae5fcfbabd184;hpb=daa256f05a1f42f66a84ad96321bc609aeb1f75b;p=yardstick.git diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py index 969bb4b38..a77a4cad6 100644 --- a/yardstick/common/utils.py +++ b/yardstick/common/utils.py @@ -13,29 +13,25 @@ # License for the specific language governing permissions and limitations # under the License. -# yardstick comment: this is a modified copy of rally/rally/common/utils.py - -from __future__ import absolute_import -from __future__ import print_function - +import collections +from contextlib import closing +import datetime import errno +import importlib +import ipaddress import logging import os +import random +import re +import socket import subprocess import sys -import collections -import socket -import random -import ipaddress -from functools import reduce -from contextlib import closing -import yaml import six from flask import jsonify from six.moves import configparser -from oslo_utils import importutils from oslo_serialization import jsonutils +from oslo_utils import encodeutils import yardstick @@ -70,54 +66,29 @@ def itersubclasses(cls, _seen=None): yield sub -def try_append_module(name, modules): - if name not in modules: - modules[name] = importutils.import_module(name) - - def import_modules_from_package(package): - """Import modules from package and append into sys.modules + """Import modules given a package name :param: package - Full package name. For example: rally.deploy.engines """ - path = [os.path.dirname(yardstick.__file__), ".."] + package.split(".") - path = os.path.join(*path) - for root, dirs, files in os.walk(path): - for filename in files: - if filename.startswith("__") or not filename.endswith(".py"): - continue - new_package = ".".join(root.split(os.sep)).split("....")[1] - module_name = "%s.%s" % (new_package, filename[:-3]) + yardstick_root = os.path.dirname(os.path.dirname(yardstick.__file__)) + path = os.path.join(yardstick_root, *package.split('.')) + for root, _, files in os.walk(path): + matches = (filename for filename in files if filename.endswith('.py') + and not filename.startswith('__')) + new_package = os.path.relpath(root, yardstick_root).replace(os.sep, + '.') + module_names = set( + '{}.{}'.format(new_package, filename.rsplit('.py', 1)[0]) + for filename in matches) + # Find modules which haven't already been imported + missing_modules = module_names.difference(sys.modules) + logger.debug('Importing modules: %s', missing_modules) + for module_name in missing_modules: try: - try_append_module(module_name, sys.modules) - except ImportError: - logger.exception("unable to import %s", module_name) - - -def parse_yaml(file_path): - try: - with open(file_path) as f: - value = yaml.safe_load(f) - except IOError: - return {} - except OSError as e: - if e.errno != errno.EEXIST: - raise - else: - return value - - -def get_param(key, default=''): - - conf_file = os.environ.get('CONF_FILE', '/etc/yardstick/yardstick.yaml') - - conf = parse_yaml(conf_file) - try: - return reduce(lambda a, b: a[b], key.split('.'), conf) - except KeyError: - if not default: - raise - return default + importlib.import_module(module_name) + except (ImportError, SyntaxError): + logger.exception('Unable to import module %s', module_name) def makedirs(d): @@ -136,13 +107,12 @@ def remove_file(path): raise -def execute_command(cmd): +def execute_command(cmd, **kwargs): exec_msg = "Executing command: '%s'" % cmd logger.debug(exec_msg) - output = subprocess.check_output(cmd.split()).split(os.linesep) - - return output + output = subprocess.check_output(cmd.split(), **kwargs) + return encodeutils.safe_decode(output, incoming='utf-8').split(os.linesep) def source_env(env_file): @@ -278,10 +248,10 @@ def set_dict_value(dic, keys, value): def get_free_port(ip): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: - while True: + port = random.randint(5000, 10000) + while s.connect_ex((ip, port)) == 0: port = random.randint(5000, 10000) - if s.connect_ex((ip, port)) != 0: - return port + return port def mac_address_to_hex_list(mac): @@ -310,7 +280,7 @@ def get_ip_version(ip_addr): return address.version -def ip_to_hex(ip_addr): +def ip_to_hex(ip_addr, separator=''): try: address = ipaddress.ip_address(six.text_type(ip_addr)) except ValueError: @@ -319,7 +289,11 @@ def ip_to_hex(ip_addr): if address.version != 4: return ip_addr - return '{:08x}'.format(int(address)) + + if not separator: + return '{:08x}'.format(int(address)) + + return separator.join('{:02x}'.format(octet) for octet in address.packed) def try_int(s, *args): @@ -332,6 +306,29 @@ def try_int(s, *args): class SocketTopology(dict): + @classmethod + def parse_cpuinfo(cls, cpuinfo): + socket_map = {} + + lines = cpuinfo.splitlines() + + core_details = [] + core_lines = {} + for line in lines: + if line.strip(): + name, value = line.split(":", 1) + core_lines[name.strip()] = try_int(value.strip()) + else: + core_details.append(core_lines) + core_lines = {} + + for core in core_details: + socket_map.setdefault(core["physical id"], {}).setdefault( + core["core id"], {})[core["processor"]] = ( + core["processor"], core["core id"], core["physical id"]) + + return cls(socket_map) + def sockets(self): return sorted(self.keys()) @@ -344,39 +341,19 @@ class SocketTopology(dict): proc in procs) -def parse_cpuinfo(cpuinfo): - socket_map = {} - - lines = cpuinfo.splitlines() - - core_details = [] - core_lines = {} - for line in lines: - if line.strip(): - name, value = line.split(":", 1) - core_lines[name.strip()] = try_int(value.strip()) - else: - core_details.append(core_lines) - core_lines = {} - - for core in core_details: - socket_map.setdefault(core["physical id"], {}).setdefault( - core["core id"], {})[core["processor"]] = ( - core["processor"], core["core id"], core["physical id"]) - - return SocketTopology(socket_map) - - def config_to_dict(config): return {section: dict(config.items(section)) for section in config.sections()} def validate_non_string_sequence(value, default=None, raise_exc=None): - if isinstance(value, collections.Sequence) and not isinstance(value, str): + # NOTE(ralonsoh): refactor this function to check if raise_exc is an + # Exception. Remove duplicate code, this function is duplicated in this + # repository. + if isinstance(value, collections.Sequence) and not isinstance(value, six.string_types): return value if raise_exc: - raise raise_exc + raise raise_exc # pylint: disable=raising-bad-type return default @@ -388,6 +365,13 @@ def join_non_strings(separator, *non_strings): return str(separator).join(str(non_string) for non_string in non_strings) +def safe_decode_utf8(s): + """Safe decode a str from UTF""" + if six.PY3 and isinstance(s, bytes): + return s.decode('utf-8', 'surrogateescape') + return s + + class ErrorClass(object): def __init__(self, *args, **kwargs): @@ -396,3 +380,33 @@ class ErrorClass(object): def __getattr__(self, item): raise AttributeError + + +class Timer(object): + def __init__(self): + super(Timer, self).__init__() + self.start = self.delta = None + + def __enter__(self): + self.start = datetime.datetime.now() + return self + + def __exit__(self, *_): + self.delta = datetime.datetime.now() - self.start + + def __getattr__(self, item): + return getattr(self.delta, item) + + +def read_meminfo(ssh_client): + """Read "/proc/meminfo" file and parse all keys and values""" + + cpuinfo = six.BytesIO() + ssh_client.get_file_obj('/proc/meminfo', cpuinfo) + lines = cpuinfo.getvalue().decode('utf-8') + matches = re.findall(r"([\w\(\)]+):\s+(\d+)( kB)*", lines) + output = {} + for match in matches: + output[match[0]] = match[1] + + return output