X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=yardstick%2Fcommon%2Futils.py;h=68c9ed63f74d1ab30702483afb262a269aca964c;hb=d3ee35dc0015e073b7138f3b02508b40fc6288b1;hp=a4f7b30dca22ea0bd260292cf4f95f09e854aa3d;hpb=2945003ca74fd3d73621c9e1745dd1662b88c2df;p=yardstick.git diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py index a4f7b30dc..68c9ed63f 100644 --- a/yardstick/common/utils.py +++ b/yardstick/common/utils.py @@ -18,15 +18,18 @@ from __future__ import absolute_import from __future__ import print_function +import datetime import errno import logging import os import subprocess import sys import collections -from functools import reduce +import socket +import random +import ipaddress +from contextlib import closing -import yaml import six from flask import jsonify from six.moves import configparser @@ -34,6 +37,7 @@ from oslo_utils import importutils from oslo_serialization import jsonutils import yardstick +from yardstick.common.yaml_loader import yaml_load logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -66,34 +70,34 @@ def itersubclasses(cls, _seen=None): yield sub -def try_append_module(name, modules): - if name not in modules: - modules[name] = importutils.import_module(name) - - def import_modules_from_package(package): """Import modules from package and append into sys.modules :param: package - Full package name. For example: rally.deploy.engines """ - path = [os.path.dirname(yardstick.__file__), ".."] + package.split(".") - path = os.path.join(*path) + yardstick_root = os.path.dirname(os.path.dirname(yardstick.__file__)) + path = os.path.join(yardstick_root, *package.split(".")) for root, dirs, files in os.walk(path): - for filename in files: - if filename.startswith("__") or not filename.endswith(".py"): - continue - new_package = ".".join(root.split(os.sep)).split("....")[1] - module_name = "%s.%s" % (new_package, filename[:-3]) + matches = (filename for filename in files if filename.endswith(".py") and + not filename.startswith("__")) + new_package = os.path.relpath(root, yardstick_root).replace(os.sep, ".") + module_names = set( + ("{}.{}".format(new_package, filename.rsplit(".py", 1)[0]) for filename in matches)) + # find modules which haven't already been imported + missing_modules = module_names.difference(sys.modules) + logger.debug("importing %s", missing_modules) + # we have already checked for already imported modules, so we don't need to check again + for module_name in missing_modules: try: - try_append_module(module_name, sys.modules) - except ImportError: + sys.modules[module_name] = importutils.import_module(module_name) + except (ImportError, SyntaxError): logger.exception("unable to import %s", module_name) def parse_yaml(file_path): try: with open(file_path) as f: - value = yaml.safe_load(f) + value = yaml_load(f) except IOError: return {} except OSError as e: @@ -103,19 +107,6 @@ def parse_yaml(file_path): return value -def get_param(key, default=''): - - conf_file = os.environ.get('CONF_FILE', '/etc/yardstick/yardstick.yaml') - - conf = parse_yaml(conf_file) - try: - return reduce(lambda a, b: a[b], key.split('.'), conf) - except KeyError: - if not default: - raise - return default - - def makedirs(d): try: os.makedirs(d) @@ -145,7 +136,7 @@ def source_env(env_file): p = subprocess.Popen(". %s; env" % env_file, stdout=subprocess.PIPE, shell=True) output = p.communicate()[0] - env = dict((line.split('=', 1) for line in output.splitlines())) + env = dict(line.split('=', 1) for line in output.splitlines() if '=' in line) os.environ.update(env) return env @@ -169,7 +160,15 @@ def write_file(path, data, mode='w'): def parse_ini_file(path): parser = configparser.ConfigParser() - parser.read(path) + + try: + files = parser.read(path) + except configparser.MissingSectionHeaderError: + logger.exception('invalid file type') + raise + else: + if not files: + raise RuntimeError('file not exist') try: default = {k: v for k, v in parser.items('DEFAULT')} @@ -210,12 +209,12 @@ def flatten_dict_key(data): for v in data.values()): return data - for k, v in six.iteritems(data): + for k, v in data.items(): if isinstance(v, collections.Mapping): - for n_k, n_v in six.iteritems(v): + for n_k, n_v in v.items(): next_data["%s.%s" % (k, n_k)] = n_v # use list because iterable is too generic - elif isinstance(v, list): + elif isinstance(v, collections.Iterable) and not isinstance(v, six.string_types): for index, item in enumerate(v): next_data["%s%d" % (k, index)] = item else: @@ -256,10 +255,151 @@ def set_dict_value(dic, keys, value): return_dic = dic for key in keys.split('.'): - return_dic.setdefault(key, {}) if key == keys.split('.')[-1]: return_dic[key] = value else: return_dic = return_dic[key] return dic + + +def get_free_port(ip): + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + while True: + port = random.randint(5000, 10000) + if s.connect_ex((ip, port)) != 0: + return port + + +def mac_address_to_hex_list(mac): + octets = ["0x{:02x}".format(int(elem, 16)) for elem in mac.split(':')] + assert len(octets) == 6 and all(len(octet) == 4 for octet in octets) + return octets + + +def safe_ip_address(ip_addr): + """ get ip address version v6 or v4 """ + try: + return ipaddress.ip_address(six.text_type(ip_addr)) + except ValueError: + logging.error("%s is not valid", ip_addr) + return None + + +def get_ip_version(ip_addr): + """ get ip address version v6 or v4 """ + try: + address = ipaddress.ip_address(six.text_type(ip_addr)) + except ValueError: + logging.error("%s is not valid", ip_addr) + return None + else: + return address.version + + +def ip_to_hex(ip_addr, separator=''): + try: + address = ipaddress.ip_address(six.text_type(ip_addr)) + except ValueError: + logging.error("%s is not valid", ip_addr) + return ip_addr + + if address.version != 4: + return ip_addr + + if not separator: + return '{:08x}'.format(int(address)) + + return separator.join('{:02x}'.format(octet) for octet in address.packed) + + +def try_int(s, *args): + """Convert to integer if possible.""" + try: + return int(s) + except (TypeError, ValueError): + return args[0] if args else s + + +class SocketTopology(dict): + + @classmethod + def parse_cpuinfo(cls, cpuinfo): + socket_map = {} + + lines = cpuinfo.splitlines() + + core_details = [] + core_lines = {} + for line in lines: + if line.strip(): + name, value = line.split(":", 1) + core_lines[name.strip()] = try_int(value.strip()) + else: + core_details.append(core_lines) + core_lines = {} + + for core in core_details: + socket_map.setdefault(core["physical id"], {}).setdefault( + core["core id"], {})[core["processor"]] = ( + core["processor"], core["core id"], core["physical id"]) + + return cls(socket_map) + + def sockets(self): + return sorted(self.keys()) + + def cores(self): + return sorted(core for cores in self.values() for core in cores) + + def processors(self): + return sorted( + proc for cores in self.values() for procs in cores.values() for + proc in procs) + + +def config_to_dict(config): + return {section: dict(config.items(section)) for section in + config.sections()} + + +def validate_non_string_sequence(value, default=None, raise_exc=None): + if isinstance(value, collections.Sequence) and not isinstance(value, str): + return value + if raise_exc: + raise raise_exc + return default + + +def join_non_strings(separator, *non_strings): + try: + non_strings = validate_non_string_sequence(non_strings[0], raise_exc=RuntimeError) + except (IndexError, RuntimeError): + pass + return str(separator).join(str(non_string) for non_string in non_strings) + + +class ErrorClass(object): + + def __init__(self, *args, **kwargs): + if 'test' not in kwargs: + raise RuntimeError + + def __getattr__(self, item): + raise AttributeError + + +class Timer(object): + def __init__(self): + super(Timer, self).__init__() + self.start = self.delta = None + + def __enter__(self): + self.start = datetime.datetime.now() + return self + + def __exit__(self, *_): + self.delta = datetime.datetime.now() - self.start + + def __getattr__(self, item): + return getattr(self.delta, item)