1 # Copyright 2013: Mirantis Inc.
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
17 from contextlib import closing
31 from flask import jsonify
32 from six.moves import configparser
33 from oslo_serialization import jsonutils
37 logger = logging.getLogger(__name__)
38 logger.setLevel(logging.DEBUG)
41 # Decorator for cli-args
42 def cliargs(*args, **kwargs):
44 func.__dict__.setdefault('arguments', []).insert(0, (args, kwargs))
49 def itersubclasses(cls, _seen=None):
50 """Generator over all subclasses of a given class in depth first order."""
52 if not isinstance(cls, type):
53 raise TypeError("itersubclasses must be called with "
54 "new-style classes, not %.100r" % cls)
55 _seen = _seen or set()
57 subs = cls.__subclasses__()
58 except TypeError: # fails only when cls is type
59 subs = cls.__subclasses__(cls)
64 for sub in itersubclasses(sub, _seen):
68 def import_modules_from_package(package, raise_exception=False):
69 """Import modules given a package name
71 :param: package - Full package name. For example: rally.deploy.engines
73 yardstick_root = os.path.dirname(os.path.dirname(yardstick.__file__))
74 path = os.path.join(yardstick_root, *package.split('.'))
75 for root, _, files in os.walk(path):
76 matches = (filename for filename in files if filename.endswith('.py')
77 and not filename.startswith('__'))
78 new_package = os.path.relpath(root, yardstick_root).replace(os.sep,
81 '{}.{}'.format(new_package, filename.rsplit('.py', 1)[0])
82 for filename in matches)
83 # Find modules which haven't already been imported
84 missing_modules = module_names.difference(sys.modules)
85 logger.debug('Importing modules: %s', missing_modules)
86 for module_name in missing_modules:
88 importlib.import_module(module_name)
89 except (ImportError, SyntaxError) as exc:
92 logger.exception('Unable to import module %s', module_name)
99 if e.errno != errno.EEXIST:
103 def remove_file(path):
107 if e.errno != errno.ENOENT:
111 def execute_command(cmd):
112 exec_msg = "Executing command: '%s'" % cmd
113 logger.debug(exec_msg)
115 output = subprocess.check_output(cmd.split()).split(os.linesep)
120 def source_env(env_file):
121 p = subprocess.Popen(". %s; env" % env_file, stdout=subprocess.PIPE,
123 output = p.communicate()[0]
124 env = dict(line.split('=', 1) for line in output.splitlines() if '=' in line)
125 os.environ.update(env)
129 def read_json_from_file(path):
130 with open(path, 'r') as f:
132 # don't use jsonutils.load() it conflicts with already decoded input
133 return jsonutils.loads(j)
136 def write_json_to_file(path, data, mode='w'):
137 with open(path, mode) as f:
138 jsonutils.dump(data, f)
141 def write_file(path, data, mode='w'):
142 with open(path, mode) as f:
146 def parse_ini_file(path):
147 parser = configparser.ConfigParser()
150 files = parser.read(path)
151 except configparser.MissingSectionHeaderError:
152 logger.exception('invalid file type')
156 raise RuntimeError('file not exist')
159 default = {k: v for k, v in parser.items('DEFAULT')}
160 except configparser.NoSectionError:
163 config = dict(DEFAULT=default,
164 **{s: {k: v for k, v in parser.items(
165 s)} for s in parser.sections()})
170 def get_port_mac(sshclient, port):
171 cmd = "ifconfig |grep HWaddr |grep %s |awk '{print $5}' " % port
172 status, stdout, stderr = sshclient.execute(cmd)
175 raise RuntimeError(stderr)
176 return stdout.rstrip()
179 def get_port_ip(sshclient, port):
180 cmd = "ifconfig %s |grep 'inet addr' |awk '{print $2}' " \
181 "|cut -d ':' -f2 " % port
182 status, stdout, stderr = sshclient.execute(cmd)
185 raise RuntimeError(stderr)
186 return stdout.rstrip()
189 def flatten_dict_key(data):
192 # use list, because iterable is too generic
193 if not any(isinstance(v, (collections.Mapping, list))
194 for v in data.values()):
197 for k, v in data.items():
198 if isinstance(v, collections.Mapping):
199 for n_k, n_v in v.items():
200 next_data["%s.%s" % (k, n_k)] = n_v
201 # use list because iterable is too generic
202 elif isinstance(v, collections.Iterable) and not isinstance(v, six.string_types):
203 for index, item in enumerate(v):
204 next_data["%s%d" % (k, index)] = item
208 return flatten_dict_key(next_data)
211 def translate_to_str(obj):
212 if isinstance(obj, collections.Mapping):
213 return {str(k): translate_to_str(v) for k, v in obj.items()}
214 elif isinstance(obj, list):
215 return [translate_to_str(ele) for ele in obj]
216 elif isinstance(obj, six.text_type):
221 def result_handler(status, data):
226 return jsonify(result)
229 def change_obj_to_dict(obj):
231 for k, v in vars(obj).items():
239 def set_dict_value(dic, keys, value):
242 for key in keys.split('.'):
243 return_dic.setdefault(key, {})
244 if key == keys.split('.')[-1]:
245 return_dic[key] = value
247 return_dic = return_dic[key]
251 def get_free_port(ip):
252 with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
253 port = random.randint(5000, 10000)
254 while s.connect_ex((ip, port)) == 0:
255 port = random.randint(5000, 10000)
259 def mac_address_to_hex_list(mac):
260 octets = ["0x{:02x}".format(int(elem, 16)) for elem in mac.split(':')]
261 assert len(octets) == 6 and all(len(octet) == 4 for octet in octets)
265 def safe_ip_address(ip_addr):
266 """ get ip address version v6 or v4 """
268 return ipaddress.ip_address(six.text_type(ip_addr))
270 logging.error("%s is not valid", ip_addr)
274 def get_ip_version(ip_addr):
275 """ get ip address version v6 or v4 """
277 address = ipaddress.ip_address(six.text_type(ip_addr))
279 logging.error("%s is not valid", ip_addr)
282 return address.version
285 def ip_to_hex(ip_addr, separator=''):
287 address = ipaddress.ip_address(six.text_type(ip_addr))
289 logging.error("%s is not valid", ip_addr)
292 if address.version != 4:
296 return '{:08x}'.format(int(address))
298 return separator.join('{:02x}'.format(octet) for octet in address.packed)
301 def try_int(s, *args):
302 """Convert to integer if possible."""
305 except (TypeError, ValueError):
306 return args[0] if args else s
309 class SocketTopology(dict):
312 def parse_cpuinfo(cls, cpuinfo):
315 lines = cpuinfo.splitlines()
321 name, value = line.split(":", 1)
322 core_lines[name.strip()] = try_int(value.strip())
324 core_details.append(core_lines)
327 for core in core_details:
328 socket_map.setdefault(core["physical id"], {}).setdefault(
329 core["core id"], {})[core["processor"]] = (
330 core["processor"], core["core id"], core["physical id"])
332 return cls(socket_map)
335 return sorted(self.keys())
338 return sorted(core for cores in self.values() for core in cores)
340 def processors(self):
342 proc for cores in self.values() for procs in cores.values() for
346 def config_to_dict(config):
347 return {section: dict(config.items(section)) for section in
351 def validate_non_string_sequence(value, default=None, raise_exc=None):
352 # NOTE(ralonsoh): refactor this function to check if raise_exc is an
353 # Exception. Remove duplicate code, this function is duplicated in this
355 if isinstance(value, collections.Sequence) and not isinstance(value, six.string_types):
358 raise raise_exc # pylint: disable=raising-bad-type
362 def join_non_strings(separator, *non_strings):
364 non_strings = validate_non_string_sequence(non_strings[0], raise_exc=RuntimeError)
365 except (IndexError, RuntimeError):
367 return str(separator).join(str(non_string) for non_string in non_strings)
370 def safe_decode_utf8(s):
371 """Safe decode a str from UTF"""
372 if six.PY3 and isinstance(s, bytes):
373 return s.decode('utf-8', 'surrogateescape')
377 class ErrorClass(object):
379 def __init__(self, *args, **kwargs):
380 if 'test' not in kwargs:
383 def __getattr__(self, item):
389 super(Timer, self).__init__()
390 self.start = self.delta = None
393 self.start = datetime.datetime.now()
396 def __exit__(self, *_):
397 self.delta = datetime.datetime.now() - self.start
399 def __getattr__(self, item):
400 return getattr(self.delta, item)
403 def read_meminfo(ssh_client):
404 """Read "/proc/meminfo" file and parse all keys and values"""
406 cpuinfo = six.BytesIO()
407 ssh_client.get_file_obj('/proc/meminfo', cpuinfo)
408 lines = cpuinfo.getvalue().decode('utf-8')
409 matches = re.findall(r"([\w\(\)]+):\s+(\d+)( kB)*", lines)
411 for match in matches:
412 output[match[0]] = match[1]
417 def find_relative_file(path, task_path):
419 Find file in one of places: in abs of path or relative to a directory path,
424 :return str: full path to file
426 # fixme: create schema to validate all fields have been provided
427 for lookup in [os.path.abspath(path), os.path.join(task_path, path)]:
433 raise IOError(errno.ENOENT, 'Unable to find {} file'.format(path))
436 def open_relative_file(path, task_path):
440 if e.errno == errno.ENOENT:
441 return open(os.path.join(task_path, path))