+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from operator import add
-
-from qtip.util.logger import QtipLogger
-
-logger = QtipLogger('calculator').get
-
-
-def dpi_calculator(samples):
- try:
- float_pps = map(lambda x: float(x), samples['pps'])
- float_bps = map(lambda x: float(x), samples['bps'])
- sum_dpi_pps = reduce(add,
- map(lambda x: x / 1000 if x > 100 else x, float_pps))
- sum_dpi_bps = reduce(add,
- map(lambda x: x / 1000 if x > 100 else x, float_bps))
-
- return {'pps': round(sum_dpi_pps / 10, 3), 'bps': round(sum_dpi_bps / 10, 3)}
- except Exception as error:
- logger.error(error)
- return {'pps': None, 'bps': None}
-
-
-def calculate_cpu_usage(cpu_idle):
- try:
- cpu_usage = round((100.0 - float(cpu_idle)), 3)
- return '{0}%'.format(str(cpu_usage))
- except Exception, error:
- logger.error(error)
- return None
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from collections import defaultdict
-from os import path
import re
-import yaml
from qtip.base import BaseActor
from qtip.base.constant import BaseProp
-from qtip.collector import calculator
from qtip.util.logger import QtipLogger
logger = QtipLogger('grep').get
-REGEX_FILE = path.join(path.dirname(__file__), 'regex.yaml')
-
class GrepProp(BaseProp):
FILENAME = 'filename'
with open(filename, 'r') as f:
return filter(lambda x: x is not None,
re.finditer(regex, f.read(), re.MULTILINE))
-
-
-def _parse_logfile(config, paths):
- captured = {}
- for regex_rules_by_file in config:
- filename = \
- '{0}/{1}'.format(paths, regex_rules_by_file[GrepProp.FILENAME])
- for regex in regex_rules_by_file['grep']:
- matches = grep_in_file(filename, regex)
- if len(matches) > 1:
- temp_dict = defaultdict(list)
- for item in [match.groupdict() for match in matches]:
- for key in item:
- temp_dict[key].append(item[key])
- captured.update(temp_dict)
- elif len(matches) == 1:
- captured.update(matches[0].groupdict())
- else:
- logger.error("Nothing is matched from {0}".format(filename))
- return captured
-
-
-# TODO: Hardcord in Danube, it will be removed in the future.
-def parse_sysinfo(config, result_dir):
- sysinfo = _parse_logfile(config, result_dir)
- if "cpu_idle" in sysinfo:
- sysinfo['cpu_usage'] = \
- calculator.calculate_cpu_usage(sysinfo['cpu_idle'])
- sysinfo.pop('cpu_idle')
- return sysinfo
-
-
-# TODO: Hardcord in Danube, it will be removed in the future.
-def parse_test_result(benchmark, config, result_dir):
- test_result = _parse_logfile(config, result_dir)
- if benchmark == 'dpi':
- return calculator.dpi_calculator(test_result)
- if benchmark == 'dhrystone' or benchmark == 'whetstone':
- return {'total_cpus': test_result['total_cpus'],
- 'single_cpu': {'num': test_result['single_cpu'],
- 'score': test_result['score'][0]},
- 'multi_cpus': {'num': test_result['multi_cpus'],
- 'score': test_result['score'][1]}}
- return test_result
-
-
-# TODO: Hardcord in Danube, it will be removed in the future.
-def parse_benchmark_result(result_dir):
- regex_config = yaml.safe_load(file(REGEX_FILE))
- benchmark = result_dir.split('/')[-1]
- result = {'name': benchmark}
-
- test_result = \
- parse_test_result(benchmark, regex_config[benchmark], result_dir)
- result['results'] = test_result.copy()
-
- sysinfo = parse_sysinfo(regex_config['sysinfo'], result_dir)
- result['sysinfo'] = sysinfo.copy()
-
- return result
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-dhrystone:
- - filename: dhrystone
- grep:
- - '^(?P<total_cpus>\d+)\sCPUs in system; running 1 parallel copy of tests$'
- - '.+\srunning (?P<single_cpu>\d+) parallel copy of tests$'
- - '.+\srunning (?P<multi_cpus>\d+) parallel copies of tests$'
- - '^System Benchmarks Index Score \(Partial Only\)\s+(?P<score>\d+\.\d)$'
-whetstone:
- - filename: whetstone
- grep:
- - '^(?P<total_cpus>\d+)\sCPUs in system; running 1 parallel copy of tests$'
- - '.+\srunning (?P<single_cpu>\d+) parallel copy of tests$'
- - '.+\srunning (?P<multi_cpus>\d+) parallel copies of tests$'
- - '^System Benchmarks Index Score \(Partial Only\)\s+(?P<score>\d+\.\d)$'
-dpi:
- - filename: dpi_dump.txt
- grep:
- - |-
- ^\s+nDPI throughput:.+?(?P<pps>\d+.\d+)\s.+\spps.+
- ?(?P<bps>\d+.\d+)\s.+\/sec
-ramspeed:
- - filename: Intmem
- grep:
- - '^INTEGER\s+BatchRun\s+Copy:\s+?(?P<integer_copy>\d+\.\d+)\sMB/s$'
- - '^INTEGER\s+BatchRun\s+Scale:\s+?(?P<integer_scale>\d+\.\d+)\sMB/s$'
- - '^INTEGER\s+BatchRun\s+Add:\s+?(?P<integer_add>\d+\.\d+)\sMB/s$'
- - '^INTEGER\s+BatchRun\s+Triad:\s+?(?P<integer_triad>\d+\.\d+)\sMB/s$'
- - '^INTEGER\s+BatchRun\s+AVERAGE:\s+?(?P<integer_average>\d+\.\d+)\sMB/s$'
- - filename: Floatmem
- grep:
- - '^FL-POINT\s+BatchRun\s+Copy:\s+?(?P<float_copy>\d+\.\d+)\sMB/s$'
- - '^FL-POINT\s+BatchRun\s+Scale:\s+?(?P<float_scale>\d+\.\d+)\sMB/s$'
- - '^FL-POINT\s+BatchRun\s+Add:\s+?(?P<float_add>\d+\.\d+)\sMB/s$'
- - '^FL-POINT\s+BatchRun\s+Triad:\s+?(?P<float_triad>\d+\.\d+)\sMB/s$'
- - '^FL-POINT\s+BatchRun\s+AVERAGE:\s+?(?P<float_average>\d+\.\d+)\sMB/s$'
-ssl:
- - filename: RSA_dump
- grep:
- - |-
- ^rsa\s+512\sbits\s.+\s+
- ?(?P<rsa_sign_512>\d+\.\d)\s+
- ?(?P<rsa_verify_512>\d+\.\d)$
- - |-
- ^rsa\s+1024\sbits\s.+\s+
- ?(?P<rsa_sign_1024>\d+\.\d)\s+
- ?(?P<rsa_verify_1024>\d+\.\d)$
- - |-
- ^rsa\s+2048\sbits\s.+\s+
- ?(?P<rsa_sign_2048>\d+\.\d)\s+
- ?(?P<rsa_verify_2048>\d+\.\d)$
- - |-
- ^rsa\s+4096\sbits\s.+\s+
- ?(?P<rsa_sign_4096>\d+\.\d)\s+
- ?(?P<rsa_verify_4096>\d+\.\d)$
- - filename: AES-128-CBC_dump
- grep:
- - |-
- ^aes-128-cbc\s+
- ?(?P<aes_128_cbc_16_bytes>\d+\.\w+)\s+
- ?(?P<aes_128_cbc_64_bytes>\d+\.\w+)\s+
- ?(?P<aes_128_cbc_256_bytes>\d+\.\w+)\s+
- ?(?P<aes_128_cbc_1024_bytes>\d+\.\w+)\s+
- ?(?P<aes_128_cbc_8192_bytes>\d+\.\w+)$
-sysinfo:
- - filename: top.log
- grep:
- - 'Cpu\(s\):.+?(?P<cpu_idle>\d+\.\d)\sid'
- - filename: inxi.log
- grep:
- - '.+\s+Host:\s+(?P<hostname>.+)\sKernel'
- - '.+\sMemory:\s+(?P<memory>.+MB)\s'
- - '^CPU\(s\):\s+(?P<cpu>.+)'
- - '.+\sDistro:\s+(?P<os>.+)'
- - '.+\sKernel:\s+(?P<kernel>.+)\sConsole'
- - '.+\s+HDD Total Size:\s+(?P<disk>.+)\s'
- - '.+\sproduct:\s+(?P<product>.+)\sv'
\ No newline at end of file
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from collections import namedtuple
-
-from ansible.executor.playbook_executor import PlaybookExecutor
-from ansible.inventory import Inventory
-from ansible.parsing.dataloader import DataLoader
-from ansible.vars import VariableManager
-
-
-class AnsibleApi(object):
-
- def __init__(self):
- self.variable_manager = VariableManager()
- self.loader = DataLoader()
- self.passwords = {}
- self.playbook_executor = None
-
- def execute_playbook(self, playbook_path, hosts_file=None,
- key_file=None, extra_vars=None):
- inventory = Inventory(loader=self.loader,
- variable_manager=self.variable_manager,
- host_list=hosts_file)
- Options = namedtuple('Options',
- ['listtags', 'listtasks', 'listhosts', 'syntax',
- 'connection', 'module_path', 'forks', 'remote_user',
- 'private_key_file', 'ssh_common_args', 'ssh_extra_args',
- 'sftp_extra_args', 'scp_extra_args', 'become',
- 'become_method', 'become_user', 'verbosity', 'check'])
- options = Options(listtags=False, listtasks=False, listhosts=False,
- syntax=False, connection='ssh', module_path=None,
- forks=100, remote_user='root', private_key_file=key_file,
- ssh_common_args=None, ssh_extra_args=None, sftp_extra_args=None,
- scp_extra_args=None, become=None, become_method=None,
- become_user='root', verbosity=None, check=False)
- self.variable_manager.extra_vars = extra_vars
-
- self.playbook_executor = PlaybookExecutor(playbooks=[playbook_path],
- inventory=inventory,
- variable_manager=self.variable_manager,
- loader=self.loader,
- options=options,
- passwords=self.passwords)
- return self.playbook_executor.run()
-
- def get_detail_playbook_stats(self):
- if self.playbook_executor:
- stats = self.playbook_executor._tqm._stats
- return map(lambda x: (x, stats.summarize(x)), stats.processed.keys())
- else:
- return None
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from collections import defaultdict
-from os import path
-from operator import add
-
-from qtip.driver.ansible_api import AnsibleApi
-from qtip.util.env import AnsibleEnvSetup
-from qtip.util.logger import QtipLogger
-
-logger = QtipLogger('ansible_driver').get
-PLAYBOOK_DIR = path.join(path.dirname(__file__), 'playbook')
-
-
-class AnsibleDriver(object):
- """driver for running performance tests with Ansible"""
-
- def __init__(self, config={}):
- self.config = config
- self.env = AnsibleEnvSetup()
- self.env_setup_flag = False
-
- @staticmethod
- def merge_two_dicts(x, y):
- '''
- It is from http://stackoverflow.com/questions/38987/
- how-can-i-merge-two-python-dictionaries-in-a-single-expression
- '''
- z = x.copy()
- z.update(y)
- return z
-
- def pre_run(self):
- if self.env_setup_flag:
- logger.info("Already setup environment......")
- else:
- logger.info("Starting to setup test environment...")
- self.env.setup(self.config)
- self.env_setup_flag = True
- logger.info("Setup test enviroment, Done!")
-
- def cleanup(self):
- self.env.cleanup()
-
- def run(self, metric_list, **kwargs):
- if 'args' in self.config:
- extra_vars = self.merge_two_dicts(kwargs, self.config['args'])
- else:
- extra_vars = kwargs
- logger.info("extra_var: {0}".format(extra_vars))
-
- tool_to_metrics = defaultdict(list)
- for metric in metric_list:
- if metric == 'dhrystone' or metric == 'whetstone':
- tool_to_metrics['unixbench'].append(metric)
- extra_vars[metric] = True
- elif metric == 'ssl':
- tool_to_metrics['openssl'].append(metric)
- else:
- tool_to_metrics[metric].append(metric)
-
- result_list = map(lambda tool: self._run_metric(tool,
- tool_to_metrics[tool],
- extra_vars),
- tool_to_metrics)
- return False not in result_list
-
- def _run_metric(self, tool, metrics, extra_vars):
- logger.info('Using {0} to measure metrics {1}'.format(tool, metrics))
-
- setup_pbook = "{0}/{1}/setup.yaml".format(PLAYBOOK_DIR, tool)
- run_pbook = "{0}/{1}/run.yaml".format(PLAYBOOK_DIR, tool)
- clean_pbook = "{0}/{1}/clean.yaml".format(PLAYBOOK_DIR, tool)
-
- if self._run_ansible_playbook(setup_pbook, extra_vars):
- self._run_ansible_playbook(run_pbook, extra_vars)
- else:
- logger.error("{0} is failed.".format(setup_pbook))
-
- return self._run_ansible_playbook(clean_pbook, extra_vars)
-
- def _run_ansible_playbook(self, pbook, extra_vars):
- ansible_api = AnsibleApi()
- logger.debug("Run {0} with extra_vars: {1}".format(pbook, extra_vars))
- ansible_api.execute_playbook(pbook, self.env.hostfile,
- self.env.keypair['private'], extra_vars)
- playbook_stats = ansible_api.get_detail_playbook_stats()
- logger.debug("playbook_stat: {0}".format(playbook_stats))
- return self.is_pass(playbook_stats)
-
- @staticmethod
- def is_pass(stats):
- return 0 == reduce(add,
- map(lambda x: x[1]['failures'] + x[1]['unreachable'],
- stats))
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-class BaseDriver(object):
- """performance testing tool driver"""
- def pre_run(self):
- pass
-
- def run(self):
- pass
-
- def post_run(self):
- pass
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- hosts: hosts
- become: yes
- remote_user: root
-
- tasks:
- - name: Cleaning tempD
- file:
- path: '{{ ansible_env.HOME }}/tempD'
- state: absent
-
- - name: Cleaning qtip_result
- file:
- path: '{{ ansible_env.HOME }}/qtip_result'
- state: absent
+++ /dev/null
-#!/bin/bash
-
-COUNTER=0
-WDIR=$PWD
-while [ $COUNTER -lt 10 ]; do
- echo $WDIR
- $( ./ndpiReader -i test.pcap >> $WDIR/dpi_dump.txt )
- let COUNTER=COUNTER+1
- echo "Run number: $COUNTER"
-done
\ No newline at end of file
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- hosts: hosts
- become: yes
- remote_user: root
-
- tasks:
- - name: Make some directories needed
- file:
- path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dpi/'
- state: directory
-
- - include: ../inxi.yaml
- vars:
- dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dpi/'
-
- - include: ../top.yaml
- vars:
- dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dpi/'
-
- - name: Run nDPI benchmark
- shell: ./dpi_average.sh
- args:
- chdir: '{{ ansible_env.HOME }}/tempD/nDPI/example/'
-
- - name: Copying result and system info to qtip result directory
- command: cp $HOME/tempD/nDPI/example/dpi_dump.txt ./
- args:
- chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dpi/'
-
- - name: Fetch result files to local manchine
- synchronize:
- mode: pull
- src: '{{ ansible_env.HOME }}/qtip_result/'
- dest: '{{ result_dir }}/'
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- hosts: localhost
- connection: local
- gather_facts: no
-
- tasks:
- - name: Making Dpi directory
- file:
- path: '{{ result_dir }}/'
- state: directory
-
-- hosts: hosts
- become: yes
- remote_user: root
-
- tasks:
- - name: Cleaning tempD directory
- file:
- path: '{{ ansible_env.HOME }}/tempD'
- state: absent
-
- - name: Cleaning qtip_result directory
- file:
- path: '{{ ansible_env.HOME }}/qtip_result'
- state: absent
-
- - include: ../prepare_env.yaml
-
- - name: Installing nDPI dependencies if CentOS
- yum:
- name: '{{ item }}'
- state: present
- when: ansible_os_family == "RedHat"
- with_items:
- - git
- - gcc
- - patch
- - perl-Time-HiRes
- - autofconf
- - automake
- - libpcap-devel libtool
-
- - name: Installing nDPI dependencies if Ubuntu
- apt:
- name: '{{ item }}'
- state: present
- when: ansible_os_family == "Debian"
- with_items:
- - git
- - gcc
- - patch
- - autoconf
- - automake
- - libpcap-dev
- - libtool
-
- - name: Making nDPI temporary directory
- file:
- path: '{{ ansible_env.HOME }}/tempD'
- state: directory
-
- - name: Clone nDPI
- git:
- repo: https://github.com/ntop/nDPI.git
- dest: '{{ ansible_env.HOME }}/tempD/nDPI'
-
- - name: Run autogen && configure Dpi && make Dpi
- command: '{{ item }}'
- with_items:
- - ./autogen.sh
- - ./configure
- - make
- args:
- chdir: '{{ ansible_env.HOME }}/tempD/nDPI/'
-
- - name: Fetching Test_pcap file
- get_url:
- url: http://build.opnfv.org/artifacts.opnfv.org/qtip/utilities/test.pcap
- dest: '{{ ansible_env.HOME }}/tempD/nDPI/example/test.pcap'
-
- - name: Fetch Averaging script
- copy:
- src: ./dpi_average.sh
- dest: '{{ ansible_env.HOME }}/tempD/nDPI/example'
- mode: 777
\ No newline at end of file
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-- name: Install Inxi when CentOS
- yum:
- name: inxi
- state: present
- when: ansible_os_family == "RedHat"
-
-- name: Install Inxi when Ubuntu
- apt:
- name: inxi
- state: present
- update_cache: yes
- when: ansible_os_family == "Debian"
-
-- name: Run inxi
- shell: inxi -b -c0 -n > inxi.log
- args:
- chdir: '{{ dest_path }}'
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- hosts: hosts
- become: yes
- remote_user: root
-
- tasks:
- - name: Cleaning Open_SSL
- file:
- path: '{{ ansible_env.HOME }}/Open_SSL'
- state: absent
-
- - name: Cleaning qtip_result
- file:
- path: '{{ ansible_env.HOME }}/qtip_result'
- state: absent
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- hosts: hosts
- become: yes
- remote_user: root
-
- tasks:
- - name: Make some directories needed
- file:
- path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ssl/'
- state: directory
-
- - include: ../inxi.yaml
- vars:
- dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ssl/'
-
- - include: ../top.yaml
- vars:
- dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ssl/'
-
- - name: Benchmarking RSA signatures and AES-128-cbc cipher encryption throughput
- shell: '{{ item }}'
- with_items:
- - ./openssl speed rsa >> RSA_dump
- - ./openssl speed -evp aes-128-cbc >> AES-128-CBC_dump
- args:
- chdir: '{{ ansible_env.HOME }}/Open_SSL/openssl-1.0.2f/apps'
-
- - name: Copying result to qtip result directory
- shell: cp ~/Open_SSL/openssl-1.0.2f/apps/*_dump ./
- args:
- chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ssl/'
-
- - name: Fetch result files to local manchine
- synchronize:
- mode: pull
- src: '{{ ansible_env.HOME }}/qtip_result/'
- dest: '{{ result_dir }}/'
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- hosts: localhost
- connection: local
- gather_facts: no
-
- tasks:
- - name: Making ssl directory
- file:
- path: '{{ result_dir }}/'
- state: directory
-
-- hosts: hosts
- become: yes
- remote_user: root
-
- tasks:
- - name: Cleaning Open_SSL directory
- file:
- path: '{{ ansible_env.HOME }}/Open_SSL'
- state: absent
-
- - name: Cleaning qtip_result directory
- file:
- path: '{{ ansible_env.HOME }}/qtip_result'
- state: absent
-
- - include: ../prepare_env.yaml
-
- - name: Installing UnixBench dependencies if CentOS
- yum:
- name: '{{ item }}'
- state: present
- when: ansible_os_family == "RedHat"
- with_items:
- - git
- - gcc
- - patch
- - perl-Time-HiRes
- - wget
- - autofconf
- - automake
- - libpcap-devel
- - libtool
-
- - name: Installing UnixBench dependencies if Ubuntu
- apt:
- name: '{{ item }}'
- state: present
- when: ansible_os_family == "Debian"
- with_items:
- - git
- - gcc
- - patch
- - perl
- - wget
- - autoconf
- - automake
- - libpcap-dev
- - libtool
-
- - name: Make Open_SSL directory
- file:
- path: '{{ ansible_env.HOME }}/Open_SSL'
- state: directory
-
- - name: Untar OpenSSL
- unarchive:
- src: http://artifacts.opnfv.org/qtip/utilities/openssl-1.0.2f.tar.gz
- dest: '{{ ansible_env.HOME }}/Open_SSL/'
- remote_src: True
-
- - name: Configure && Make && Install OpenSSL
- shell: "{{ item }}"
- with_items:
- - ./config
- - make
- - make install
- args:
- chdir: '{{ ansible_env.HOME }}/Open_SSL/openssl-1.0.2f'
\ No newline at end of file
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-- name: Epel Release install when CentOS
- yum:
- name: epel-release
- state: present
- when: ansible_os_family == "RedHat"
-
-- name: Software Properties Common
- apt:
- name: software-properties-common
- state: present
- when: ansible_os_family == "Debian"
-
-- name: Adding ubuntu backport main repo
- apt_repository:
- repo: deb http://archive.ubuntu.com/ubuntu/ {{ansible_distribution_release}}-backports main restricted universe multiverse
- state: present
- when: ansible_os_family == "Debian"
-
-- name: Adding ubuntu main repo
- apt_repository:
- repo: deb http://archive.ubuntu.com/ubuntu/ {{ansible_distribution_release}} main restricted universe multiverse
- when: ansible_os_family == "Debian"
-
-- name: Install ansible copy dependencies if remote host has selinux enabled
- yum:
- name: libselinux-python
- state: present
- when: ansible_os_family == "RedHat"
-
-- name: Install ansiblle copy dependencies if remote host has selinux enaled
- apt:
- name: python-selinux
- state: present
- when: ansible_os_family == "Debian"
-
-- name: Install rsync when CentOS
- yum:
- name: rsync
- state: present
- when: ansible_os_family == "RedHat"
-
-- name: Install rsync when Ubuntu
- apt:
- name: rsync
- state: present
- when: ansible_os_family == "Debian"
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- hosts: hosts
- become: yes
- remote_user: root
-
- tasks:
- - name: Cleaning ramspeed
- file:
- path: '{{ ansible_env.HOME }}/ramspeed'
- state: absent
-
- - name: Cleaning qtip_result
- file:
- path: '{{ ansible_env.HOME }}/qtip_result'
- state: absent
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- hosts: hosts
- become: yes
- remote_user: root
-
- tasks:
- - name: Make some directories needed
- file:
- path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ramspeed/'
- state: directory
-
- - include: ../inxi.yaml
- vars:
- dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ramspeed/'
-
- - include: ../top.yaml
- vars:
- dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ramspeed/'
-
- - name: Benchmarking IntMem Bandwidth and FloatMem Bandwidth
- shell: '{{ item }}'
- with_items:
- - ~/ramspeed/ramsmp-3.5.0/ramsmp -b 3 -l 5 -p 1 >> Intmem
- - ~/ramspeed/ramsmp-3.5.0/ramsmp -b 6 -l 5 -p 1 >> Floatmem
- args:
- chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ramspeed/'
-
- - name: Fetch result files to local manchine
- synchronize:
- mode: pull
- src: '{{ ansible_env.HOME }}/qtip_result/'
- dest: '{{ result_dir }}/'
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- hosts: localhost
- connection: local
- gather_facts: no
-
- tasks:
- - name: Making ramspeed directory
- file:
- path: '{{ result_dir }}/'
- state: directory
-
-- hosts: hosts
- become: yes
- remote_user: root
-
- tasks:
- - name: Cleaning ramspeed directory
- file:
- path: '{{ ansible_env.HOME }}/ramspeed'
- state: absent
-
- - name: Cleaning qtip_result directory
- file:
- path: '{{ ansible_env.HOME }}/qtip_result'
- state: absent
-
- - include: ../prepare_env.yaml
-
- - name: Installing RAM_Speed dependencies if CentOS
- yum:
- name: '{{ item }}'
- state: present
- when: ansible_os_family == "RedHat"
- with_items:
- - gcc
- - wget
-
- - name: Installing RAM_Speed dependencies if Ubuntu
- apt:
- name: '{{ item }}'
- state: present
- when: ansible_os_family == "Debian"
- with_items:
- - gcc
- - wget
-
- - name: Making ramspeed temporary directory
- file:
- path: '{{ ansible_env.HOME }}/ramspeed'
- state: directory
-
- - name: Fetch and untar ramspeed.tar.gz
- unarchive:
- # TODO: Need to upload this file to http://artifacts.opnfv.org/qtip/utilities
- src: https://docs.google.com/uc?id=0B92Bp5LZTM7gRFctalZLMktTNDQ
- dest: '{{ ansible_env.HOME }}/ramspeed/'
- remote_src: True
-
- - name: Build ramsmp
- shell: ./build.sh
- args:
- chdir: '{{ ansible_env.HOME }}/ramspeed/ramsmp-3.5.0'
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-# zhihui.wu1@zte.com.cn
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- hosts: hosts
- become: yes
- remote_user: root
-
- tasks:
- - name: Cleaning tempT
- file:
- path: '{{ ansible_env.HOME }}/tempT'
- state: absent
-
- - name: Cleaning qtip_result
- file:
- path: '{{ ansible_env.HOME }}/qtip_result'
- state: absent
-
-
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-# zhihui.wu1@zte.com.cn
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- name: Make dhrystone directories
- file:
- path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dhrystone/'
- state: directory
-
-- include: ../inxi.yaml
- vars:
- dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dhrystone/'
-
-- include: ../top.yaml
- vars:
- dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dhrystone/'
-
-- name: Run dhrystone
- shell: ./Run -v dhrystone
- args:
- chdir: '{{ ansible_env.HOME }}/tempT/UnixBench/'
-
-- name: Copying result to qtip result directory
- shell: '{{ item }}'
- with_items:
- - mv ~/tempT/UnixBench/results/*.log ./
- - mv ~/tempT/UnixBench/results/*.html ./
- - mv ~/tempT/UnixBench/results/* ./dhrystone
- args:
- chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dhrystone/'
-
-- name: Fetch dhrystone result files to local manchine
- synchronize:
- mode: pull
- src: '{{ ansible_env.HOME }}/qtip_result/'
- dest: '{{ result_dir }}/'
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-# zhihui.wu1@zte.com.cn
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- hosts: hosts
- become: yes
- remote_user: root
-
- tasks:
- - set_fact:
- is_dhrystone: "{{ dhrystone | default(False) }}"
- is_whetstone: "{{ whetstone | default(False) }}"
-
- - include: ./dhrystone.yaml
- when: "{{ is_dhrystone }}"
-
- - include: ./whetstone.yaml
- when: "{{ is_whetstone }}"
\ No newline at end of file
+++ /dev/null
-#############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-# zhihui.wu1@zte.com.cn
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-#############################################################################
-
-- hosts: localhost
- connection: local
- gather_facts: no
-
- tasks:
- - name: Making dhrystone directory
- file:
- path: '{{ result_dir }}/'
- state: directory
-
-- hosts: hosts
- become: yes
- remote_user: root
-
- tasks:
- - name: Cleaning tempT directory
- file:
- path: '{{ ansible_env.HOME }}/tempT'
- state: absent
-
- - name: Cleaning qtip_result directory
- file:
- path: '{{ ansible_env.HOME }}/qtip_result'
- state: absent
-
- - include: ../prepare_env.yaml
-
- - name: Installing UnixBench dependencies if CentOS
- yum:
- name: '{{ item }}'
- state: present
- when: ansible_os_family == "RedHat"
- with_items:
- - git
- - gcc
- - patch
- - perl-Time-HiRes
-
- - name: Installing UnixBench dependencies if Ubuntu
- apt:
- name: '{{ item }}'
- state: present
- when: ansible_os_family == "Debian"
- with_items:
- - git
- - gcc
- - patch
- - perl
-
- - name: Clone UnixBench
- git:
- repo: https://github.com/kdlucas/byte-unixbench.git
- dest: '{{ ansible_env.HOME }}/tempT/'
-
- - name: Make UnixBench1
- shell: make --directory $HOME/tempT/UnixBench/
\ No newline at end of file
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-# zhihui.wu1@zte.com.cn
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-- name: Make whetstone directories
- file:
- path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/whetstone/'
- state: directory
-
-- include: ../inxi.yaml
- vars:
- dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/whetstone/'
-
-- include: ../top.yaml
- vars:
- dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/whetstone/'
-
-- name: Run whetstone
- shell: ./Run -v whetstone
- args:
- chdir: '{{ ansible_env.HOME }}/tempT/UnixBench/'
-
-- name: Copying result to qtip result directory
- shell: '{{ item }}'
- with_items:
- - mv ~/tempT/UnixBench/results/*.log ./
- - mv ~/tempT/UnixBench/results/*.html ./
- - mv ~/tempT/UnixBench/results/* ./whetstone
- args:
- chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/whetstone/'
-
-- name: Fetch whetstone result files to local manchine
- synchronize:
- mode: pull
- src: '{{ ansible_env.HOME }}/qtip_result/'
- dest: '{{ result_dir }}/'
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE corp. and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-import argparse
-import json
-import os
-from os import path
-import sys
-import time
-
-from qtip.collector.parser import grep
-from qtip.driver.ansible_driver import AnsibleDriver
-from qtip.util.logger import QtipLogger
-
-logger = QtipLogger('runner').get
-
-ALL_BENCHMARKS = ['dpi', 'ramspeed', 'ssl', 'dhrystone', 'whetstone']
-
-
-def parse_args(args):
- parser = argparse.ArgumentParser()
- parser.add_argument('-d', '--dest', required=True,
- help='the destination where results will be stored.')
- parser.add_argument('-b', '--benchmark', required=True, action='append',
- help='the benchmark you want to execute.')
- return parser.parse_args(args)
-
-
-def run_benchmark(result_dir, benchmarks):
- if not path.isdir(result_dir):
- os.makedirs(result_dir)
- driver = AnsibleDriver({'args': {'result_dir': result_dir}})
- driver.pre_run()
- result = driver.run(benchmarks)
- driver.cleanup()
- return result
-
-
-def generate_report(result_dir, start_time, stop_time):
- output = {
- "plan_name": "compute_qpi",
- "start_time": start_time,
- "stop_time": stop_time,
- "sut": []
- }
- output.update(parse_result(result_dir))
- output.update({'stop_time': stop_time})
- with open('{0}/result.json'.format(result_dir), 'w+') as f:
- json.dump(output, f, indent=4, sort_keys=True)
-
-
-def parse_result(result_dir):
- sut_template = {'sut': []}
- nodes_list = os.listdir(result_dir)
- for node in nodes_list:
- node_output_template = {
- 'name': node,
- 'type': 'baremetal',
- 'qpis': []
- }
- qpi_result = {'name': 'compute_qpi', 'benchmarks': []}
- for benchmark in os.listdir('{0}/{1}'.format(result_dir, node)):
- benchmark_result = \
- grep.parse_benchmark_result(
- '{0}/{1}/{2}'.format(result_dir, node, benchmark))
- qpi_result['benchmarks'].append(benchmark_result)
- node_output_template['qpis'].append(qpi_result)
- sut_template['sut'].append(node_output_template)
- return sut_template
-
-
-def main(args=sys.argv[1:]):
- args = parse_args(args)
-
- if not path.isdir(str(args.dest)):
- logger.error("The destination {0} you give doesn't exist. "
- "Please check!".format(args.dest))
- sys.exit(1)
-
- if args.benchmark == ['all']:
- args.benchmark = ALL_BENCHMARKS
- elif len(set(args.benchmark).difference(ALL_BENCHMARKS)) != 0:
- logger.error("Please check benchmarks name. The supported benchmarks are"
- "{0}".format(ALL_BENCHMARKS))
- logger.info("Start to run benchmark test: {0}.".format(args.benchmark))
-
- start_time = time.strftime("%Y-%m-%d-%H-%M")
- logger.info("start_time: {0}".format(start_time))
- if not args.dest.endswith('/'):
- args.dest += '/'
- result_dir = args.dest + 'qtip-' + start_time
- ansible_result = run_benchmark(result_dir, args.benchmark)
- stop_time = time.strftime("%Y-%m-%d-%H-%M")
- logger.info("stop_time: {0}".format(stop_time))
- if not ansible_result:
- logger.error("Bechmarks run failed. Cann't generate any report.")
- sys.exit(1)
- generate_report(result_dir, start_time, stop_time)
-
-
-if __name__ == "__main__":
- main()
+++ /dev/null
-#! /bin/bash
-##############################################################################
-# Copyright (c) 2017 ZTE corp. and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-DEST_IP=$1
-PRIVATE_KEY=$2
-PUBLIC_KEY=$3
-sshoptions="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
-
-case "$INSTALLER_TYPE" in
- fuel)
- ssh $sshoptions -i $PRIVATE_KEY root@$DEST_IP "sed -i '/$PUBLIC_KEY/d' /root/.ssh/authorized_keys"
- ;;
-esac
+++ /dev/null
-#!/bin/bash
-##############################################################################
-#Copyright (c) 2016 Ericsson AB, ZTE and others.
-#jose.lausuch@ericsson.com
-#All rights reserved. This program and the accompanying materials
-#are made available under the terms of the Apache License, Version 2.0
-#which accompanies this distribution, and is available at
-#http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-usage(){
- echo "usage: $0 -t <installer_type> -i <installer_ipaddr> -d <dest_hostfile>" >&2
-}
-
-info() {
- logger -s -t "generate_host_file.info" "$*"
-}
-
-error() {
- logger -s -t "generate_host_file.error" "$*"
- exit 1
-}
-
-verify_connectivity(){
- local ip=$1
- info "Verifying connectivity to $ip..."
- for i in $(seq 0 10); do
- if ping -c 1 -W 1 $ip > /dev/null; then
- info "$ip is reachable!"
- return 0
- fi
- sleep 1
- done
- error "Can not talk to $ip."
-}
-
-:${DEPLOY_TYPE:=''}
-
-#Getoptions
-while getopts ":t:i:d:" optchar; do
- case "${optchar}" in
- t) installer_type=${OPTARG} ;;
- i) installer_ipaddr=${OPTARG} ;;
- d) dest_hostfile=${OPTARG} ;;
- *) echo "Non-option argument: '-${OPTARG}'" >&2
- usage
- exit 2
- ;;
- esac
-done
-
-#set vars from env if not provided by user as options
-installer_type=${installer_type:-$INSTALLER_TYPE}
-installer_ipaddr=${installer_ipaddr:-$INSTALLER_IP}
-
-if [ -z $installer_type ] || [ -z $installer_ipaddr ]; then
- usage
- exit 2
-fi
-
-ssh_options="-oUserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
-
-#Start fetching compute ip
-if [ "$installer_type" == "fuel" ]; then
- verify_connectivity $installer_ipaddr
-
- env=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ipaddr} \
- 'fuel env'|grep operational|head -1|awk '{print $1}') &> /dev/null
- if [ -z $env ]; then
- error "No operational environment detected in Fuel"
- fi
- env_id="${FUEL_ENV:-$env}"
-
- # Check if compute is alive (online='True')
- IPS=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ipaddr} \
- "fuel node --env ${env_id} | grep compute | grep 'True\| 1' | awk -F\| '{print \$5}' " | \
- sed 's/ //g') &> /dev/null
-
-
-elif [ "$installer_type" == "apex" ]; then
- echo "not implement now"
- exit 1
-
-elif [ "$installer_type" == "compass" ]; then
- # need test
- verify_connectivity $installer_ipaddr
- IPS=$(sshpass -p'root' ssh 2>/dev/null $ssh_options root@${installer_ipaddr} \
- 'mysql -ucompass -pcompass -Dcompass -e"select * from cluster;"' \
- | awk -F"," '{for(i=1;i<NF;i++)if($i~/\"host[4-5]\"/) {print $(i+1);}}' \
- | grep -oP "\d+.\d+.\d+.\d+")
-
-elif [ "$installer_type" == "joid" ]; then
- echo "not implement now"
- exit 1
-
-elif [ "$installer_type" == "foreman" ]; then
- echo "not implement now"
- exit 1
-
-else
- error "Installer $installer is not supported by this script"
-fi
-
-if [ -z "$IPS" ]; then
- error "The compute node $IPS are not up. Please check that the POD is correctly deployed."
-else
- echo "-------- all compute node ips: --------"
- rm $dest_hostfile
- touch $dest_hostfile
- echo "[hosts]" >> $dest_hostfile
- echo "$IPS" >> $dest_hostfile
- cat $dest_hostfile
-fi
-
-exit 0
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-from collections import defaultdict
-import os
-from os import path
-import re
-import socket
-import sys
-import time
-
-import paramiko
-
-from qtip.util.logger import QtipLogger
-
-logger = QtipLogger('env').get
-
-SCRIPT_DIR = path.join(path.dirname(__file__), path.pardir, 'scripts')
-KEYNAME = 'QtipKey'
-PRIVATE_KEY = '{0}/qtip/{1}'.format(os.environ['HOME'], KEYNAME)
-PUBLIC_KEY = PRIVATE_KEY + '.pub'
-HOST_FILE = '{0}/qtip/hosts'.format(os.environ['HOME'])
-
-
-def all_files_exist(*files):
- if len(files) == 0:
- return False
- flag = True
- for f_item in files:
- flag &= path.isfile(f_item)
- logger.info("Is {0} existed: {1}".format(f_item, flag))
- return flag
-
-
-def clean_file(*files):
- if len(files) == 0:
- logger.info('Nothing to clean')
- return False
-
- def clean(f):
- try:
- if all_files_exist(f):
- os.remove(f)
- logger.info("Removed: {0}".format(f))
- else:
- logger.info("Not exists: {0}".format(f))
- return True
- except OSError as error:
- logger.error("Not able to Remove: {0}".format(f), error)
- return False
-
- results = map(clean, files)
- return len(results) == len(files) and False not in results
-
-
-class AnsibleEnvSetup(object):
- def __init__(self):
- self.keypair = defaultdict(str)
- self.hostfile = None
- self.host_ip_list = []
-
- def setup(self, config={}):
- try:
- if 'hostfile' in config:
- self.check_hostfile(config['hostfile'])
- else:
- self.generate_default_hostfile()
- self.fetch_host_ip_from_hostfile()
- if 'keypair' in config:
- self.check_keypair(config['keypair'])
- else:
- self.generate_default_keypair()
- self.pass_keypair_to_remote()
- self.check_hosts_ssh_connectivity()
- except Exception as error:
- logger.info(error)
- sys.exit(1)
-
- def check_keypair(self, keypair):
- self.keypair = defaultdict(str)
- if all_files_exist(keypair, '{0}.pub'.format(keypair)):
- self.keypair['private'] = keypair
- self.keypair['public'] = '{0}.pub'.format(keypair)
- else:
- raise RuntimeError("The keypairs you in the configuration file"
- " is invalid or not existed.")
-
- def generate_default_keypair(self):
- if not all_files_exist(PRIVATE_KEY, PUBLIC_KEY):
- logger.info("Generate default keypair {0} under "
- "{1}".format(KEYNAME, os.environ['HOME']))
- cmd = '''ssh-keygen -t rsa -N "" -f {0} -q -b 2048
- -C qtip@insecure'''.format(PRIVATE_KEY)
- os.system(cmd)
- self.keypair['private'] = PRIVATE_KEY
- self.keypair['public'] = PUBLIC_KEY
-
- def pass_keypair_to_remote(self):
- results = map(lambda ip: self._pass_keypair(ip, self.keypair['private']),
- self.host_ip_list)
-
- if not (len(results) == len(self.host_ip_list) and False not in results):
- raise RuntimeError("Failed on passing keypair to remote.")
-
- @staticmethod
- def _pass_keypair(ip, private_key):
- try:
- os.system('ssh-keyscan %s >> /root/.ssh/known_hosts' % ip)
- time.sleep(2)
- ssh_cmd = '%s/qtip_creds.sh %s %s' % (SCRIPT_DIR, ip, private_key)
- os.system(ssh_cmd)
- logger.info('Pass keypair to remote hosts {0} successfully'.format(ip))
- return True
- except Exception as error:
- logger.error(error)
- return False
-
- def check_hostfile(self, hostfile):
- if all_files_exist(hostfile):
- self.hostfile = hostfile
- else:
- raise RuntimeError(
- "The hostfile {0} is invalid or not existed.".format(hostfile))
-
- def generate_default_hostfile(self):
- try:
- # check whether the file is already existed
- self.check_hostfile(HOST_FILE)
- except Exception:
- logger.info("Generate default hostfile {0} under "
- "{1}".format(HOST_FILE, os.environ['HOME']))
- self._generate_hostfile_via_installer()
-
- def _generate_hostfile_via_installer(self):
- self.hostfile = None
-
- installer_type = str(os.environ['INSTALLER_TYPE'].lower())
- installer_ip = str(os.environ['INSTALLER_IP'])
-
- if installer_type not in ["fuel"]:
- raise ValueError("{0} is not supported".format(installer_type))
- if not installer_ip:
- raise ValueError(
- "The value of environment variable INSTALLER_IP is empty.")
-
- cmd = "bash %s/generate_host_file.sh -t %s -i %s -d %s" % \
- (SCRIPT_DIR, installer_type, installer_ip, HOST_FILE)
- os.system(cmd)
-
- self.hostfile = HOST_FILE
-
- def fetch_host_ip_from_hostfile(self):
- self.host_ip_list = []
- logger.info('Fetch host ips from hostfile...')
- with open(self.hostfile, 'r') as f:
- self.host_ip_list = re.findall('\d+.\d+.\d+.\d+', f.read())
- if self.host_ip_list:
- logger.info("The remote compute nodes: {0}".format(self.host_ip_list))
- else:
- raise ValueError("The hostfile doesn't include host ip addresses.")
-
- def check_hosts_ssh_connectivity(self):
- results = map(lambda ip: self._ssh_is_ok(ip, self.keypair['private']),
- self.host_ip_list)
- if not (len(results) == len(self.host_ip_list) and False not in results):
- raise RuntimeError("Failed on checking hosts ssh connectivity.")
-
- @staticmethod
- def _ssh_is_ok(ip, private_key, attempts=100):
- logger.info('Check hosts {0} ssh connectivity...'.format(ip))
- ssh = paramiko.SSHClient()
- ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
- ssh.connect(ip, key_filename=private_key)
-
- for attempt in range(attempts):
- try:
- stdin, stdout, stderr = ssh.exec_command('uname')
- if not stderr.readlines():
- logger.info("{0}: SSH test successful.".format(ip))
- return True
- except socket.error:
- logger.debug("%s times ssh test......failed." % str(attempt + 1))
- if attempt == (attempts - 1):
- return False
- time.sleep(2)
- return False
-
- def cleanup(self):
- CI_DEBUG = os.getenv('CI_DEBUG')
-
- if CI_DEBUG is not None and CI_DEBUG.lower() == 'true':
- logger.info("DEBUG Mode: please do cleanup by manual.")
- else:
- with open(self.keypair['public'], 'r') as f:
- key = f.read().strip('\n').replace('/', '\/')
- if key:
- for ip in self.host_ip_list:
- logger.info("Cleanup authorized_keys from {0}...".format(ip))
- cmd = '''bash {0}/cleanup_creds.sh {1} {2} "{3}"'''.format(
- SCRIPT_DIR, ip, self.keypair['private'], key)
- os.system(cmd)
- else:
- logger.error("Nothing in public key file.")
-
- logger.info("Cleanup hostfile and keypair.")
- clean_file(self.hostfile,
- self.keypair['private'],
- self.keypair['public'])
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-from qtip.runner.base import BaseRunner
-
-
-def test_constructor():
- runner = BaseRunner()
- assert isinstance(runner, BaseRunner)
+++ /dev/null
-###############################################################
-# Copyright (c) 2017 ZTE Corporation
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import time
-
-import pytest
-import mock
-from collections import defaultdict
-import socket
-
-from qtip.util import env
-from qtip.util.env import AnsibleEnvSetup
-
-
-@pytest.fixture(scope='session')
-def ansible_envsetup():
- return AnsibleEnvSetup()
-
-
-@pytest.fixture()
-def hostfile(tmpdir):
- fake_hostfile = tmpdir.join('hosts')
- fake_hostfile.write("[hosts]\n")
- fake_hostfile.write("10.20.0.3")
- return fake_hostfile
-
-
-@pytest.fixture()
-def private_key(tmpdir):
- fake_private_key = tmpdir.join('QtipKey')
- fake_private_key.write("fake keypair")
- return fake_private_key
-
-
-@pytest.fixture()
-def public_key(tmpdir):
- fake_public_key = tmpdir.join('QtipKey.pub')
- fake_public_key.write("fake public key")
- return fake_public_key
-
-
-def test_all_files_exist(tmpdir):
- exist_file = tmpdir.mkdir('qtip').join('hello.txt')
- exist_file.write("hello")
- non_exist_file = tmpdir.strpath + '/tmp.txt'
- assert env.all_files_exist() is False
- assert env.all_files_exist(str(exist_file))
- assert env.all_files_exist(non_exist_file) is False
- assert env.all_files_exist(str(exist_file), non_exist_file) is False
-
-
-def test_clean_file(tmpdir):
- exist_file = tmpdir.mkdir('qtip').join('hello.txt')
- exist_file.write("hello")
- non_exist_file = tmpdir.strpath + '/tmp.txt'
-
- assert env.clean_file() is False
- assert env.clean_file(str(exist_file))
- assert env.clean_file(non_exist_file)
-
-
-def test_init(ansible_envsetup):
- assert 'AnsibleEnvSetup' in str(type(ansible_envsetup))
- assert ansible_envsetup.keypair == defaultdict(str)
- assert ansible_envsetup.hostfile is None
- assert ansible_envsetup.host_ip_list == []
-
-
-def test_setup_exception(mocker, ansible_envsetup, hostfile):
- with mock.patch.object(AnsibleEnvSetup, 'check_hostfile', side_effect=RuntimeError()):
- mock_os = mocker.patch('sys.exit')
- ansible_envsetup.setup({'hostfile': str(hostfile)})
- assert mock_os.call_count == 1
-
-
-# TODO(zhihui_wu) Need find a smart way to write this pytest
-def test_setup(mocker, ansible_envsetup):
- mock_check_hostfile = \
- mocker.patch.object(AnsibleEnvSetup, 'check_hostfile')
- mock_generate_default_hostfile = \
- mocker.patch.object(AnsibleEnvSetup, 'generate_default_hostfile')
- mock_fetch_ip = \
- mocker.patch.object(AnsibleEnvSetup, 'fetch_host_ip_from_hostfile')
- mock_check_keypair = \
- mocker.patch.object(AnsibleEnvSetup, 'check_keypair')
- mock_generate_default_keypair = \
- mocker.patch.object(AnsibleEnvSetup, 'generate_default_keypair')
- mock_pass_keypair = \
- mocker.patch.object(AnsibleEnvSetup, 'pass_keypair_to_remote')
- mock_check_ssh = \
- mocker.patch.object(AnsibleEnvSetup, 'check_hosts_ssh_connectivity')
-
- ansible_envsetup.setup({'keypair': str(private_key),
- 'hostfile': str(hostfile)})
- mock_check_hostfile.assert_called_with(str(hostfile))
- mock_fetch_ip.assert_called_with()
- mock_check_keypair.assert_called_with(str(private_key))
- mock_pass_keypair.assert_called_with()
- mock_check_ssh.assert_called_with()
-
- ansible_envsetup.setup({'keypair': str(private_key)})
- mock_generate_default_hostfile.assert_called_with()
- mock_fetch_ip.assert_called_with()
- mock_check_keypair.assert_called_with(str(private_key))
- mock_pass_keypair.assert_called_with()
- mock_check_ssh.assert_called_with()
-
- ansible_envsetup.setup({'hostfile': str(hostfile)})
- mock_check_hostfile.assert_called_with(str(hostfile))
- mock_fetch_ip.assert_called_with()
- mock_generate_default_keypair.assert_called_with()
- mock_pass_keypair.assert_called_with()
- mock_check_ssh.assert_called_with()
-
- ansible_envsetup.setup()
- mock_generate_default_hostfile.assert_called_with()
- mock_fetch_ip.assert_called_with()
- mock_generate_default_keypair.assert_called_with()
- mock_pass_keypair.assert_called_with()
- mock_check_ssh.assert_called_with()
-
-
-def test_check_keypair(mocker, ansible_envsetup, private_key, public_key):
- with mocker.patch.object(env, 'all_files_exist', return_value=True):
- ansible_envsetup.check_keypair(str(private_key))
- assert ansible_envsetup.keypair['private'] == str(private_key)
- assert ansible_envsetup.keypair['public'] == str(public_key)
-
-
-def test_check_keypair_failed(mocker, ansible_envsetup):
- mocker.patch.object(env, 'all_files_exist', return_value=False)
- with pytest.raises(RuntimeError) as excinfo:
- ansible_envsetup.check_keypair(str(private_key))
- assert 'The keypairs you in the configuration file ' \
- 'is invalid or not existed.' == str(excinfo.value)
- assert ansible_envsetup.keypair['private'] == ''
- assert ansible_envsetup.keypair['public'] == ''
-
-
-@pytest.mark.parametrize("file_existence, expected", [
- (True, 0),
- (False, 1)
-])
-def test_generate_default_keypair(mocker, ansible_envsetup, file_existence, expected):
- mock_os = mocker.patch('os.system')
- mocker.patch.object(env, 'all_files_exist', return_value=file_existence)
- ansible_envsetup.generate_default_keypair()
- assert mock_os.call_count == expected
- assert ansible_envsetup.keypair['private'] == env.PRIVATE_KEY
- assert ansible_envsetup.keypair['public'] == env.PUBLIC_KEY
-
-
-@pytest.mark.parametrize("ips, expected", [
- (['10.20.0.3'], 1),
- (['10.20.0.3', '10.20.0.4'], 2)
-])
-def test_pass_keypair_to_remote_successful(mocker, ansible_envsetup, ips, expected):
- ansible_envsetup.host_ip_list = ips
- mock_pass_keypair = \
- mocker.patch.object(AnsibleEnvSetup, '_pass_keypair', return_value=True)
- ansible_envsetup.pass_keypair_to_remote()
- assert mock_pass_keypair.call_count == expected
-
-
-def test_pass_keypair_to_remote_failed(mocker, ansible_envsetup):
- ansible_envsetup.host_ip_list = ['10.20.0.3']
- mocker.patch.object(AnsibleEnvSetup, '_pass_keypair', return_value=False)
- with pytest.raises(RuntimeError) as excinfo:
- ansible_envsetup.pass_keypair_to_remote()
- assert "Failed on passing keypair to remote." in str(excinfo.value)
-
-
-def test_pass_keypair(monkeypatch, mocker, ansible_envsetup):
- monkeypatch.setattr(time, 'sleep', lambda s: None)
- mock_os = mocker.patch('os.system')
- ansible_envsetup._pass_keypair('10.20.0.3', str(private_key))
- assert mock_os.call_count == 2
-
-
-def test_pass_keypair_exception(ansible_envsetup):
- with mock.patch('os.system', side_effect=Exception()) as mock_os:
- result = ansible_envsetup._pass_keypair('10.20.0.3', str(private_key))
- assert result is False
- assert mock_os.call_count == 1
-
-
-def test_check_hostfile(mocker, ansible_envsetup, hostfile):
- ansible_envsetup.check_hostfile(str(hostfile))
- assert ansible_envsetup.hostfile == str(hostfile)
-
- with pytest.raises(RuntimeError) as excinfo:
- mocker.patch.object(env, 'all_files_exist', return_value=False)
- ansible_envsetup.check_hostfile(str(hostfile))
- assert str(excinfo.value) == 'The hostfile {0} is invalid or not ' \
- 'existed.'.format(str(hostfile))
-
-
-def test_default_hostfile_non_existed(mocker, ansible_envsetup):
- with mocker.patch.object(env, 'all_files_exist', return_value=False):
- mock_generate_hostfile_via_installer = \
- mocker.patch.object(AnsibleEnvSetup,
- '_generate_hostfile_via_installer')
- ansible_envsetup.generate_default_hostfile()
- mock_generate_hostfile_via_installer.assert_called_once_with()
-
-
-def test_default_hostfile_existed(mocker, ansible_envsetup):
- with mocker.patch.object(env, 'all_files_exist', return_value=True):
- mock_generate_hostfile_via_installer = \
- mocker.patch.object(AnsibleEnvSetup,
- '_generate_hostfile_via_installer')
- ansible_envsetup.generate_default_hostfile()
- mock_generate_hostfile_via_installer.assert_not_called()
-
-
-@pytest.mark.parametrize("test_input, expected", [
- (({}, KeyError), 'INSTALLER_TYPE'),
- (({'INSTALLER_TYPE': 'fuel'}, KeyError), 'INSTALLER_IP'),
- (({'INSTALLER_TYPE': 'fuel_1', 'INSTALLER_IP': '10.20.0.2'}, ValueError),
- 'fuel_1 is not supported'),
- (({'INSTALLER_TYPE': 'fuel', 'INSTALLER_IP': ''}, ValueError),
- 'The value of environment variable INSTALLER_IP is empty')
-])
-def test_generate_hostfile_via_installer_exception(monkeypatch, ansible_envsetup, test_input, expected):
- if test_input[0]:
- for key in test_input[0]:
- monkeypatch.setenv(key, test_input[0][key])
-
- with pytest.raises(test_input[1]) as excinfo:
- ansible_envsetup._generate_hostfile_via_installer()
- assert expected in str(excinfo.value)
-
-
-def test_generate_hostfile_via_installer(monkeypatch, mocker, ansible_envsetup):
- monkeypatch.setenv('INSTALLER_TYPE', 'fuel')
- monkeypatch.setenv('INSTALLER_IP', '10.20.0.2')
- mock_os = mocker.patch('os.system')
- ansible_envsetup._generate_hostfile_via_installer()
- assert mock_os.call_count == 1
- assert ansible_envsetup.hostfile == env.HOST_FILE
-
-
-def test_fetch_host_ip_from_hostfile(ansible_envsetup, hostfile):
- ansible_envsetup.hostfile = str(hostfile)
- ansible_envsetup.fetch_host_ip_from_hostfile()
- assert ansible_envsetup.host_ip_list == ['10.20.0.3']
-
-
-def test_fetch_host_ip_from_empty_hostfile(ansible_envsetup, tmpdir):
- empty_hostfile = tmpdir.join('empty_hostfile')
- empty_hostfile.write("")
- ansible_envsetup.hostfile = str(empty_hostfile)
- with pytest.raises(ValueError) as excinfo:
- ansible_envsetup.fetch_host_ip_from_hostfile()
- assert str(excinfo.value) == "The hostfile doesn't include host ip addresses."
-
-
-@pytest.mark.parametrize("ips, expected", [
- (['10.20.0.3'], 1),
- (['10.20.0.3', '10.20.0.4'], 2)
-])
-def test_check_hosts_ssh_connectivity(mocker, ansible_envsetup, ips, expected):
- ansible_envsetup.host_ip_list = ips
- mock_ssh_is_ok = \
- mocker.patch.object(AnsibleEnvSetup, '_ssh_is_ok', return_value=True)
- ansible_envsetup.check_hosts_ssh_connectivity()
- assert mock_ssh_is_ok.call_count == expected
-
-
-def test_check_hosts_ssh_connectivity_failed(mocker, ansible_envsetup):
- ansible_envsetup.host_ip_list = ['10.20.0.3']
- mocker.patch.object(AnsibleEnvSetup, '_ssh_is_ok', return_value=False)
- with pytest.raises(RuntimeError) as excinfo:
- ansible_envsetup.check_hosts_ssh_connectivity()
- assert "Failed on checking hosts ssh connectivity." == str(excinfo.value)
-
-
-@pytest.mark.parametrize("stderrinfo, expected", [
- ('', True),
- ('sorry', False)
-])
-def test_ssh_is_ok(mocker, ansible_envsetup, private_key, stderrinfo, expected):
- stderr = mock.MagicMock()
- stderr.readlines.return_value = stderrinfo
- mock_sshclient = mocker.patch('paramiko.SSHClient')
- test_ssh_client = mock_sshclient.return_value
- test_ssh_client.exec_command.return_value = ('', '', stderr)
- result = ansible_envsetup._ssh_is_ok('10.20.0.3', str(private_key))
- assert result == expected
- test_ssh_client.connect.assert_called_once_with(
- '10.20.0.3', key_filename=str(private_key))
- test_ssh_client.exec_command.assert_called_with('uname')
-
-
-def test_ssh_exception(monkeypatch, mocker, ansible_envsetup):
- monkeypatch.setattr(time, 'sleep', lambda s: None)
- mock_sshclient = mocker.patch('paramiko.SSHClient')
- test_ssh_client = mock_sshclient.return_value
- test_ssh_client.exec_command.side_effect = socket.error()
- result = ansible_envsetup._ssh_is_ok('10.20.0.3', str(private_key), attempts=1)
- assert result is False
Code Review / qtip.git / commitdiff