##############################################################################
FROM ubuntu:14.04
+MAINTAINER MatthewLi <matthew.lijun@huawei.com>
LABEL image=opnfv/bottlenecks
#new test suite required packages can be added here
RUN apt-get update && apt-get install -y \
+ libffi-dev \
+ libssl-dev \
+ libxml2-dev \
+ libxslt1-dev \
curl \
wget \
git \
sshpass \
python \
- python-glanceclient \
- python-heatclient \
- python-keystoneclient \
- python-neutronclient \
- python-novaclient
+ python-dev \
+ python-pip \
+ python-setuptools && \
+ easy_install -U setuptools
RUN apt-get -y autoremove && \
apt-get clean
RUN mkdir -p ${REPOS_DIR}
RUN git config --global http.sslVerify false
-# TODO fix code clone handler
-#RUN git clone https://gerrit.opnfv.org/gerrit/bottlenecks ${BOTTLENECKS_REPO_DIR}
-#RUN git clone https://gerrit.opnfv.org/gerrit/releng ${RELENG_REPO_DIR}
+RUN git clone https://gerrit.opnfv.org/gerrit/bottlenecks ${BOTTLENECKS_REPO_DIR}
+RUN git clone https://gerrit.opnfv.org/gerrit/releng ${RELENG_REPO_DIR}
+RUN easy_install pytz
+RUN pip install -r ${REPOS_DIR}/bottlenecks/requirements.txt
-suite_prefix_config: /home/opnfv/bottlenecks/testsuites/
+suite_prefix_config: /home/opnfv/bottlenecks/testsuites
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 Huawei Tech. and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+appdirs==1.4.0
+Babel==2.2.0
+backport-ipaddress==0.1
+cliff==2.0.0
+cmd2==0.6.8
+coverage==4.1b2
+debtcollector==1.3.0
+ecdsa==0.13
+extras==0.0.3
+fixtures==1.4.0
+flake8==2.5.4
+funcsigs==0.4
+functools32==3.2.3.post2
+futures==3.0.5
+iso8601==0.1.11
+Jinja2==2.8
+jsonpatch==1.13
+jsonpointer==1.10
+jsonschema==2.5.1
+keystoneauth1==2.3.0
+linecache2==1.0.0
+lxml==3.5.0
+MarkupSafe==0.23
+mccabe==0.4.0
+mock==1.3.0
+monotonic==1.0
+msgpack-python==0.4.7
+netaddr==0.7.18
+netifaces==0.10.4
+nose==1.3.7
+openstacksdk==0.8.1
+os-client-config==1.16.0
+oslo.config==3.9.0
+oslo.i18n==3.4.0
+oslo.serialization==2.4.0
+oslo.utils==3.7.0
+paramiko==1.16.0
+pbr==1.8.1
+pep8==1.7.0
+positional==1.0.1
+prettytable==0.7.2
+pycrypto==2.6.1
+pyflakes==1.0.0
+pyparsing==2.1.0
+pyrsistent==0.11.12
+python-cinderclient==1.6.0
+python-glanceclient==2.0.0
+python-heatclient==1.0.0
+python-keystoneclient==2.3.1
+python-mimeparse==1.5.1
+python-neutronclient==4.1.1
+python-novaclient==3.3.0
+python-openstackclient==2.2.0
+python-subunit==1.2.0
+python-swiftclient==3.0.0
+pytz==2015.7
+PyYAML==3.11
+requests==2.9.1
+requestsexceptions==1.1.3
+scp==0.10.2
+simplejson==3.8.2
+six==1.10.0
+stevedore==1.12.0
+testrepository==0.0.20
+testtools==2.0.0
+traceback2==1.4.0
+unicodecsv==0.14.1
+unittest2==1.1.0
+warlock==1.2.0
+wrapt==1.10.6
check_suite="$1"
case $check_suite in
"-rubbos")
- $SUITE_PREFIX=$SUITE_PREFIX_CONFIG/rubbos/testcase_cfg
+ SUITE_PREFIX=$SUITE_PREFIX_CONFIG/rubbos/testcase_cfg
;;
"-vstf")
- $SUITE_PREFIX=$SUITE_PREFIX_CONFIG/vstf/testcase_cfg
+ SUITE_PREFIX=$SUITE_PREFIX_CONFIG/vstf/testcase_cfg
;;
esac
all_testcases_insuite=`cat /tmp/all_testcases.yaml | awk -F '/' '{print $NF}' | awk -F '.' '{print $1}'`
all_testcases=(${all_testcases_insuite})
- if ["${TEST_CASE}" != ""]; then
+ if [ "${TEST_CASE}" != "" ]; then
testcase_exec=(${TEST_CASE// /})
for i in "${testcase_exec[@]}"; do
- if [[ " ${all_test_cases[*]} " != *" $i "* ]]; then
+ if [[ " ${all_testcases[*]} " != *" $i "* ]]; then
error "unknown test case: $i. available test cases are: ${all_test_cases[@]}"
fi
done
if [[ -f $test_file ]]; then
testcases=($(cat $test_file))
else
- error("no rubbos test suite file ")
+ error "no rubbos test suite file"
fi
for i in "${testcases[@]}"; do
#check if the testcase is legal or not
#adjust config parameters, different test suite has different methods, take rubbos as an example
#run test case, different test suite has different methods
file={$BASEDIR}/testsuites/rubbos/testcase_cfg/{$i}.yaml
- python ${BOTTLENECK_TOP_DIR}/testsuites/rubbos/run_rubbos.py -c $file
+ python /home/opnfv/bottlenecks/testsuites/rubbos/run_rubbos.py -c $file
done
;;
"vstf")
if [[ -f $test_file ]]; then
testcases=($(cat $test_file))
else
- error("no vstf test suite file ")
+ error "no vstf test suite file "
fi
for i in "${testcases[@]}"; do
#check if the testcase is legal or not
backup => false,
}
- # bench/rubbos.properties.template
- file {'${rubbos_home}/bench/rubbos.properties.template':
+ # Client/rubbos.properties.template
+ file {'${rubbos_home}/Client/rubbos.properties.template':
ensure => absent,
- path => "${rubbos_home}/bench/rubbos.properties.template",
+ path => "${rubbos_home}/Client/rubbos.properties.template",
show_diff => false,
backup => false,
}
backup => false,
}
- # rubbos.properties.template
- file {'${rubbos_home}/bench/rubbos.properties.template':
+ # Client/rubbos.properties.template
+ file {'${rubbos_home}/Client/rubbos.properties.template':
ensure => file,
- path => "${rubbos_home}/bench/rubbos.properties.template",
+ path => "${rubbos_home}/Client/rubbos.properties.template",
source => "puppet:///modules/rubbos_client/rubbos.properties.template",
show_diff => false,
backup => false,
+ require => Exec['tar xvzf ${rubbos_home}/Client.tar.gz'],
}
- # executable scripts
+ # bench/executable scripts
file {'${rubbos_home}/bench/run_emulator.sh':
ensure => file,
path => "${rubbos_home}/bench/run_emulator.sh",
File['${rubbos_home}/Makefile'],
File['${rubbos_home}/config.mk'],
Exec['tar xvzf ${rubbos_home}/Client.tar.gz']],
- #File['${rubbos_home}/Client']],
}
exec {'ant jar':
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+
+
import os
import argparse
import time
-import heatclient
-import keystoneclient
-import glanceclient
-import novaclient
+import logging
+import urllib2
+import shutil
+from heatclient.client import Client as HeatClient
+from keystoneclient.v2_0.client import Client as KeystoneClient
+from glanceclient.v2.client import Client as GlanceClient
+from novaclient.client import Client as NovaClient
#------------------------------------------------------
# parser for configuration files in each test case
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--conf",
help="configuration files for the testcase, in yaml format",
- default="rubbos_1-1-1.yaml")
+ default="/tmp/rubbos_1-1-1.yaml")
args = parser.parse_args()
#--------------------------------------------------
# logging configuration
#--------------------------------------------------
-logger = logging.getLogger('run_rubbos')
-logger.setLevel(logging.DEBUG)
+logger = logging.getLogger(__name__)
+
def _get_keystone_client():
- keystone_client = keystoneclient.v2_0.client.Client(
+ keystone_client = KeystoneClient(
auth_url=os.environ.get('OS_AUTH_URL'),
username=os.environ.get('OS_USERNAME'),
password=os.environ.get('OS_PASSWORD'),
def _get_heat_client():
keystone = _get_keystone_client()
heat_endpoint = keystone.service_catalog.url_for(service_type='orchestration')
- heat_client = heatclient.client.Client('1', endpoint=heat_endpoint, token=keystone.auth_token)
-
+ heat_client = HeatClient('1', endpoint=heat_endpoint, token=keystone.auth_token)
return heat_client
def _get_glance_client():
keystone = _get_keystone_client()
- glance_endpoint = keystone.service_catalog.url_for(service_type='image',
- endpoint_type='publicURL')
- return glanceclient.v2.client.Client(glance_endpoint, token=keystone.auth_token)
+ glance_endpoint = keystone.service_catalog.url_for(service_type='image', endpoint_type='publicURL')
+ return GlanceClient(glance_endpoint, token=keystone.auth_token)
def _get_nova_client():
- nova_client = novaclient.client.Client("2", auth_url=os.environ.get('OS_AUTH_URL'),
- username=os.environ.get('OS_USERNAME'),
- password=os.environ.get('OS_PASSWORD'),
- tenant_name=os.environ.get('OS_TENANT_NAME'),
- region_name=os.environ.get('OS_REGION_NAME'),
- cacert=os.environ.get('OS_CACERT'))
+ nova_client = NovaClient("2", os.environ.get('OS_USERNAME'),
+ os.environ.get('OS_PASSWORD'),
+ os.environ.get('OS_TENANT_NAME'),
+ os.environ.get('OS_AUTH_URL'))
return nova_client
-def download_url(url, dest_path):
- """
- Download a file to a destination path given a URL
- """
- name = url.rsplit('/')[-1]
- dest = dest_path + "/" + name
+def _download_url(src_url, dest_dir):
+ ''' Download a file to a destination path given a URL'''
+ file_name = src_url.rsplit('/')[-1]
+ dest = dest_dir + "/" + file_name
try:
- response = urllib2.urlopen(url)
+ response = urllib2.urlopen(src_url)
except (urllib2.HTTPError, urllib2.URLError):
- return False
+ return None
with open(dest, 'wb') as f:
shutil.copyfileobj(response, f)
- return True
+ return file_name
+
+
+def rubbos_stack_satisfy(name="bottlenecks_rubbos_stack", status="CREATE_COMPLETE"):
+ heat = _get_heat_client()
+ for stack in heat.stacks.list():
+ if status == None and stack.stack_name == name:
+ # Found target stack
+ print "Found stack, name=" + stack.stack_name
+ return True
+ elif stack.stack_name == name and stack.stack_status==status:
+ print "Found stack, name=" + stack.stack_name + ", status=" + stack.stack_status
+ return True
+ return False
def rubbos_env_prepare(template=None):
- """ Prepare for rubbos running env
- """
- #logger.info("generate heat template for the testcase based on template '%s'." % template)
- pass
+ print "========== Prepare rubbos environment =========="
+ logger.info("Generate heat template for the testcase based on template '%s'." % template)
def rubbos_env_cleanup():
+ print "========== Cleanup rubbos environment =========="
glance = _get_glance_client()
heat = _get_heat_client()
nova = _get_nova_client()
- for stack in self.heat.stacks.list():
- heat.stacks.delete(stack.id)
+ for image in glance.images.list():
+ if image.name.find("bottlenecks_rubbos") >= 0:
+ print "Delete image, id:" + str(image.id) + ", name:" + str(image.name)
+ glance.images.delete(image.id)
+
+ for keypair in nova.keypairs.list():
+ if keypair.name.find("bottlenecks_rubbos") >= 0:
+ print "Delete keypair, id:" + keypair.id + ", name:" + keypair.name
+ nova.keypairs.delete(keypair.id)
- for image in self.glance.images.list():
- glance.images.delete(image.id)
+ for flavor in nova.flavors.list():
+ if flavor.name.find("bottlenecks_rubbos") >= 0:
+ print "Delete flavor, id:" + str(flavor.id) + ", name:" + flavor.name
+ nova.flavors.delete(flavor.id)
- for keypair in self.nova.keypairs.list():
- nova.keypairs.delete(keypair.id)
+ for stack in heat.stacks.list():
+ if stack.stack_name.find("bottlenecks_rubbos") >= 0:
+ print "Delete stack, id: " + str(stack.id) + ", name:" + str(stack.stack_name
+ heat.stacks.delete(stack.id)
- for flavor in self.nova.flavors.list():
- nova.flavors.delete(flavor.id)
+ timeInProgress = 0
+ while rubbos_stack_satisfy(name="bottlenecks_rubbos_stack", status=None) and timeInProgress < 60:
+ time.sleep(5)
+ timeInProgress = timeInProgress + 5
- logger.info("openstack env cleanup")
+ if rubbos_stack_satisfy(name="bottlenecks_rubbos_stack", status=None) == True:
+ print "Failed to clean the stack"
+ return False
+ else:
+ return True
-def rubbos_load_image(name=None):
- file_url = '/tmp'
- download_url(image_url,file_url)
+def rubbos_create_images(src_url=None, image_name="bottlenecks_rubbos_image"):
+ print "========== Create rubbos image in OS =========="
+ dest_dir = '/tmp'
+ file_name = _download_url(src_url, dest_dir)
+ #file_name = "bottlenecks-trusty-server.img"
+ if file_name == None:
+ return False
glance = _get_glance_client()
- image_args = {'name': name,
- 'disk-format': 'qcow2',
- 'container_format': 'bare',
- 'file': '/tmp/bottlenecks-trusty-server.img'}
- image = glance.images.create(**image_args)
- if not (image.id):
- logger.error("failed to upload rubbos image to openstack")
- exit(-1)
+ imagefile = dest_dir + "/" + file_name
+ image = glance.images.create(name=image_name, disk_format="qcow2", container_format="bare")
+ with open(imagefile) as fimage:
+ glance.images.upload(image.id, fimage)
+
+ timeInQueue = 0
+ img_status = image.status
+ while img_status == "queued" and timeInQueue < 30:
+ print " image's status: " + img_status
+ time.sleep(1)
+ timeInQueue = timeInQueue + 1
+ img_status = glance.images.get(image.id).status
+
+ print "After %d seconds, the image's status is [%s]" %(timeInQueue, img_status)
+ return True if img_status == "active" else False
+
+def rubbos_create_keypairs(key_path, name="bottlenecks_rubbos_keypair"):
+ print "========== Add rubbos keypairs in OS =========="
+ nova = _get_nova_client()
+ with open(key_path) as pkey:
+ nova.keypairs.create(name=name, public_key=pkey.read())
+
+def rubbos_create_flavors(name="bottlenecks_rubbos_flavor", ram=4096, vcpus=2, disk=10):
+ print "========== Create rubbos flavors in OS =========="
+ nova = _get_nova_client()
+ nova.flavors.create(name=name, ram=ram, vcpus=vcpus, disk=disk)
-def rubbos_create_instance(template_file=None, rubbos_parameters=None):
+def rubbos_create_instance(template_file, rubbos_parameters=None, stack_name="bottlenecks_rubbos_stack"):
+ print "========== Create rubbos instances =========="
heat = _get_heat_client()
- template = open(template_file, 'r')
- rubbos_stack = heat.stacks.create(stack_name='rubbos', template=template.read(), parameters=rubbos_parameters)
- uid = rubbos_stack['stack']['id']
-def rubbos_stack_check(stack_name=None):
- for stack in heat.stacks.list():
- if stack.stack_name == stack_name:
- return stack.stack_status
- return 'NOT_FOUND'
+ with open(template_file) as template:
+ stack = heat.stacks.create(stack_name=stack_name, template=template.read(), parameters=rubbos_parameters)
+
+ stack_id = stack['stack']['id']
+ stack_status = heat.stacks.get(stack_id).stack_status
+
+ print "Created stack, id=" + stack_id + ", status=" + stack_status
+
+ timeInProgress= 0
+ while stack_status == "CREATE_IN_PROGRESS" and timeInProgress < 150:
+ print " stack's status: %s, after %d seconds" %(stack_status, timeInProgress)
+ time.sleep(5)
+ timeInProgress = timeInProgress + 5
+ stack_status = heat.stacks.get(stack_id).stack_status
+
+ print "After %d seconds, the stack's status is [%s]" %(timeInProgress, stack_status)
+ return True if stack_status == "CREATE_COMPLETE" else False
def rubbos_run():
pass
def main():
global Heat_template
+ global Bottlenecks_repo_dir
global image_url
-
+ Bottlenecks_repo_dir = "/tmp/opnfvrepo/bottlenecks"
+ #Bottlenecks_repo_dir = "/root/wyg/bottlenecks" # Test dir in local env
+
image_url = 'http://artifacts.opnfv.org/bottlenecks/rubbos/bottlenecks-trusty-server.img'
if not (args.conf):
- logger.error("configuration files are not set for testcase")
+ logger.error("Configuration files are not set for testcase")
exit(-1)
else:
Heat_template = args.conf
- parameters={'image': 'rubbos',
- 'key_name': 'rubbos-key',
- 'public_net': 'net04_ext'}
+ parameters={'image': 'bottlenecks_rubbos_image',
+ 'key_name': 'bottlenecks_rubbos_keypair',
+ 'flavor': 'm1.small',
+ 'public_net': 'ext-net'}
+
+ print "Heat_template_file: " + Heat_template
+ print "parameters:\n" + str(parameters)
+
+ image_created = False
+ stack_created = False
rubbos_env_prepare(Heat_template)
rubbos_env_cleanup()
- rubbos_load_image(name='rubbos')
- rubbos_create_instance(template_file=Heat_template, rubbos_paramters=parameters)
- time.sleep(400)
- rubbos_stack_check(stack_name='rubbos')
+
+ image_created = rubbos_create_images(image_url)
+ keyPath = Bottlenecks_repo_dir + "/utils/infra_setup/bottlenecks_key/bottlenecks_key.pub"
+ rubbos_create_keypairs(key_path=keyPath)
+ rubbos_create_flavors()
+
+ if image_created == True:
+ stack_created = rubbos_create_instances(template_file=Heat_template, rubbos_parameters=parameters, stack_name="bottlenecks_rubbos_stack")
+ else:
+ print "Cannot create instances, as Failed to create image(s)."
+ exit (-1)
+
rubbos_run()
rubbos_env_cleanup()
+++ /dev/null
-#!/bin/bash
-###############################################################################
-# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-usage="Script to run the tests in rubbos.
-
-usage:
- bash $(basename "$0") [-h|--help] [-s <test suite>]
-
-where:
- -h|--help show the help text
- -t|--test run specifif test case scenario
- <test case> examples:
- rubbos_1-1-1, rubbos_1-2-1
-
-examples:
- $(basename "$0")
- $(basename "$0") -t rubbos_1-1-1"
-
-while [[ $# > 0 ]]
- do
- key="$1"
- case $key in
- -h|--help)
- echo "$usage"
- exit 0
- shift
- ;;
- -t|--test)
- CASE="$2"
- shift
- ;;
- *)
- echo "unkown option $1 $2"
- exit 1
- ;;
- esac
- shift
-done
-
-BASEDIR=`dirname $0`
-
-#has been checked in upper layer run_tests.sh
-#run tests
-if [ "${CASE}" != "" ]; then
- case_exec=(${CASE//,/})
- for case_exe in "${case_exec[@]}"; do
- client_num=4
- tomcat_num=$(echo "$case_exe"| awk -F '-' '{print $2}')
- mysql_num=$(echo "$case_exe"| awk -F '-' '{print $3')
-
- hosts=(rubbos_control rubbos_benchmark rubbos_httpd)
- for((i=1; i <= client_num; i++)); do
- hosts=(${hosts[*]} client$i)
- done
- for((i=1; i <= tomcat_num; i++)); do
- hosts=(${hosts[*]} tomcat$i)
- done
- for((i=1; i <= mysql_num; i++)); do
- hosts=(${hosts[*]} mysql$i)
- done
- bash $BOTTLENECKS_TOP_DIR/utils/infra_setup/heat_template/rubbos_heat_template/HOT_create_instance.sh
- done
-fi
import subprocess as subp
import sys
-from collector import Collector
+from rubbos_collector import RubbosCollector
from uploader import Uploader
+def printUsage():
+ print "Usage: python process_data.py required_params(**) optional_params([])"
+ print " ** -i|--input input_data_dir"
+ print " ** -s|--suite suite_name"
+ print " ** -c|--conf conf_file"
+ print " [] -o|--output output_file"
+ print " [] -u|--upload yes|no"
-#process data
-if len(sys.argv)!=3:
- print "Wrong arguments, please input 2 parameters, 1st as raw data path; "\
- "2nd as config yaml!!"
- exit (1)
-data_home = sys.argv[1]
-conf = sys.argv[2]
+def process(input_dir, suite_name):
+ result = dict()
+ if suite_name == "rubbos":
+ result = RubbosCollector().collect_data(input_dir)
+ return result
-#1collect result
-result = Collector().collect_data(data_home)
-print "Result collected:\n%s" % result
+def writeResult(output_file, result):
+ f = open(output_file, "w")
+ if isinstance(result, list):
+ for elem in result:
+ f.write(str(elem) + "\n")
+ f.close()
-#2upload result
-Uploader(conf).upload_result("rubbos", result)
+def uploadResult(conf, suite_name, result):
+ Uploader(conf).upload_result(suite_name, result)
+def main():
+ if len(sys.argv) < 7 or len(sys.argv) % 2 == 0:
+ printUsage()
+ exit (1)
+ i = 1
+ params = dict()
+ while (i < len(sys.argv)):
+ print sys.argv[i]
+ if sys.argv[i]=="-i" or sys.argv[i]=="--input":
+ params["input"] = sys.argv[i+1]
+ if sys.argv[i]=="-s" or sys.argv[i]=="--suite":
+ params["suite"] = sys.argv[i+1]
+ if sys.argv[i]=="-c" or sys.argv[i]=="--conf":
+ params["conf"] = sys.argv[i+1]
+ if sys.argv[i]=="-o" or sys.argv[i]=="--output":
+ params["output"] = sys.argv[i+1]
+ if sys.argv[i]=="-u" or sys.argv[i]=="--upload":
+ params["upload"] = sys.argv[i+1]
+ i = i+2
+ if not(params.has_key("input") and params.has_key("suite") and params.has_key("conf")):
+ print "Lack some required parameters."
+ exit (1)
+
+ result = process(params["input"], params["suite"])
+ print "Results:"
+ for elem in result:
+ print elem
+
+ if params.has_key("output"):
+ writeResult(params["output"],result)
+
+ if params.has_key("upload") and params["upload"].lower()=="yes":
+ uploadResult(params["conf"], params["suite"], result)
+
+if __name__=="__main__":
+ main()
out,err = subp.Popen(cmd, stdout=subp.PIPE, shell=True).communicate()
return out.strip()
-
def get_onetime_data(dir_name):
cmd = "grep -in 'remote client nodes' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name
client_node_num = int(exec_shell(cmd))
cmd = "grep -n 'Number of clients' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name
each_client_num = int(exec_shell(cmd))
total_client = (client_node_num+1) * each_client_num
- cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "<B>" \'FNR%%4==0 {printf "%%s\\n", $3 }\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
+
+ cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "<B>" \'{if (FNR%%2==0 && FNR%%4!=0) {printf "%%s\\n", $3}}\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
throughput = int(exec_shell(cmd))
- return total_client, throughput
+ cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "<B>" \'{if (FNR==4) {printf "%%s\\n", $4}}\'|awk -F "</B>" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
+ request = int(exec_shell(cmd))
+
+ cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "<B>" \'{if (FNR==4) {printf "%%s\\n", $5}}\'|awk -F "</B>" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
+ error_request = int(exec_shell(cmd))
+ return total_client, throughput, request, error_request
-class Collector(object):
+class RubbosCollector(object):
def __init__(self):
pass
-
def collect_data(self, data_home):
cmd = 'ls -l %s |grep ^d|awk \'{print $9}\'' % data_home
result = []
for subdir in exec_shell(cmd).split('\n'):
- total_client, throughput = get_onetime_data(data_home+'/'+subdir)
- result.append({'client':total_client, 'throughput':throughput})
- result.sort(key=lambda x:x['client'])
+ total_client, throughput, request, error_request = get_onetime_data(data_home+'/'+subdir)
+ result.append({'client':total_client, 'throughput':throughput, 'request':request, 'error_request':error_request})
+ result.sort(key=lambda x:x['client'])
return result;
-
--- /dev/null
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+##############################################
+# Usage: ./deploy.sh paras_conf outout_dir
+##############################################
+SCRIPT_DIR=$(cd "$(dirname "$0")"; pwd)
+tool_dir=${0%/*}
+
+function print_usage()
+{
+ echo "Usage: ./deploy.sh paras_conf output_dir"
+}
+
+## sanity check
+if [ ! -f $tool_dir/mac_generator.sh ] || \
+ [ ! -f $tool_dir/libvirt_template.xml ] || \
+ [ ! -f $tool_dir/meta-data.template ] || \
+ [ ! -f $tool_dir/user-data.template ] || \
+ [ ! -f $tool_dir/p-master-user-data.template ] || \
+ [ ! -f $tool_dir/p-agent-user-data.template ]; then
+ echo "Lack some necessary files for this tool!"
+ echo "deploy.sh"
+ echo "mac_generator.sh"
+ echo "1 xml, and 4 template"
+ exit 1
+fi
+
+## Check input
+if [ $# != 2 ]; then
+ print_usage
+ exit 1
+fi
+if [ ! -f $1 ]; then
+ echo "Cannot find file: "$1
+ exit 1
+fi
+if [ -d $2 ]; then
+ echo "Ouput dir $2 exist!"
+ exit 1
+fi
+
+## Assign parameters
+host_names=""
+puppet_enable="true"
+master_host=""
+vm_mem=
+vm_cpu_cores=
+image_url=
+image_name=
+ipaddr_start=
+trusted_ssh_pub_keys_file=
+while read line
+do
+ line=(${line//=/ })
+ case ${line[0]} in
+ "host_names" )
+ host_names=${line[1]}
+ ;;
+ "puppet_enable" )
+ puppet_enable=${line[1]}
+ ;;
+ "master_host" )
+ master_host=${line[1]}
+ ;;
+ "vm_mem" )
+ vm_mem=${line[1]}
+ ;;
+ "vm_cpu_cores" )
+ vm_cpu_cores=${line[1]}
+ ;;
+ "image_url" )
+ image_url=${line[1]}
+ ;;
+ "image_name" )
+ image_name=${line[1]}
+ ;;
+ "ipaddr_start" )
+ ipaddr_start=${line[1]}
+ ;;
+ "trusted_ssh_pub_keys_file" )
+ trusted_ssh_pub_keys_file=${line[1]}
+ ;;
+ esac
+done < $1
+
+echo "puppet_enable="$puppet_enable
+
+# Check parameters in conf file
+if [ $puppet_enable == "true" ] ; then
+ if [ ${#master_host} == 0 ];then
+ echo "Should specify master_host!"
+ exit 1
+ else
+ result=$(echo ${host_names} | grep "${master_host}")
+ if [ ${result} == "" ]; then
+ echo "Specified master_host is invalid!"
+ exit 1
+ fi
+ fi
+fi
+
+# Define and Prepare needed data
+mac_arr=
+hostname_arr=(${host_names//,/ })
+virt_num=${#hostname_arr[@]}
+ip_arr=()
+replaced_hosts=""
+replaced_ssh_keys=""
+output_dir=$2
+work_dir=
+host_vm_dir=
+cache_dir=
+function init(){
+ # Generate mac address
+ local mac_generator=${tool_dir}/mac_generator.sh
+ chmod +x $mac_generator
+ mac_str=$($mac_generator $virt_num)
+ mac_arr=($mac_str)
+
+ # Generate hosts info
+ local ip=""
+ i=0
+ for host in "${hostname_arr[@]}"; do
+ ip=${ipaddr_prefix}$((i+$ipaddr_idx))
+ ip_arr+=($ip)
+ # Note the format, especially the space
+ replaced_hosts=${replaced_hosts}" "${ip}" "${host}"\n"
+ let i=i+1
+ done
+
+ # Generate ssh public keys
+ echo "## trusted_ssh_pub_keys_file --> "${trusted_ssh_pub_keys_file}
+ if [ ${trusted_ssh_pub_keys_file} != "" ] && [ -f ${trusted_ssh_pub_keys_file} ]; then
+ while read line
+ do
+ # Note the format, especially space
+ replaced_ssh_keys=${replaced_ssh_keys}" - "${line}"\n"
+ done < ${trusted_ssh_pub_keys_file}
+ # delete last "\n" in replaced_ssh_keysi
+ replaced_ssh_keys=${replaced_ssh_keys%\\n}
+ fi
+
+ # Prepare needed folder and files
+ if [ ${output_dir:0-1} == "/" ]; then
+ output_dir=${output_dir%/*}
+ fi
+ output_file=$output_dir/hosts.info
+ mkdir -p $output_dir
+ touch $output_file
+ echo "## Output host_info file --> "$output_file
+
+ work_dir=$output_dir/work
+ host_vm_dir=$work_dir/vm
+ cache_dir=$work_dir/cache
+ mkdir -p $work_dir
+ mkdir -p $host_vm_dir
+ mkdir -p $cache_dir
+
+ # Cache img file
+ echo "## Cache img file"
+ curl --connect-timeout 10 -o ${cache_dir}/$image_name $image_url
+}
+
+# Bring up instances/vms
+sub_ip_arr=(${ipaddr_start//./ })
+ipaddr_prefix=${sub_ip_arr[0]}"."${sub_ip_arr[1]}"."${sub_ip_arr[2]}"."
+ipaddr_idx=${sub_ip_arr[3]}
+function bring_up() {
+ i=0
+ while (($i < $virt_num))
+ do
+ echo "Bring up a vm, hostname: ${hostname_arr[$i]}, ip: ${ip_arr[$i]}, mac: ${mac_arr[$i]}"
+ vm_dir=$host_vm_dir/${hostname_arr[$i]}
+ mkdir -p $vm_dir
+ cp ${cache_dir}/$image_name $vm_dir
+
+ sed -e "s/REPLACE_IPADDR/${ip_arr[$i]}/g" \
+ -e "s/REPLACE_GATEWAY/${ipaddr_prefix}1/g" \
+ -e "s/REPLACE_HOSTNAME/${hostname_arr[$i]}/g" \
+ ${tool_dir}/meta-data.template > ${cache_dir}/meta-data
+
+ if [ ${puppet_enable} == "true" ]; then
+ # Use puppet user data
+ echo "hostname: "${hostname_arr[$i]}
+ if [ ${hostname_arr[$i]} == ${master_host} ]; then
+ cp ${tool_dir}/p-master-user-data.template ${cache_dir}/user-data.template
+ else
+ cp ${tool_dir}/p-agent-user-data.template ${cache_dir}/user-data.templatate
+ fi
+ sed -e "s#REPLACED_TRUSTED_PUB_SSH_KEYS#${replaced_ssh_keys}#g" \
+ -e "s#REPLACED_HOSTS_INFO#${replaced_hosts}#g" \
+ -e "s/REPLACED_PUPPET_MASTER_SERVER/${master_host}/g" \
+ ${cache_dir}/user-data.template > ${cache_dir}/user-data
+ else
+ # Use common user data
+ echo "## Use common user-data.template"
+ cp ${tool_dir}/user-data.template ${cache_dir}/user-data
+ fi
+
+ genisoimage -output seed.iso -volid cidata -joliet -rock ${cache_dir}/user-data ${cache_dir}/meta-data
+ mv seed.iso ${vm_dir}/
+ # Create vm xml
+ sed -e "s/REPLACE_MEM/$vm_mem/g" \
+ -e "s/REPLACE_CPU/$vm_cpu_cores/g" \
+ -e "s/REPLACE_NAME/${hostname_arr[$i]}/g" \
+ -e "s#REPLACE_IMAGE#$vm_dir/disk.img#g" \
+ -e "s#REPLACE_SEED_IMAGE#$vm_dir/seed.iso#g" \
+ -e "s/REPLACE_MAC_ADDR/${mac_arr[$i]}/g" \
+ ${tool_dir}/libvirt_template.xml > ${vm_dir}/libvirt.xml
+
+ echo "${ip_arr[$i]} ${hostname_arr[$i]}" >> $output_file
+
+ echo "Will define xml from:"${vm_dir}"/libvirt.xml"
+ echo "start: "${hostname_arr[$i]}
+ sudo virsh define ${vm_dir}/libvirt.xml
+ sudo virsh start ${hostname_arr[$i]}
+ let i=i+1
+ rm -rf ${cache_dir}/meta-data ${cache_dir}/user-data
+ done
+}
+
+function clean(){
+ rm -rf ${cache_dir}
+}
+
+init
+bring_up
+clean
#!/bin/bash
##############################################################################
-# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-# this is a modified copy from compass4nfv/deploy/mac_generator.sh
+## This script is used in 'create_libvirt_vm.sh'
function mac_address_part() {
hex_number=$(printf '%02x' $RANDOM)
number_length=${#hex_number}
done
echo ${machines}
-
--- /dev/null
+#cloud-config
+
+users:
+ - name: ubuntu
+ ssh-authorized-keys:
+REPLACED_TRUSTED_PUB_SSH_KEYS
+ - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwmx8/86Vc1yqqplMPkm/w6wB2r5Yl6CjON85Ko5SzPEpnysJwO5ygbMClQTbzCUztIpFeO8D6xdWlhPmhMEJIWtRh+Dx0swB+ByMraGTP92xxKo2ukWzYI6eu25LmJX+a1lmVPmewQq+6dPzIwgCZvwD8UVe5L+d0EP2rb3iniQVoBzksUmhL3N6jp2Lgpscfi1yXFgjFdLpDsCcFcM+Q6wQUC2+qGBYa5RxqercWzISSbvAsYwC7UepmjFpcj3W/WX2QcOCDIlAzeURvC8PbOdzCtgeMhuhH2499BcIUSzPMpjzvpRiHr5SeF/zrPeW+2Ei1MfHi1qrBCxS8JBqt bottlenecks@bottlenecks.opnfv.org
+ sudo: ['ALL=(ALL) NOPASSWD:ALL']
+ groups: sudo
+ shell: /bin/bash
+
+runcmd:
+ - restart ssh
+ - wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb
+ - sudo dpkg -i puppetlabs-release-trusty.deb
+ - sudo apt-get update
+ - sudo apt-get -y install puppet
+ - sed -i '/^factpath/a basemodulepath=\/etc\/puppet\/modules' /etc/puppet/puppet.conf
+ - sed -i '/^factpath/a server=REPLACED_PUPPET_MASTER_SERVER' /etc/puppet/puppet.conf
+ - sed -i '/^factpath/a runinterval=1d' /etc/puppet/puppet.conf
+ - sed -i '/^templatedir/d' /etc/puppet/puppet.conf
+ - sed -i 's/START=no/START=yes/g' /etc/default/puppet
+
+write_files:
+ - path: /etc/hosts
+ content: |
+ 127.0.0.1 localhost
+
+ # The following lines are desirable for IPv6 capable hosts
+ ::1 ip6-localhost ip6-loopback
+ fe00::0 ip6-localnet
+ ff00::0 ip6-mcastprefix
+ ff02::1 ip6-allnodes
+ ff02::2 ip6-allrouters
+ ff02::3 ip6-allhosts
+
+ # hosts info for all vms
+REPLACED_HOSTS_INFO
+
+final_message: "The system with puppet is finally up, after $UPTIME seconds."
--- /dev/null
+#cloud-config
+
+users:
+ - name: ubuntu
+ ssh-authorized-keys:
+REPLACED_TRUSTED_PUB_SSH_KEYS
+ - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwmx8/86Vc1yqqplMPkm/w6wB2r5Yl6CjON85Ko5SzPEpnysJwO5ygbMClQTbzCUztIpFeO8D6xdWlhPmhMEJIWtRh+Dx0swB+ByMraGTP92xxKo2ukWzYI6eu25LmJX+a1lmVPmewQq+6dPzIwgCZvwD8UVe5L+d0EP2rb3iniQVoBzksUmhL3N6jp2Lgpscfi1yXFgjFdLpDsCcFcM+Q6wQUC2+qGBYa5RxqercWzISSbvAsYwC7UepmjFpcj3W/WX2QcOCDIlAzeURvC8PbOdzCtgeMhuhH2499BcIUSzPMpjzvpRiHr5SeF/zrPeW+2Ei1MfHi1qrBCxS8JBqt bottlenecks@bottlenecks.opnfv.org
+ sudo: ['ALL=(ALL) NOPASSWD:ALL']
+ groups: sudo
+ shell: /bin/bash
+
+runcmd:
+ - restart ssh
+ - wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb
+ - sudo dpkg -i puppetlabs-release-trusty.deb
+ - sudo apt-get update
+ - sudo apt-get -y install puppetserver
+ - sed -i '/^factpath/a basemodulepath=\/etc\/puppet\/modules' /etc/puppet/puppet.conf
+ - sed -i '/^factpath/a server=REPLACED_PUPPET_MASTER_SERVER' /etc/puppet/puppet.conf
+ - sed -i '/^factpath/a runinterval=1d' /etc/puppet/puppet.conf
+ - sed -i '/^templatedir/d' /etc/puppet/puppet.conf
+
+write_files:
+ - path: /etc/hosts
+ content: |
+ 127.0.0.1 localhost
+
+ # The following lines are desirable for IPv6 capable hosts
+ ::1 ip6-localhost ip6-loopback
+ fe00::0 ip6-localnet
+ ff00::0 ip6-mcastprefix
+ ff02::1 ip6-allnodes
+ ff02::2 ip6-allrouters
+ ff02::3 ip6-allhosts
+
+ # hosts info for all vms
+REPLACED_HOSTS_INFO
+
+final_message: "The system with puppet is finally up, after $UPTIME seconds."
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+## This is a sample configuration file, as one input parameter for 'create_libvirt_vm.sh'
+host_names=test-master,test-agent
+puppet_enable=true
+master_host=test-master
+vm_mem=4096
+vm_cpu_cores=4
+image_url=file:///home/ubuntu/trusty-server-cloudimg-amd64-btnks.img
+image_name=disk.img
+ipaddr_start=192.168.122.201
+trusted_ssh_pub_keys_file=/home/ubuntu/trusted_ssh_pub_keys
+++ /dev/null
-#!/bin/bash
-##############################################################################
-# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-# this is a modified copy of compass4nfv/deploy/host_virtual.sh
-
-set -x
-
-SCRIPT_DIR=`cd ${BASH_SOURCE[0]%/*};pwd`
-WORK_DIR=${SCRIPT_DIR}/work
-
-mkdir -p $WORK_DIR
-
-source ./env_config.sh
-
-host_vm_dir=$WORK_DIR/vm
-
-
-function download_iso()
-{
- mkdir -p ${WORK_DIR}/cache
- curl --connect-timeout 10 -o ${WORK_DIR}/cache/$IMAGE_NAME $IMAGE_URL
-}
-
-
-function tear_down_machines() {
- for i in $HOSTNAMES; do
- echo "tear down machine:" $i
- sudo virsh destroy $i
- sudo virsh undefine $i
- rm -rf $host_vm_dir/$i
- done
-}
-
-
-function get_host_macs() {
- local mac_generator=${SCRIPT_DIR}/mac_generator.sh
- local machines=
-
- chmod +x $mac_generator
- mac_array=$($mac_generator $VIRT_NUMBER)
- machines=$(echo $mac_array)
-
- echo $machines
-}
-
-function launch_host_vms() {
- mac_array=($(get_host_macs))
-
- echo ${mac_array[2]}
- echo ${mac_array[*]}
-
- old_ifs=$IFS
- IFS=,
- tear_down_machines
- echo "bringing up vms ${mac_array[*]}"
- i=0
- for host in $HOSTNAMES; do
- echo "creating vm disk for instance $host" \
- "ip ${IPADDR_PREFIX}$((i+11))" \
- "mac ${mac_array[$i]}"
- vm_dir=$host_vm_dir/$host
- mkdir -p $vm_dir
-
- cp ${WORK_DIR}/cache/$IMAGE_NAME $vm_dir
-
- # create seed.iso
- sed -e "s/REPLACE_IPADDR/${IPADDR_PREFIX}$((i+11))/g" \
- -e "s/REPLACE_GATEWAY/${IPADDR_PREFIX}1/g" \
- -e "s/REPLACE_HOSTNAME/${host}/g" \
- meta-data_template \
- > meta-data
- cp user-data_template user-data
- genisoimage -output seed.iso -volid cidata -joliet -rock user-data meta-data
- cp seed.iso $vm_dir
-
- # create vm xml
- sed -e "s/REPLACE_MEM/$VIRT_MEM/g" \
- -e "s/REPLACE_CPU/$VIRT_CPUS/g" \
- -e "s/REPLACE_NAME/$host/g" \
- -e "s#REPLACE_IMAGE#$vm_dir/disk.img#g" \
- -e "s#REPLACE_SEED_IMAGE#$vm_dir/seed.iso#g" \
- -e "s/REPLACE_MAC_ADDR/${mac_array[$i]}/g" \
- libvirt_template.xml \
- > $vm_dir/libvirt.xml
-
- sudo virsh define $vm_dir/libvirt.xml
- sudo virsh start $host
- let i=i+1
- done
- IFS=$old_ifs
- rm -rf meta-data user-data seed.iso
-}
-
-
-download_iso
-launch_host_vms
-
-set +x
-
+++ /dev/null
-export HOSTNAMES=${HOSTNAMES:-"control,httpd,mysql,tomcat,client1,client2,client3,client4,benchmark"}
-export VIRT_NUMBER=9
-export VIRT_MEM=4096
-export VIRT_CPUS=4
-export IMAGE_URL=${IMAGE_URL:-"file:///bottlenecks/trusty-server-cloudimg-amd64-disk1.img"}
-export IMAGE_NAME=disk.img
-export IPADDR_PREFIX="192.168.122."
Code Review / bottlenecks.git / commitdiff