# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
\ No newline at end of file
+##############################################################################
##############################################################################
-import subprocess as subp
import sys
from rubbos_collector import RubbosCollector
from uploader import Uploader
def printUsage():
- print "Usage: python process_data.py required_params(**) optional_params([])"
+ print ("Usage: python process_data.py required_params(**)"
+ " optional_params([])")
print " ** -i|--input input_data_dir"
print " ** -s|--suite suite_name"
print " ** -c|--conf conf_file"
def get_onetime_data(dir_name):
- cmd = "grep -in 'remote client nodes' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name
+ cmd = ("grep -in 'remote client nodes' %s/index.html|awk "
+ "'{print $5}'|awk -F '<' '{print $1}'" % dir_name)
client_node_num = int(exec_shell(cmd))
- cmd = "grep -n 'Number of clients' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name
+ cmd = ("grep -n 'Number of clients' %s/index.html|awk "
+ "'{print $5}'|awk -F '<' '{print $1}'" % dir_name)
each_client_num = int(exec_shell(cmd))
total_client = (client_node_num + 1) * each_client_num
- cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "<B>" \'{if (FNR%%2==0 && FNR%%4!=0) {printf "%%s\\n", $3}}\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
+ cmd = ('grep -n "throughput" %s/stat_client*.html |awk -F "<B>"'
+ ' \'{if (FNR%%2==0 && FNR%%4!=0) {printf "%%s\\n", $3}}\'|awk \''
+ 'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name)
throughput = int(exec_shell(cmd))
- cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "<B>" \'{if (FNR==4) {printf "%%s\\n", $4}}\'|awk -F "</B>" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
+ cmd = ('grep -n "Total" %s/stat_client*.html |awk -F "<B>"'
+ ' \'{if (FNR==4) {printf "%%s\\n", $4}}\'|awk -F "</B>"'
+ ' \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name)
request = int(exec_shell(cmd))
- cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "<B>" \'{if (FNR==4) {printf "%%s\\n", $5}}\'|awk -F "</B>" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
+ cmd = ('grep -n "Total" %s/stat_client*.html |awk -F "<B>"'
+ ' \'{if (FNR==4) {printf "%%s\\n", $5}}\'|awk -F "</B>"'
+ ' \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name)
error_request = int(exec_shell(cmd))
return total_client, throughput, request, error_request
cmd = 'ls -l %s |grep ^d|awk \'{print $9}\'' % data_home
result = []
for subdir in exec_shell(cmd).split('\n'):
- total_client, throughput, request, error_request = get_onetime_data(
- data_home + '/' + subdir)
+ total_client, throughput, request, error_request = \
+ get_onetime_data(data_home + '/' + subdir)
result.append({'client': total_client,
'throughput': throughput,
'request': request,
def _test():
- #data = '{"details": [{"client": 200, "throughput": 20}, {"client": 300, "throughput": 20}], "case_name": "rubbos"}'
+ # data = '{"details": [{"client": 200, "throughput": 20},
+ # {"client": 300, "throughput": 20}], "case_name": "rubbos"}'
if len(sys.argv) < 2:
print ("no argumens input!!")
exit(1)
data = json.load(stream)
Uploader().upload_result(data)
+
if __name__ == "__main__":
_test()
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
\ No newline at end of file
+##############################################################################
return api_version
return DEFAULT_HEAT_API_VERSION
+
def get_nova_api_version():
api_version = os.getenv('OS_COMPUTE_API_VERSION')
if api_version is not None:
if api_version is not None:
log.info("GLANCE_API_VERSION is set in env as '%s'", api_version)
return api_version
- return DEFAULT_GLANCE_API_VERSION
\ No newline at end of file
+ return DEFAULT_GLANCE_API_VERSION
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-import files
# ------------------------------------------------------
# Directories and file locations
i += 1
if self.uuid is not None:
- sys.exit("delete stack failed!!!")
+ sys.exit("delete stack failed!!!")
else:
- HeatStack.stacks.remove(self)
+ HeatStack.stacks.remove(self)
@staticmethod
def delete_all():
--- /dev/null
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# bottlenecks: this file is copied from yardstick and slightly modified
+##############################################################################
+
+# bottlenecks-img-modify - download and modify a Ubuntu cloud image
+#
+# The actual customization is done by a script passed with an absolute path as
+# the only single argument. The command needs to be invoked as sudo
+#
+# Example invocation:
+# bottlenecks-img-modify /home/opnfv/bottlenecks/utils/tools/ubuntu-server-cloudimg-modify.sh
+#
+# Warning: the script will create files by default in:
+# /tmp/workspace/bottlenecks
+# the files will be owned by root!
+#
+# TODO: image resize is needed if the base image is too small
+#
+
+set -ex
+
+die() {
+ echo "error: $1" >&2
+ exit 1
+}
+
+test $# -eq 1 -o $# -eq 2 || die "no image specific script as argument"
+test $(id -u) -eq 0 || die "should invoke using sudo"
+
+cmd=$1
+RELEASE=$2
+test -x $cmd
+mountdir="/mnt/bottlenecks"
+workspace=${WORKSPACE:-"/tmp/workspace/bottlenecks"}
+host=${HOST:-"cloud-images.ubuntu.com"}
+release=${RELEASE:-"xenial"}
+image_path="${release}/current/${release}-server-cloudimg-amd64-disk1.img"
+image_url=${IMAGE_URL:-"https://${host}/${image_path}"}
+md5sums_path="${release}/current/MD5SUMS"
+md5sums_url=${MD5SUMS_URL:-"https://${host}/${md5sums_path}"}
+
+imgfile="${workspace}/bottlenecks-image.img"
+raw_imgfile="${workspace}/bottlenecks-${release}-server.raw"
+filename=$(basename $image_url)
+
+# download and checksum base image, conditionally if local copy is outdated
+download() {
+ test -d $workspace || mkdir -p $workspace
+ cd $workspace
+ rm -f MD5SUMS # always download the checksum file to a detect stale image
+ wget $md5sums_url
+ test -e $filename || wget -nc --progress=dot:giga $image_url
+ grep $filename MD5SUMS | md5sum -c ||
+ if [ $? -ne 0 ]; then
+ rm $filename
+ wget -nc --progress=dot:giga $image_url
+ grep $filename MD5SUMS | md5sum -c
+ fi
+
+ for i in $(seq 0 9); do
+ [ -a /dev/loop$i ] || mknod -m 660 /dev/loop$i b 7 $i
+ done
+
+ qemu-img convert $filename $raw_imgfile
+ cd -
+}
+
+# mount image
+setup() {
+ # qemu-img resize $raw_imgfile +5GB
+ mkdir -p $mountdir
+
+ loopdevice=$(kpartx -l $raw_imgfile | head -1 | cut -f1 -d ' ')
+
+ kpartx -av $raw_imgfile
+ # for trouble shooting
+ sleep 2
+ dmsetup ls
+ fdisk -l /dev/${loopdevice:0:5} || true
+ mount /dev/mapper/$loopdevice $mountdir
+ mount -t proc none $mountdir/proc
+
+ cp $cmd $mountdir/$(basename $cmd)
+}
+
+# modify image running a script using in a chrooted environment
+modify() {
+ # resolv.conf does not exist in base image, pass nameserver value from host
+ nameserver_ip=$(grep -m 1 '^nameserver' \
+ /etc/resolv.conf | awk '{ print $2 '})
+
+ # prevent init scripts from running during install
+ echo $'#!/bin/sh\nexit 101' >$mountdir/usr/sbin/policy-rc.d
+ chmod a+x $mountdir/usr/sbin/policy-rc.d
+
+ chroot $mountdir /$(basename $cmd) $nameserver_ip
+
+ rm -rf $mountdir/usr/sbin/policy-rc.d
+
+ umount -f $mountdir/proc
+ umount $mountdir
+
+ qemu-img convert -c -o compat=0.10 -O qcow2 $raw_imgfile $imgfile
+
+ if dmsetup table | grep $loopdevice; then
+ dmsetup clear $loopdevice || true
+ fi
+}
+
+# cleanup (umount) the image
+cleanup() {
+ # designed to be idempotent
+ mount | grep $mountdir/proc && umount $mountdir/proc
+ mount | grep $mountdir && umount $mountdir
+ mount | grep "/mnt/vivid" && umount "/mnt/vivid"
+
+ if [ -f $raw_imgfile ]; then
+ #kpartx -dv $raw_imgfile sometimes failed, we should checked it agein.
+ #if [ -z "$(kpartx -l $raw_imgfile | grep 'loop deleted')" ]; then
+ # kpartx -dv $raw_imgfile
+ #fi
+ kpartx -dv $raw_imgfile || true
+ fi
+
+ rm -f $raw_imgfile
+ rm -rf $mountdir
+}
+
+exitcode=""
+error_trap()
+{
+ local rc=$?
+
+ set +e
+
+ if [ -z "$exitcode" ]; then
+ exitcode=$rc
+ fi
+
+ dmesg -T | tail -50
+
+ cleanup
+
+ echo "Image build failed with $exitcode"
+
+ exit $exitcode
+}
+
+main() {
+ cleanup
+
+ trap "error_trap" EXIT SIGTERM
+
+ download
+ setup
+ modify
+
+ trap - EXIT SIGTERM
+ cleanup
+
+ echo "the modified image is found here: $imgfile"
+}
+
+main
--- /dev/null
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# bottlenecks: this file is copied from yardstick and slightly modified
+##############################################################################
+
+# installs required packages
+# must be run from inside the image (either chrooted or running)
+
+set -ex
+
+if [ $# -eq 1 ]; then
+ nameserver_ip=$1
+
+ # /etc/resolv.conf is a symbolic link to /run, restore at end
+ rm /etc/resolv.conf
+ echo "nameserver $nameserver_ip" > /etc/resolv.conf
+ echo "nameserver 8.8.8.8" >> /etc/resolv.conf
+ echo "nameserver 8.8.4.4" >> /etc/resolv.conf
+fi
+
+# Add hostname to /etc/hosts.
+# Allow console access via pwd
+cat <<EOF >/etc/cloud/cloud.cfg.d/10_etc_hosts.cfg
+manage_etc_hosts: True
+password: RANDOM
+chpasswd: { expire: False }
+ssh_pwauth: True
+EOF
+
+#install some software
+apt-get update
+apt-get install -y \
+ netperf \
+ sysstat
+
+# restore symlink
+ln -sf /run/resolvconf/resolv.conf /etc/resolv.conf
Code Review / bottlenecks.git / commitdiff