python \
python-dev \
python-pip \
+ vim \
python-setuptools && \
easy_install -U setuptools==30.0.0
RUN easy_install pytz
RUN pip install -r ${REPOS_DIR}/bottlenecks/requirements.txt
+RUN pip install -U /home/opnfv/bottlenecks
image: opnfv/yardstick:latest
volumes:
- /var/run/docker.sock:/var/run/docker.sock
+ - /tmp/:/tmp/
ports:
- "8888:5000"
+ privileged: true
+ environment:
+ - INSTALLER_IP=192.168.200.2
+ - INSTALLER_TYPE=compass
bottlenecks:
restart: always
build: bottlenecks/
+ volumes:
+ - /var/run/docker.sock:/var/run/docker.sock
+ - /tmp/:/tmp/
+ environment:
+ - INSTALLER_IP=192.168.200.2
+ - INSTALLER_TYPE=compass
+ - DEBUG=true
wrapt==1.10.6
pyroute2==0.4.10
elasticsearch==5.0.1
+docker
setup(
name="bottlenecks",
version="master",
- py_modules=['bottlenecks_cli'],
+ py_modules=['cli/bottlenecks_cli'],
packages=find_packages(),
include_package_data=True,
package_data={
],
},
)
-
posca_factor_system_bandwidth by default.'''
import importlib
+import sys
import utils.parser as conf_parser
import utils.logger as log
INTERPRETER = "/usr/bin/python"
def main():
- test_level = "testcase"
- test_name = "posca_factor_system_bandwidth"
+ test_level = sys.argv[1]
+ test_name = sys.argv[2]
posca_run(test_level, test_name)
--- /dev/null
+# Sample stress task config file
+# Three scenarios run in parallel pinging one target vm.
+# Multiple context are used to specify the host and target VMs.
+
+load_manager:
+ scenarios:
+ tool: ping
+ test_times: 100
+ package_size:
+ num_stack: 5, 10, 20
+ package_loss: 10%
+
+ contexts:
+ stack_create: yardstick
+ flavor:
+ yardstick_test_ip:
+ yardstick_test_dir: "samples"
+ yardstick_testcase: "ping_bottlenecks"
+
+dashboard:
+ dashboard: "y"
+ dashboard_ip:
--- /dev/null
+#!/usr/bin/python
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+'''This file realize a function of creating dashboard of stress ping test'''
+import ConfigParser
+from elasticsearch import Elasticsearch
+import json
+import os
+import utils.logger as log
+from utils.parser import Parser as conf_parser
+
+LOG = log.Logger(__name__).getLogger()
+config = ConfigParser.ConfigParser()
+es = Elasticsearch()
+dashboard_path = os.path.join(conf_parser.test_dir,
+ "posca",
+ "testcase_dashboard")
+dashboard_dir = dashboard_path + "/"
+
+
+def dashboard_send_data(runner_config, test_data):
+ global es
+ es_ip = runner_config['dashboard_ip'].split(':')
+ es = Elasticsearch([{'host': es_ip[0]}])
+ res = es.index(index="bottlenecks",
+ doc_type=test_data["testcase"],
+ body=test_data["data_body"])
+ if res['created'] == "False":
+ LOG.error("date send to kibana have errors ", test_data["data_body"])
+
+
+def posca_stress_ping(runner_config):
+ global es
+ es_ip = runner_config['dashboard_ip'].split(':')
+ es = Elasticsearch([{'host': es_ip[0]}])
+ # Create bottlenecks index
+ with open(dashboard_dir + 'posca_stress_ping_index_pattern.json')\
+ as index_pattern:
+ doc = json.load(index_pattern)
+ res = es.index(
+ index=".kibana",
+ doc_type="index-pattern",
+ id="bottlenecks",
+ body=doc)
+ if res['created'] == "True":
+ LOG.info("bottlenecks index-pattern has created")
+ else:
+ LOG.info("bottlenecks index-pattern has existed")
+
+ with open(dashboard_dir + 'posca_system_bandwidth_config.json')\
+ as index_config:
+ doc = json.load(index_config)
+ res = es.index(index=".kibana", doc_type="config", id="4.6.1", body=doc)
+ if res['created'] == "True":
+ LOG.info("bottlenecks config has created")
+ else:
+ LOG.info("bottlenecks config has existed")
+
+ # Configure discover panel
+ with open(dashboard_dir + 'posca_stress_ping_discover.json')\
+ as index_discover:
+ doc = json.load(index_discover)
+ res = es.index(
+ index=".kibana",
+ doc_type="search",
+ id="posca_stress_ping",
+ body=doc)
+ if res['created'] == "True":
+ LOG.info("posca_stress_ping search has created")
+ else:
+ LOG.info("posca_stress_ping search has existed")
+
+ # Create testing data in line graph
+ with open(dashboard_dir + 'posca_stress_ping_histogram.json')\
+ as line_data:
+ doc = json.load(line_data)
+ res = es.index(
+ index=".kibana",
+ doc_type="visualization",
+ id="posca_stress_ping_histogram",
+ body=doc)
+ if res['created'] == "True":
+ LOG.info("posca_stress_ping visualization has created")
+ else:
+ LOG.info("posca_stress_ping visualization has existed")
+
+ # Create comparison results in line chart
+ with open(dashboard_dir + 'posca_stress_ping_table.json')\
+ as line_char:
+ doc = json.load(line_char)
+ res = es.index(
+ index=".kibana",
+ doc_type="visualization",
+ id="posca_stress_ping_table",
+ body=doc)
+ if res['created'] == "True":
+ LOG.info("posca_stress_ping visualization has created")
+ else:
+ LOG.info("posca_stress_ping visualization has existed")
+
+ # Create dashboard
+ with open(dashboard_dir + 'posca_stress_ping_dashboard.json')\
+ as dashboard:
+ doc = json.load(dashboard)
+ res = es.index(
+ index=".kibana",
+ doc_type="dashboard",
+ id="posca_stress_ping",
+ body=doc)
+ if res['created'] == "True":
+ LOG.info("posca_stress_ping dashboard has created")
+ else:
+ LOG.info("posca_stress_ping dashboard has existed")
--- /dev/null
+{
+ "title": "posca_stress_ping",
+ "hits": 0,
+ "description": "",
+ "panelsJSON": "[{\"id\":\"posca_stress_ping_histogram\",\"type\":\"visualization\",\"panelIndex\":1,\"size_x\":6,\"size_y\":4,\"col\":1,\"row\":1},{\"id\":\"posca_stress_ping_table\",\"type\":\"visualization\",\"panelIndex\":2,\"size_x\":6,\"size_y\":4,\"col\":7,\"row\":1}]",
+ "optionsJSON": "{\"darkTheme\":false}",
+ "uiStateJSON": "{}",
+ "version": 1,
+ "timeRestore": false,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}]}"
+ }
+}
\ No newline at end of file
--- /dev/null
+{
+ "title": "posca_stress_ping",
+ "description": "",
+ "hits": 0,
+ "columns": [
+ "_source"
+ ],
+ "sort": [
+ "_score",
+ "desc"
+ ],
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"index\":\"bottlenecks\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"
+ }
+}
\ No newline at end of file
--- /dev/null
+{
+ "title": "posca_stress_ping_histogram",
+ "visState": "{\"title\":\"New Visualization\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"success_rate\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"number_of_users\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+ "uiStateJSON": "{}",
+ "description": "",
+ "savedSearchId": "posca_stress_ping",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"filter\":[]}"
+ }
+}
\ No newline at end of file
--- /dev/null
+{
+ "title": "bottlenecks",
+ "fields": "[{\"name\":\"success_times\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"number_of_users\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"duration_time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"success_rate\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false}]"
+}
\ No newline at end of file
--- /dev/null
+{
+ "title": "posca_stress_ping_table",
+ "visState": "{\"title\":\"New Visualization\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"duration_time\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"number_of_users\",\"size\":100,\"order\":\"asc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"success_times\",\"size\":100,\"order\":\"asc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"success_rate\",\"size\":100,\"order\":\"asc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+ "uiStateJSON": "{}",
+ "description": "",
+ "savedSearchId": "posca_stress_ping",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"filter\":[]}"
+ }
+}
\ No newline at end of file
LOG = log.Logger(__name__).getLogger()
config = ConfigParser.ConfigParser()
es = Elasticsearch()
-dashboard_dir = os.path.join(conf_parser.test_dir,
- "posca",
- "testcase_dashboard")
+dashboard_path = os.path.join(conf_parser.test_dir,
+ "posca",
+ "testcase_dashboard")
+dashboard_dir = dashboard_path + "/"
def dashboard_send_data(runner_config, test_data):
--- /dev/null
+#!/usr/bin/env python
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+'''This file realize the function of run posca ping stress test script
+This file contain several part:
+Frist is create a script to realize several threading run'''
+
+import utils.logger as log
+import uuid
+import json
+import os
+import multiprocessing
+import docker
+import datetime
+from utils.parser import Parser as conf_parser
+import utils.env_prepare.quota_prepare as quota_prepare
+import utils.env_prepare.stack_prepare as stack_prepare
+import testsuites.posca.testcase_dashboard.posca_stress_ping as DashBoard
+# --------------------------------------------------
+# logging configuration
+# --------------------------------------------------
+LOG = log.Logger(__name__).getLogger()
+
+test_dict = {
+ "action": "runTestCase",
+ "args": {
+ "opts": {
+ "task-args": {}
+ },
+ "testcase": "ping_bottlenecks"
+ }
+}
+testfile = os.path.basename(__file__)
+testcase, file_format = os.path.splitext(testfile)
+
+
+def env_pre(con_dic):
+ stack_prepare._prepare_env_daemon()
+ quota_prepare.quota_env_prepare()
+ client = docker.from_env()
+ con = client.containers.get('bottleneckcompose_yardstick_1')
+ cmd = ('yardstick env prepare')
+ LOG.info("yardstick envrionment prepare!")
+ stdout = con.exec_run(cmd)
+ LOG.debug(stdout)
+
+
+def do_test(test_config, con_dic):
+ out_file = ("/tmp/yardstick_" + str(uuid.uuid4()) + ".out")
+ client = docker.from_env()
+ con = client.containers.get('bottleneckcompose_yardstick_1')
+ cmd = ('yardstick task start /home/opnfv/repos/yardstick/'
+ 'samples/ping_bottlenecks.yaml --output-file ' + out_file)
+ stdout = con.exec_run(cmd)
+ LOG.debug(stdout)
+ with open(out_file) as f:
+ data = json.load(f)
+ if data["status"] == 1:
+ LOG.info("yardstick run success")
+ out_value = 1
+ else:
+ LOG.error("yardstick error exit")
+ out_value = 0
+ os.remove(out_file)
+ return out_value
+
+
+def config_to_result(num, out_num, during_date):
+ testdata = {}
+ test_result = {}
+ test_result["number_of_users"] = float(num)
+ test_result["success_times"] = out_num
+ test_result["success_rate"] = out_num / num
+ test_result["duration_time"] = during_date
+ testdata["data_body"] = test_result
+ testdata["testcase"] = testcase
+ return testdata
+
+
+def func_run(condic):
+ test_config = {}
+ test_date = do_test(test_config, condic)
+ return test_date
+
+
+def run(test_config):
+ con_dic = test_config["load_manager"]
+ test_num = con_dic['scenarios']['num_stack'].split(',')
+ if con_dic["contexts"]["yardstick_test_ip"] is None:
+ con_dic["contexts"]["yardstick_test_ip"] =\
+ conf_parser.ip_parser("yardstick_test_ip")
+
+ if test_config["dashboard"]["dashboard"] == 'y':
+ if test_config["dashboard"]["dashboard_ip"] is None:
+ test_config["dashboard"]["dashboard_ip"] =\
+ conf_parser.ip_parser("dashboard")
+ LOG.info("Create Dashboard data")
+ DashBoard.posca_stress_ping(test_config["dashboard"])
+
+ LOG.info("bottlenecks envrionment prepare!")
+ env_pre(con_dic)
+ LOG.info("yardstick envrionment prepare done!")
+
+ for value in test_num:
+ result = []
+ out_num = 0
+ num = int(value)
+ pool = multiprocessing.Pool(processes=num)
+ LOG.info("begin to run %s thread" % num)
+
+ starttime = datetime.datetime.now()
+ for i in range(0, int(num)):
+ result.append(pool.apply_async(func_run, (con_dic, )))
+ pool.close()
+ pool.join()
+ for res in result:
+ out_num = out_num + float(res.get())
+
+ endtime = datetime.datetime.now()
+ LOG.info("%s thread success %d times" % (num, out_num))
+ during_date = (endtime - starttime).seconds
+
+ data_reply = config_to_result(num, out_num, during_date)
+ if test_config['dashboard']['dashboard'] == 'y':
+ DashBoard.dashboard_send_data(test_config['dashboard'], data_reply)
+ conf_parser.result_to_file(data_reply, test_config["out_file"])
+
+ if out_num < num:
+ success_rate = ('%d/%d' % (out_num, num))
+ LOG.error('error thread: %d '
+ 'the successful rate is %s'
+ % (num - out_num, success_rate))
+ break
+ LOG.info('END POSCA stress ping test')
--- /dev/null
+#!/usr/bin/env python
+##############################################################################
+# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+import os
+import commands
+import utils.logger as log
+import utils.infra_setup.heat.manager as client_manager
+
+LOG = log.Logger(__name__).getLogger()
+
+neutron_quota = {"subnet": -1,
+ "network": -1,
+ "floatingip": -1,
+ "subnetpool": -1,
+ "router": -1,
+ "port": -1}
+
+nova_quota = {"ram": -1,
+ "cores": -1,
+ "instances": -1,
+ "key_pairs": -1,
+ "fixed_ips": -1,
+ "floating_ips": -1,
+ "server_groups": -1,
+ "injected_files": -1,
+ "metadata_items": -1,
+ "security_groups": -1,
+ "security_group_rules": -1,
+ "server_group_members": -1,
+ "injected_file_content_bytes": -1,
+ "injected_file_path_bytes": -1}
+
+
+def quota_env_prepare():
+ tenant_name = os.getenv("OS_TENANT_NAME")
+ cmd = ("openstack project list | grep " +
+ tenant_name +
+ " | awk '{print $2}'")
+
+ result = commands.getstatusoutput(cmd)
+ if result[0] == 0:
+ LOG.info(result[1])
+ else:
+ LOG.error("can't get openstack project id")
+ return 1
+
+ openstack_id = result[1]
+
+ nova_client = client_manager._get_nova_client()
+ neutron_client = client_manager._get_neutron_client()
+
+ nova_q = nova_client.quotas.get(openstack_id).to_dict()
+ neutron_q = neutron_client.show_quota(openstack_id)
+ LOG.info(tenant_name + "tenant nova and neutron quota(previous) :")
+ LOG.info(nova_q)
+ LOG.info(neutron_q)
+
+ nova_client.quotas.update(openstack_id, **nova_quota)
+ neutron_client.update_quota(openstack_id,
+ {'quota': neutron_quota})
+ LOG.info("Quota has been changed!")
+
+ nova_q = nova_client.quotas.get(openstack_id).to_dict()
+ neutron_q = neutron_client.show_quota(openstack_id)
+ LOG.info(tenant_name + "tenant nova and neutron quota(now) :")
+ LOG.info(nova_q)
+ LOG.info(neutron_q)
+ return 0
_source_file(rc_file)
- _append_external_network(rc_file)
+ # _append_external_network(rc_file)
# update the external_network
- _source_file(rc_file)
+ # _source_file(rc_file)
def _get_remote_rc_file(rc_file, installer_ip, installer_type):
test_dict = {
"action": "createInfluxDBContainer",
}
- requests.post(
+ responce = requests.post(
base_url, data=json.dumps(test_dict), headers=headers)
+ ask_data = json.loads(responce.text)
+ task_id = ask_data["result"]["task_id"]
LOG.info("waiting for creating InfluxDB")
time.sleep(30)
- LOG.info("Done, creating InflxDB Container")
+ return task_id
+
+
+def yardstick_env_prepare(con_dic):
+ base_url = ("http://%s/yardstick/env/action"
+ % (con_dic['yardstick_test_ip']))
+ test_dict = {
+ "action": "prepareYardstickEnv",
+ }
+ LOG.info("waiting for yardstick environment prepare")
+ reponse = requests.post(
+ base_url, data=json.dumps(test_dict), headers=headers)
+ ask_data = json.loads(reponse.text)
+ task_id = ask_data["result"]["task_id"]
+ LOG.info("Done, yardstick environment prepare complete!")
+ return task_id
def find_condition(con_dic):
ch = logging.StreamHandler()
log_formatter = ('%(asctime)s '
- '%(name)s %(filename)s:%(lineno)d '
+ '%(filename)s:%(lineno)d '
'%(levelname)s %(message)s')
formatter = logging.Formatter(log_formatter)
Code Review / bottlenecks.git / commitdiff