From: Yang (Gabriel) Yu Date: Tue, 4 Sep 2018 11:03:46 +0000 (+0000) Subject: Merge "add testsuite and testcase list pages in frontend of testing-scheduler" X-Git-Tag: opnfv-7.1.0~9 X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?p=bottlenecks.git;a=commitdiff_plain;h=d5f7a33fc251788320b03ae2217611f14ec68fab;hp=26b13e50c888bc0ca03c0f68731f7c5abff0c6b6 Merge "add testsuite and testcase list pages in frontend of testing-scheduler" --- diff --git a/monitor/monitoring.sh b/monitor/monitoring.sh index 26b63fc5..758f77ec 100644 --- a/monitor/monitoring.sh +++ b/monitor/monitoring.sh @@ -52,7 +52,7 @@ while [[ $# > 0 ]] INSTALLER_TYPE="$2" shift ;; - -i|--openstack-env) + -o|--openstack-env) OPENSTACK_ENV="$2" shift ;; @@ -104,7 +104,6 @@ sudo docker run --name bottlenecks-node-exporter \ -v "/proc:/host/proc:ro" \ -v "/sys:/host/sys:ro" \ -v "/:/rootfs:ro" \ - --net="host" \ quay.io/prometheus/node-exporter:v0.14.0 \ -collector.procfs /host/proc \ -collector.sysfs /host/sys \ diff --git a/monitor/uninstall.py b/monitor/uninstall.py index 3a9cf0c7..26351722 100644 --- a/monitor/uninstall.py +++ b/monitor/uninstall.py @@ -50,3 +50,4 @@ local_del_docker('cadvisor') local_del_docker('barometer') local_del_docker('grafana') local_del_docker('collectd') +local_del_docker('openstack-exporter') diff --git a/requirements/requirements.txt b/requirements/requirements.txt index dacffcea..381cb5cd 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -18,3 +18,4 @@ nose==1.3.7 pyroute2==0.4.10 elasticsearch==5.0.1 docker==2.0.2 +kubernetes==6.0.0 diff --git a/testing-scheduler/server/__init__.py b/testing-scheduler/server/__init__.py new file mode 100644 index 00000000..e8198009 --- /dev/null +++ b/testing-scheduler/server/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/testing-scheduler/server/conductorclient/__init__.py b/testing-scheduler/server/conductorclient/__init__.py new file mode 100644 index 00000000..bb02be17 --- /dev/null +++ b/testing-scheduler/server/conductorclient/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/testing-scheduler/server/conductorclient/mock_tasks.json b/testing-scheduler/server/conductorclient/mock_tasks.json new file mode 100644 index 00000000..4fea48bf --- /dev/null +++ b/testing-scheduler/server/conductorclient/mock_tasks.json @@ -0,0 +1,13 @@ +{ + "task_group_1":[ + { + "name": "http_yardstick_test", + "retryCount": 3, + "timeOutSeconds": 1200, + "timeOutPolicy": "TIME_OUT_WF", + "retryLogic": "FIXED", + "retryDelaySeconds": 600, + "responseTimeOutSeconds": 3600 + } + ] +} \ No newline at end of file diff --git a/testing-scheduler/server/conductorclient/mock_workflow.json b/testing-scheduler/server/conductorclient/mock_workflow.json new file mode 100644 index 00000000..8f6251c0 --- /dev/null +++ b/testing-scheduler/server/conductorclient/mock_workflow.json @@ -0,0 +1,24 @@ +{ + "name": "workflow_demo_05", + "description": "run a workflow of yardstick test service", + "version": 1, + "tasks": [ + { + "name": "http_yardstick_test", + "taskReferenceName": "ping_test", + "inputParameters": { + "http_request": { + "uri": "http://192.168.199.105:8080/greet", + "method": "GET" + } + }, + "type": "HTTP" + } + ], + "outputParameters": { + "header": "${ping_test.output.response.headers}", + "response": "${ping_test.output.response.body}", + "status": "${ping_test.output.response.statusCode}" + }, + "schemaVersion": 2 +} \ No newline at end of file diff --git a/testing-scheduler/server/conductorclient/run_new_workflow.py b/testing-scheduler/server/conductorclient/run_new_workflow.py new file mode 100644 index 00000000..0acb96a0 --- /dev/null +++ b/testing-scheduler/server/conductorclient/run_new_workflow.py @@ -0,0 +1,71 @@ +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from conductor import conductor +import json + + +class WorkflowMgr(object): + def __init__(self, serverAddr): + self._serverAddr = serverAddr + '/api' + self._metaDataClient = conductor.MetadataClient(self._serverAddr) + self._workflowClient = conductor.WorkflowClient(self._serverAddr) + self._tasksDefined = False + self._workflowDefined = False + self._workflowName = "" + + def setTaskDef(self, taskJson): + jsonObj = json.loads(taskJson) + print "define tasks:\n", taskJson + for (k, v) in jsonObj.items(): + self._metaDataClient.registerTaskDefs(v) + self._tasksDefined = True + + def setWorkflowDef(self, workflowJson): + jsonObj = json.loads(workflowJson) + print "define workflow:\n", workflowJson + try: + self._metaDataClient.createWorkflowDef(jsonObj) + except Exception as e: + print e + self._workflowName = jsonObj['name'] + self._workflowDefined = True + + def startWorkflow(self, param={}): + workflowId = '' + if not self._tasksDefined: + print "error: please define the task at first\n" + elif not self._workflowDefined: + print "error: please define the workflow at first\n" + else: + workflowId = self._workflowClient.startWorkflow( + self._workflowName, param) + return workflowId + + def setTaskDefFromFile(self, taskFilePath): + with open(taskFilePath, 'r') as f: + self.setTaskDef(f.read()) + + def setWorkflowFromFile(self, workflowFilePath): + with open(workflowFilePath, 'r') as f: + self.setWorkflowDef(f.read()) + + +# test demo +def main(): + serverAddr = "http://192.168.199.131:8080" + wfMgr = WorkflowMgr(serverAddr) + wfMgr.setTaskDefFromFile('mock_tasks.json') + wfMgr.setWorkflowFromFile('mock_workflow.json') + inputParam = {'input': 'fake'} + wfMgr.startWorkflow(inputParam) + + +if __name__ == "__main__": + main() diff --git a/testing-scheduler/server/python_modules.txt b/testing-scheduler/server/python_modules.txt new file mode 100644 index 00000000..62da9c26 --- /dev/null +++ b/testing-scheduler/server/python_modules.txt @@ -0,0 +1,14 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +flask +flask_cors +pyyaml +pyaml +requests \ No newline at end of file diff --git a/testing-scheduler/server/setup.py b/testing-scheduler/server/setup.py new file mode 100644 index 00000000..398f8763 --- /dev/null +++ b/testing-scheduler/server/setup.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +'''This file realize the function of how to setup server of testing-scheduler +to your environment. This use setuptools tool to setup''' + +from setuptools import setup, find_packages + + +setup( + name="testing-scheduler-server", + version="0.1", + packages=find_packages(), + include_package_data=True, + package_data={ + 'src': [ + 'env/context/*.yaml', + 'env/service/*.yaml', + 'conductor_processor/*.json' + ], + 'test': [ + 'test_case/*/*.yaml' + ] + } +) diff --git a/testing-scheduler/server/src/__init__.py b/testing-scheduler/server/src/__init__.py new file mode 100644 index 00000000..e8198009 --- /dev/null +++ b/testing-scheduler/server/src/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/testing-scheduler/server/src/conductor_processor/__init__.py b/testing-scheduler/server/src/conductor_processor/__init__.py new file mode 100644 index 00000000..bb02be17 --- /dev/null +++ b/testing-scheduler/server/src/conductor_processor/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/testing-scheduler/server/src/conductor_processor/defaultTaskFile.json b/testing-scheduler/server/src/conductor_processor/defaultTaskFile.json new file mode 100644 index 00000000..a98a5819 --- /dev/null +++ b/testing-scheduler/server/src/conductor_processor/defaultTaskFile.json @@ -0,0 +1,9 @@ +{ + "name": "", + "retryCount": 6, + "timeOutSeconds": 1200, + "timeOutPolicy": "TIME_OUT_WF", + "retryLogic": "FIXED", + "retryDelaySeconds": 3, + "responseTimeOutSeconds": 3600 +} \ No newline at end of file diff --git a/testing-scheduler/server/src/conductor_processor/defaultWorkflowFile.json b/testing-scheduler/server/src/conductor_processor/defaultWorkflowFile.json new file mode 100644 index 00000000..8f6251c0 --- /dev/null +++ b/testing-scheduler/server/src/conductor_processor/defaultWorkflowFile.json @@ -0,0 +1,24 @@ +{ + "name": "workflow_demo_05", + "description": "run a workflow of yardstick test service", + "version": 1, + "tasks": [ + { + "name": "http_yardstick_test", + "taskReferenceName": "ping_test", + "inputParameters": { + "http_request": { + "uri": "http://192.168.199.105:8080/greet", + "method": "GET" + } + }, + "type": "HTTP" + } + ], + "outputParameters": { + "header": "${ping_test.output.response.headers}", + "response": "${ping_test.output.response.body}", + "status": "${ping_test.output.response.statusCode}" + }, + "schemaVersion": 2 +} \ No newline at end of file diff --git a/testing-scheduler/server/src/conductor_processor/task.py b/testing-scheduler/server/src/conductor_processor/task.py new file mode 100644 index 00000000..6f25aef8 --- /dev/null +++ b/testing-scheduler/server/src/conductor_processor/task.py @@ -0,0 +1,28 @@ +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import json +import os + + +class TaskFile(object): + def __init__(self, taskName='task_0'): + self._defaultConfFile = self._getFilePath("defaultTaskFile.json") + with open(self._defaultConfFile) as defaultConf: + self._jsonObj = json.load(defaultConf) + self._jsonObj['name'] = taskName + + def generateFromStep(self, stepObject): + self._jsonObj['name'] = stepObject.getName() + print "taskFile:", self._jsonObj['name'] + return self._jsonObj + + def _getFilePath(self, fileName): + dirPath = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(dirPath, fileName) diff --git a/testing-scheduler/server/src/conductor_processor/workflow.py b/testing-scheduler/server/src/conductor_processor/workflow.py new file mode 100644 index 00000000..19f0896c --- /dev/null +++ b/testing-scheduler/server/src/conductor_processor/workflow.py @@ -0,0 +1,243 @@ +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import random +import collections +import re +from src.conductor_processor.task import TaskFile + + +class WorkflowFile(object): + def __init__(self, name): + self._name = "workflow_" + name + "(%s)" % getRandString(10) + self._description = '' + self._version = 1 + self._schemaVersion = 2 + self._tasks = [] + self._outputParameters = {} + + def getDict(self): + d = collections.OrderedDict() + d['name'] = self._name + d['description'] = self._description + d['version'] = self._version + d['schemaVersion'] = self._schemaVersion + d['tasks'] = self._tasks + d['outputParameters'] = self._outputParameters + + return d + + def generateMetaData(self, flowList, stepObjArr): + flowParser = FlowParser(flowList, stepObjArr) + self._tasks, taskMetaList = flowParser.parseMainFlow() + normalTasks = flowParser.getNormalTaskList() + for normalTask in normalTasks: + taskName = normalTask['name'] + referenceName = normalTask['taskReferenceName'] + self._outputParameters["%s(%s)" % (taskName, referenceName)] = \ + "${%s.output.response.body}" % referenceName + return self.getDict(), taskMetaList + + +class FlowParser(object): + def __init__(self, flowList, stepObjArr): + self._mainFlow = {} + self._subFlowDict = {} + self._stepObjArr = stepObjArr + self._normalTasks = [] + for flow in flowList: + if flow['name'] == "main": + self._mainFlow = flow + else: + self._subFlowDict[flow['name']] = flow + + def parseMainFlow(self): + return self.parseOrderList(self._mainFlow['orders'], self._stepObjArr) + + def parse(self, obj, stepObjArr): + if isinstance(obj, str): + return self.parseFlow(obj, stepObjArr) + else: + return self.parseOrderList(obj, stepObjArr) + + def parseFlow(self, flowName, stepObjArr): + orderList = self._subFlowDict[flowName]['orders'] + return self.parseOrderList(orderList, stepObjArr) + + def parseOrderList(self, orderList, stepObjArr): + tasks = [] + taskMetaAllList = [] + for order in orderList: + if order['type'] == "normal": + genTask = NormalTask(order, stepObjArr, self) + self._normalTasks.append(genTask) + elif order['type'] == "switch": + genTask = SwitchTask(order, stepObjArr, self) + elif order['type'] == "parallel": + genTask = ParallelTask(order, stepObjArr, self) + tasks.append(genTask.getDict()) + + if order['type'] == "parallel": + joinTask = genTask.getJoinTask() + tasks.append(joinTask.getDict()) + + taskMetaList = genTask.getTaskMetaList() + if taskMetaList is not None: + taskMetaAllList.extend(taskMetaList) + return tasks, taskMetaAllList + + def getNormalTaskList(self): + normalTasksDict = [] + for normalTask in self._normalTasks: + normalTasksDict.append(normalTask.getDict()) + return normalTasksDict + + def getNormalTask(self, stepId): + for normalTask in self._normalTasks: + if normalTask.getStepId() == stepId: + return normalTask + return None + + +class BaseWorkflowTask(object): + def __init__(self, name): + self._name = name + self._taskReferenceName = self._name + "_task_%s" % getRandString(10) + self._type = '' + self._args = {} + + def __str__(self): + dictObj = self.getDict() + return str(dictObj) + + def getDict(self): + d1 = { + "name": self._name, + "taskReferenceName": self._taskReferenceName, + "type": self._type + } + return dict(d1, **self._args) + + def getName(self): + return self._name + + def getReferenceName(self): + return self._taskReferenceName + + def getTaskMetaList(self): + taskFile = TaskFile() + return [taskFile.generateFromStep(self)] + + +class NormalTask(BaseWorkflowTask): + def __init__(self, order, stepObjArr, flowParser): + relatedStepObj = stepObjArr[order['step'] - 1] + super(NormalTask, self).__init__(relatedStepObj.getName()) + self._taskReferenceName = "task_%s" % getRandString(10) + self._stepId = relatedStepObj.getId() + self._type = "HTTP" + self._args['inputParameters'] = relatedStepObj.getArgs() + self._paramTransform(self._args['inputParameters'], flowParser) + print "NormalTask:----------------------\n", relatedStepObj.getArgs() + + def _paramTransform(self, argsDict, flowParser): + for (k, v) in argsDict.items(): + if isinstance(v, str): + if re.match("^\(\(\d+\..*\)\)", v): + v = v[2:-2] + stepId, outputParam = v.split(".") + stepId = int(stepId) + normalTask = flowParser.getNormalTask(stepId) + if normalTask is None: + continue + argsDict[k] = "${%s.output.response.body.%s}" % \ + (normalTask.getReferenceName(), outputParam) + elif isinstance(v, dict): + self._paramTransform(v, flowParser) + + def getStepId(self): + return self._stepId + + +class SwitchTask(BaseWorkflowTask): + seqNumber = 0 + + def __init__(self, order, stepObjArr, flowParser): + super(SwitchTask, self).__init__("switch_" + str(SwitchTask.seqNumber)) + SwitchTask.seqNumber = SwitchTask.seqNumber + 1 + if 'name' in order: + self._name = order['name'] + self._type = "DECISION" + caseValueParam = 'value' + order['value'] = order['value'][2:-2] + stepId, outputParam = order['value'].split(".") + stepId = int(stepId) + normalTask = flowParser.getNormalTask(stepId) + caseValue = "${%s.output.response.body.%s}" % \ + (normalTask.getReferenceName(), outputParam) + self._args['inputParameters'] = {caseValueParam: caseValue} + self._args['caseValueParam'] = caseValueParam + self._args['decisionCases'] = {} + self._childTaskMetaList = [] + for case, caseOrders in order['cases'].items(): + self._args['decisionCases'][case], taskMetaList = \ + flowParser.parse(caseOrders, stepObjArr) + if taskMetaList is not None: + self._childTaskMetaList.extend(taskMetaList) + + def getTaskMetaList(self): + selfTaskMetaList = super(SwitchTask, self).getTaskMetaList() + selfTaskMetaList.extend(self._childTaskMetaList) + return selfTaskMetaList + + +class ParallelTask(BaseWorkflowTask): + seqNumber = 0 + + def __init__(self, order, stepObjArr, flowParser): + InstSeqNumber = ParallelTask.seqNumber + super(ParallelTask, self).__init__("parallel_" + str(InstSeqNumber)) + ParallelTask.seqNumber = ParallelTask.seqNumber + 1 + if 'name' in order: + self._name = order['name'] + self._type = "FORK_JOIN" + self._args['forkTasks'] = [] + self._childTaskMetaList = [] + lastTasksNameList = [] + parallelList = order['parallel'].items() + parallelList.sort() + for key, orderList in parallelList: + print orderList + taskList, taskMetaList = flowParser.parse(orderList, stepObjArr) + self._args['forkTasks'].append(taskList) + lastTasksNameList.append(taskList[-1]['taskReferenceName']) + if taskMetaList is not None: + self._childTaskMetaList.extend(taskMetaList) + self._joinTaskObj = ParallelJoinTask(InstSeqNumber, lastTasksNameList) + + def getTaskMetaList(self): + selfTaskMetaList = super(ParallelTask, self).getTaskMetaList() + selfTaskMetaList.extend(self._childTaskMetaList) + selfTaskMetaList.extend(self._joinTaskObj.getTaskMetaList()) + return selfTaskMetaList + + def getJoinTask(self): + return self._joinTaskObj + + +class ParallelJoinTask(BaseWorkflowTask): + def __init__(self, seqNumber, joinOnList): + super(ParallelJoinTask, self).__init__( + "paralleljoin_" + str(seqNumber)) + self._type = "JOIN" + self._args['joinOn'] = joinOnList + + +def getRandString(length): + return "".join(random.choice(str("0123456789")) for i in range(length)) diff --git a/testing-scheduler/server/src/env/context/context.yaml b/testing-scheduler/server/src/env/context/context.yaml new file mode 100644 index 00000000..58ed4614 --- /dev/null +++ b/testing-scheduler/server/src/env/context/context.yaml @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- + +hosts: + node26: + name: node26 + ip: 192.168.1.26 + port: 22 + user: root + password: 123456 + + node27: + name: node27 + ip: 192.168.1.27 + port: 22 + user: root + password: 123456 + + node28: + name: node28 + ip: 192.168.1.28 + port: 22 + user: root + password: 123456 + + node29: + name: node29 + ip: 192.168.1.29 + port: 22 + user: root + password: 123456 + + node30: + name: node30 + ip: 192.168.1.30 + port: 22 + user: root + password: 123456 + + node31: + name: node31 + ip: 192.168.1.31 + port: 22 + user: root + password: 123456 + + node32: + name: node32 + ip: 192.168.1.32 + port: 22 + user: root + password: 123456 + + +cassandra: + name: node24 + ip: 192.168.1.24 + port: 22 + user: root + password: 123456 + +sprout: + name: node32 + ip: 192.168.1.32 + port: 22 + user: root + password: 123456 \ No newline at end of file diff --git a/testing-scheduler/server/src/env/service/ansible.yaml b/testing-scheduler/server/src/env/service/ansible.yaml new file mode 100644 index 00000000..7b97602d --- /dev/null +++ b/testing-scheduler/server/src/env/service/ansible.yaml @@ -0,0 +1,65 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- + +ansible: + apis: + - method: POST + name: faultload + params: + - description: call user count + name: call_user + - description: workload time + name: duration + - description: faultload time + name: fault_duration + - description: register speed + name: initial_reg_rate + - description: multiplier + name: multiplier + - description: register user count + name: reg_user + baseuri: faultload + template: + uri: ((baseuri)) + body: + call_user: ((call_user)) + duration: ((duration)) + fault_duration: ((fault_duration)) + initial_reg_rate: ((initial_reg_rate)) + multiplier: ((multiplier)) + reg_user: ((reg_user)) + + - method: POST + name: workload + params: + - description: call user count + name: call_user + - description: workload time + name: duration + - description: register speed + name: initial_reg_rate + - description: multiplier + name: multiplier + - description: register user count + name: reg_user + baseuri: workload + template: + uri: ((baseuri)) + body: + call_user: ((call_user)) + duration: ((duration)) + fault_duration: ((fault_duration)) + initial_reg_rate: ((initial_reg_rate)) + multiplier: ((multiplier)) + reg_user: ((reg_user)) + + ip: 100.64.227.222 + port: 9006 diff --git a/testing-scheduler/server/src/env/service/greet.yaml b/testing-scheduler/server/src/env/service/greet.yaml new file mode 100644 index 00000000..1b21f413 --- /dev/null +++ b/testing-scheduler/server/src/env/service/greet.yaml @@ -0,0 +1,68 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- + +greet: + ip: t-scheduler-server + port: 5312 + apis: + - + name: greet + method: GET + baseuri: greet + params: + - name: name + description: user name + template: + uri: ((baseuri))?name=((name)) + response: + result: int + next: + aa: str + bb: int + - + name: answer + method: POST + baseuri: answer + params: + - name: ping + description: param ping + template: + uri: ((baseuri)) + body: + ping: ((ping)) + + - + name: ten + method: GET + baseuri: ten + template: + uri: ((baseuri)) + + + - + name: switch + method: GET + baseuri: switch + template: + uri: ((baseuri)) + return: + - + result + + - + name: switch_2 + method: GET + baseuri: switch_2 + template: + uri: ((baseuri)) + return: + - + result diff --git a/testing-scheduler/server/src/rest/__init__.py b/testing-scheduler/server/src/rest/__init__.py new file mode 100644 index 00000000..b5139031 --- /dev/null +++ b/testing-scheduler/server/src/rest/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/testing-scheduler/server/src/rest/router.py b/testing-scheduler/server/src/rest/router.py new file mode 100644 index 00000000..b9cbd928 --- /dev/null +++ b/testing-scheduler/server/src/rest/router.py @@ -0,0 +1,485 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from flask import Flask +from flask import jsonify +from flask import request +from flask_cors import CORS +import os +import json +import time +import pyaml +import yaml +import traceback + +import src.test_parser as test_parser + + +BASE_DIR = os.path.abspath(os.path.dirname(__file__)) +TESTSUITE_DIR = os.path.join(BASE_DIR, "..", "..", "test", "test_case") +SERVICE_DIR = os.path.join(BASE_DIR, "..", "env", "service") +CONTEXT_FILE_DIR = os.path.join(BASE_DIR, "..", "env", "context", + "context.yaml") +app = Flask(__name__) +CORS(app) + + +############### +# 1. EXECUTE API +########################################################################### +@app.route("/") +def hello(): + return "Hello, World! This is a greet from parser." + SERVICE_DIR + + +@app.route("/execute/testcase", methods=['POST']) +def runTestcase(): + suiteName = request.values.get('suiteName') + caseName = request.values.get('caseName') + try: + casePath = os.path.join(TESTSUITE_DIR, suiteName, caseName) + if os.path.exists(casePath): + workflowId = test_parser.parse(casePath) + if workflowId is None or workflowId == '': + return jsonify({"code": 500, "error": "Server Error."}) + return jsonify({"code": 200, "result": {"workflowId": workflowId}}) + else: + return jsonify({"code": 300, "error": "no such test case: %s" % + (os.path.join(suiteName, caseName))}) + except BaseException, e: + return returnServerError(e) + + +@app.route("/story-content") +def getStoryContent(): + try: + story_name = request.args['story'] + service_name = request.args['service'] + storyFileDir = os.path.join("/tmp", "generate_workflow.json") + with open(storyFileDir, "r") as f: + storyContent = f.read() + except BaseException, e: + return returnServerError(e) + + result = {"code": 200, "result": + {"service": service_name, "story": story_name, + "content": storyContent}} + return jsonify(result) + + +############### +# 2. TESTCASE CRUD +########################################################################### +@app.route("/testsuite/list") +def getAllSuite(): + res = [] + id = 1 + try: + for fileName in os.listdir(TESTSUITE_DIR): + suiteInfo = {} + suiteInfo["id"] = id + suiteInfo["testsuite"] = fileName + res.append(suiteInfo) + id = id + 1 + except BaseException, e: + print e + app.logger.error(traceback.format_exc()) + return jsonify({"code": 500, "error": "Server error"}) + + return jsonify({"code": 200, "result": res}) + + +@app.route("/testsuite/content") +def getSuiteContent(): + res = [] + id = 1 + try: + suiteName = request.values.get("suiteName") + exSuitePath = os.path.join(TESTSUITE_DIR, suiteName) + if os.path.exists(exSuitePath): + for fileName in os.listdir(exSuitePath): + tcInfo = {} + tcInfo["id"] = id + tcInfo["testcase"] = fileName + res.append(tcInfo) + id = id + 1 + else: + return jsonify({"code": 300, "error": "no such test suite!"}) + except BaseException, e: + print e + app.logger.error(traceback.format_exc()) + return jsonify({"code": 500, "error": "Server error"}) + + return jsonify({"code": 200, "result": res}) + + +@app.route("/testcase/content") +def getTCContent(): + res = "" + editorRes = "" + try: + suiteName = request.values.get("suiteName") + caseName = request.values.get("caseName") + casePath = os.path.join(suiteName, caseName) + casePath = os.path.join(TESTSUITE_DIR, casePath) + if os.path.exists(casePath): + with open(casePath, "r") as f: + fileContent = f.read() + res = fileContent + editorRes = test_parser.getWebTestcase(yaml.load(res)) + else: + return jsonify({"code": 300, "error": "no such file!"}) + except BaseException, e: + print e + app.logger.error(traceback.format_exc()) + return jsonify({"code": 500, "error": "Server error"}) + + return jsonify({"code": 200, "result": + {"content": res, "editorContent": editorRes}}) + + +@app.route("/testsuite/new", methods=['POST']) +def addNewSuite(): + try: + suiteName = request.values.get("suiteName") + for fileName in os.listdir(TESTSUITE_DIR): + if fileName == suiteName: + return jsonify({"code": 300, + "error": "testsuite already exists!"}) + testSuitePath = os.path.join(TESTSUITE_DIR, suiteName) + os.mkdir(testSuitePath) + except BaseException, e: + return returnServerError(e) + + return jsonify({"code": 200, "result": "ok"}) + + +@app.route("/testsuite/delete", methods=['POST']) +def deleteSuite(): + try: + suiteName = request.values.get("suiteName") + for fileName in os.listdir(TESTSUITE_DIR): + if fileName == suiteName: + testSuitePath = os.path.join(TESTSUITE_DIR, fileName) + del_file(testSuitePath) + os.rmdir(testSuitePath) + return jsonify({"code": 200, "result": "ok"}) + except BaseException, e: + return returnServerError(e) + + return jsonify({"code": 300, "error": "no such testsuite!"}) + + +def del_file(path): + for i in os.listdir(path): + path_file = os.path.join(path, i) + if os.path.isfile(path_file): + os.remove(path_file) + else: + del_file(path_file) + + +@app.route("/testcase/new", methods=['POST']) +def createTestcase(): + try: + suiteName = request.values.get("suiteName") + caseName = request.values.get("caseName") + exSuitePath = os.path.join(TESTSUITE_DIR, suiteName) + if os.path.exists(exSuitePath): + for fileName in os.listdir(exSuitePath): + if fileName == caseName: + return jsonify({"code": 301, + "error": "testcase already exists!"}) + casePath = os.path.join(exSuitePath, caseName) + with open(casePath, "w") as f: + # the next line is a placeholder. + print f + else: + return jsonify({"code": 300, "error": "no such test suite!"}) + except BaseException, e: + return returnServerError(e) + + return jsonify({"code": 200, "result": "ok"}) + + +@app.route("/testcase/delete", methods=['POST']) +def deleteTestcase(): + try: + suiteName = request.values.get("suiteName") + caseName = request.values.get("caseName") + exSuitePath = os.path.join(TESTSUITE_DIR, suiteName) + if os.path.exists(exSuitePath): + for fileName in os.listdir(exSuitePath): + if fileName == caseName: + casePath = os.path.join(exSuitePath, caseName) + os.remove(casePath) + return jsonify({"code": 200, "result": "ok"}) + return jsonify({"code": 301, "error": "no such test case!"}) + else: + return jsonify({"code": 300, "error": "no such test suite!"}) + except BaseException, e: + return returnServerError(e) + + +@app.route("/testcase/save", methods=["POST"]) +def saveTCContent(): + try: + suiteName = request.values.get("suiteName") + caseName = request.values.get("caseName") + stepList = json.loads(request.values.get("stepList")) + subflowList = json.loads(request.values.get("subflowList")) + mainOrdersList = json.loads(request.values.get("mainOrdersList")) + jsonObj = {"stepList": stepList, "subflowList": subflowList, + "mainOrdersList": mainOrdersList} + parseData = test_parser.parseWebTestcase(jsonObj) + + casePath = os.path.join(suiteName, caseName) + casePath = os.path.join(TESTSUITE_DIR, casePath) + if os.path.exists(casePath): + with open(casePath, "w") as f: + pyaml.dump(parseData, f, safe=True) + else: + return jsonify({"code": 300, "error": "no such file!"}) + except BaseException, e: + return returnServerError(e) + + return jsonify({"code": 200, "result": "save success"}) + + +############### +# 3.1 API FOR SERVICE +############################################################ +@app.route("/service/list") +def getAllServices(): + res = [] + try: + for fileName in os.listdir(SERVICE_DIR): + serviceName = os.path.splitext(fileName)[0] + res.append(serviceName) + except BaseException, e: + return returnServerError(e) + + return jsonify({"code": 200, "result": res}) + + +@app.route("/service/content") +def getServiceContent(): + res = {} + try: + serviceName = request.values.get("serviceName") + for fileName in os.listdir(SERVICE_DIR): + if serviceName == os.path.splitext(fileName)[0]: + res["actions"] = [] + filePath = os.path.join(SERVICE_DIR, fileName) + with open(filePath, "r") as f: + content = yaml.load(f) + apisArr = content[serviceName]['apis'] + for i in range(len(apisArr)): + apisArr[i].pop("method") + apisArr[i].pop("baseuri") + res["actions"] = apisArr + except BaseException, e: + return returnServerError(e) + + if res == {}: + return jsonify({"code": 300, "error": "no such service!"}) + + return jsonify({"code": 200, "result": res}) + + +def paramTransform(paramDict): + res = [] + for (key, value) in paramDict.items(): + paramJson = {} + paramJson["name"] = key + paramJson["description"] = value["help"] + if "params" in value: + paramJson["params"] = paramTransform(value["params"]) + res.append(paramJson) + return res + + +@app.route("/service/action_response") +def actionResponse(): + res = {} + try: + serviceName = request.values.get("serviceName") + actionName = request.values.get("actionName") + for fileName in os.listdir(SERVICE_DIR): + if serviceName == os.path.splitext(fileName)[0]: + res["responseParams"] = [] + filePath = os.path.join(SERVICE_DIR, fileName) + with open(filePath, "r") as f: + content = yaml.load(f) + apisArr = content[serviceName]['apis'] + for i in range(len(apisArr)): + if actionName == apisArr[i]['name'] and ( + "response" in apisArr[i]): + res["responseParams"] = apisArr[i]["response"] + except BaseException, e: + return returnServerError(e) + if res == {}: + return jsonify({"code": 300, "error": "no such service!"}) + return jsonify({"code": 200, "result": res}) + + +############### +# 3.2 API FOR ENVIRONMENT SERVICE AND CONTEXT +########################################################################### +@app.route('/env/getAllServices') +def getAllService(): + res = [] + id = 1 + try: + for fileName in os.listdir(SERVICE_DIR): + item = {} + item['id'] = id + item['name'] = os.path.splitext(fileName)[0] + filePath = os.path.join(SERVICE_DIR, fileName) + filemt = time.localtime(os.stat(filePath).st_mtime) + item['time'] = time.strftime("%Y-%m-%d", filemt) + res.append(item) + id = id + 1 + except BaseException, e: + return returnServerError(e) + return jsonify({"code": 200, "result": res}) + + +@app.route('/env/getService') +def getService(): + try: + serviceName = request.values.get('serviceName') + serviceFile = serviceName + '.yaml' + servicePath = os.path.join(SERVICE_DIR, serviceFile) + if os.path.exists(servicePath): + with open(servicePath, "r") as f: + serviceDict = yaml.load(f) + serviceDict = serviceDict[serviceName] + return jsonify({"code": 200, "result": serviceDict}) + else: + return jsonify({"code": 300, "error": "no such service!"}) + except BaseException, e: + return returnServerError(e) + + +@app.route('/env/createService', methods=['POST']) +def createService(): + try: + name = str(request.values.get('name')) + ip = str(request.values.get('ip')) + port = int(request.values.get('port')) + apis = json.loads(request.values.get('apis')) + service = { + name: { + 'ip': ip, + 'port': port, + 'apis': apis + } + } + serviceJson = json.dumps(service, indent=True) + print serviceJson + app.logger.debug(service) + + serviceFile = name + '.yaml' + servicePath = os.path.join(SERVICE_DIR, serviceFile) + with open(servicePath, 'w') as f: + pyaml.dump(service, f, safe=True) + except BaseException, e: + return returnServerError(e) + return jsonify({"code": 200, "result": "create success!"}) + + +@app.route('/env/editService', methods=['POST']) +def editService(): + try: + oldName = str(request.values.get('oldName')) + name = str(request.values.get('newName')) + ip = str(request.values.get('ip')) + port = int(request.values.get('port')) + apis = json.loads(request.values.get('apis')) + app.logger.debug(apis) + service = { + name: { + 'ip': ip, + 'port': port, + 'apis': apis + } + } + serviceJson = json.dumps(service, indent=True) + print serviceJson + app.logger.debug(service) + + for fileName in os.listdir(SERVICE_DIR): + serviceName = os.path.splitext(fileName)[0] + if serviceName == oldName: + filePath = os.path.join(SERVICE_DIR, fileName) + os.remove(filePath) + + serviceFile = name + '.yaml' + servicePath = os.path.join(SERVICE_DIR, serviceFile) + with open(servicePath, 'w') as f: + pyaml.dump(service, f, safe=True) + except BaseException, e: + return returnServerError(e) + return jsonify({"code": 200, "result": "edit success!"}) + + +@app.route('/env/deleteService', methods=['POST']) +def deleteService(): + try: + name = str(request.values.get('serviceName')) + + for fileName in os.listdir(SERVICE_DIR): + serviceName = os.path.splitext(fileName)[0] + if serviceName == name: + filePath = os.path.join(SERVICE_DIR, fileName) + os.remove(filePath) + except BaseException, e: + return returnServerError(e) + return jsonify({"code": 200, "result": "delete success!"}) + + +@app.route('/env/getContext') +def getContext(): + try: + with open(CONTEXT_FILE_DIR, "r") as f: + fileContent = f.read() + res = fileContent + except BaseException, e: + return returnServerError(e) + return jsonify({"code": 200, "result": {"context": res}}) + + +@app.route('/env/editContext', methods=['POST']) +def editContext(): + try: + context = request.values.get("context") + test = yaml.load(context) + print test + with open(CONTEXT_FILE_DIR, "w") as f: + f.write(context) + except yaml.constructor.ConstructorError, e: + app.logger.error(traceback.format_exc()) + return jsonify({"code": 500, "error": + "context content error: not a .yaml file!"}) + except BaseException, e: + return returnServerError(e) + + return jsonify({"code": 200, "result": "edit context success!"}) +########################################################################### + + +def returnServerError(e, msg="Server Error"): + print e + app.logger.error(traceback.format_exc()) + return jsonify({"code": 500, "error": msg}) + + +if __name__ == "__main__": + app.run(host='0.0.0.0', port=5310) diff --git a/testing-scheduler/server/src/rest/test_service_demo.py b/testing-scheduler/server/src/rest/test_service_demo.py new file mode 100644 index 00000000..e6f4e38e --- /dev/null +++ b/testing-scheduler/server/src/rest/test_service_demo.py @@ -0,0 +1,77 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from flask import Flask +from flask_cors import CORS +from flask import request +from flask import jsonify +import time +import json +from random import randint + +app = Flask(__name__) +CORS(app) + + +@app.route("/greet") +def greet(): + return "hello" + + +@app.route("/answer", methods=["POST"]) +def answer(): + app.logger.debug(request.form) + app.logger.debug(request.data) + if jsonify(request.form) != {} and 'ping' in request.form: + return "answer: ping is: \"" + request.form['ping'] + "\" end." + elif request.data != "": + requestDict = json.loads(request.data) + if 'ping' in requestDict: + return "answer: the ping is: \"" + requestDict['ping'] + "\" end." + else: + return "answer ping is null" + + +@app.route("/answer2", methods=["POST"]) +def answer2(): + return "ok" + + +@app.route("/five") +def sleepFiveSeconds(): + time.sleep(5) + return "five: receive the request." + + +@app.route("/ten") +def sleepTenSeconds(): + time.sleep(10) + return "ten: receive the request." + + +@app.route("/switch") +def switchValue(): + value = randint(0, 10) + if value > 4: + return jsonify({'code': 200, 'result': 'A'}) + else: + return jsonify({'code': 200, 'result': 'B'}) + + +@app.route("/switch_2") +def switchValue_2(): + value = randint(0, 10) + if value > 4: + return jsonify({'code': 200, 'result': 'C'}) + else: + return jsonify({'code': 200, 'result': 'D'}) + + +if __name__ == "__main__": + app.run(host='0.0.0.0', port=5312, debug=True) diff --git a/testing-scheduler/server/src/step/__init__.py b/testing-scheduler/server/src/step/__init__.py new file mode 100644 index 00000000..e8198009 --- /dev/null +++ b/testing-scheduler/server/src/step/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/testing-scheduler/server/src/step/general_test_step.py b/testing-scheduler/server/src/step/general_test_step.py new file mode 100644 index 00000000..2f9e8bcc --- /dev/null +++ b/testing-scheduler/server/src/step/general_test_step.py @@ -0,0 +1,87 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from src.step.test_step import TestStep +import os +import yaml +import re + + +class GeneralTestStep(TestStep): + __step_type__ = "test" + + def __init__(self, id, name, service, action, args, context): + super(GeneralTestStep, self).__init__( + self.__step_type__, id, name, service, action, args, context) + self._stepParse() + self.action() + + def _contextTransform(self, argsDict): + for (k, v) in argsDict.items(): + if isinstance(v, str): + if re.match('^\(\(context\..*\)\)', v): + v = v[10:-2] + layers = v.split(".") + contextData = self._context + for layer in layers: + contextData = contextData[layer] + argsDict[k] = contextData + elif isinstance(v, dict): + self._contextTransform(v) + + def _stepParse(self): + self._args_temp = self._args + self._args = {} + + # transform the service config + envFilePath = os.path.join( + self._getCurrentDir(), "..", "env", + "service", self._serviceName + ".yaml") + requestParam = {} + with open(envFilePath, 'r') as f: + conf = yaml.load(f) + conf = conf[self._serviceName] + for apiItem in conf["apis"]: + if apiItem['name'] == self._serviceInterface: + interfaceConf = apiItem + if interfaceConf is None: + return + + # transform the args config + self._contextTransform(self._args_temp) + + interfaceUri = interfaceConf['baseuri'] + \ + interfaceConf['template']['uri'][11:] + interfaceUri = "http://%s:%s/%s" % ( + conf['ip'], conf['port'], interfaceUri) + requestParam['uri'] = self._uriTransform(interfaceUri) + + requestParam['method'] = interfaceConf['method'] + if requestParam["method"] == "POST": + requestParam['body'] = interfaceConf['template']['body'] + self._paramTransform(requestParam['body'], self._args_temp) + self._args['http_request'] = requestParam + + def _uriTransform(self, uri): + return re.sub("\(\(.*?\)\)", self._uriResReplace, uri) + + def _uriResReplace(self, match): + matchTrim = match.group()[2:-2] + return self._args_temp[matchTrim] + + def _paramTransform(self, argsTemplate, argsDict): + for (k, v) in argsTemplate.items(): + if isinstance(v, str): + if re.match('^\(\(.*\)\)', v): + argsTemplate[k] = argsDict[v[2:-2]] + elif isinstance(v, dict): + self._paramTransform(v, argsDict) + + def start(self): + pass diff --git a/testing-scheduler/server/src/step/monitor.py b/testing-scheduler/server/src/step/monitor.py new file mode 100644 index 00000000..6deb9e2e --- /dev/null +++ b/testing-scheduler/server/src/step/monitor.py @@ -0,0 +1,57 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import json +import os +from src.step.test_step import TestStep + + +class MonitorStep(TestStep): + __step_type__ = 'monitor' + + def __init__(self, name, service, action, args): + super(MonitorStep, self).__init__(name, service, action, args) + self._argsParse() + self.action() + + def _argsParse(self): + if self._call == "REST": + currentDirPath = os.path.dirname(os.path.abspath(__file__)) + envDirPath = os.path.abspath(os.path.join( + currentDirPath, os.pardir, os.pardir, 'env')) + envFilePath = os.path.join( + envDirPath, "%s.json" % self._service['name']) + with open(envFilePath) as f: + propDict = json.load(f) + self._args['ip'] = propDict['ip'] + self._args['port'] = propDict['port'] + self._args['api'] = "%s/%s" % ( + propDict['api_map']['workload'], self._args['command']) + exclude = {'ip', 'port', 'api', 'command', 'method'} + self._args['req_body'] = { + key: value for key, value in + self._args.items() if key not in exclude} + + def setUp(self): + print "monitor setUp" + + def uninstall(self): + print "monitor uninstall" + + def start(self): + print "monitor start...." + + def stop(self): + print "monitor stop" + + +if __name__ == "__main__": + service = {"name": "ansible", "call": "REST"} + monitor = MonitorStep( + "monitor_cpu", service, "start", **{"target": "abc:qq"}) diff --git a/testing-scheduler/server/src/step/step_manager.py b/testing-scheduler/server/src/step/step_manager.py new file mode 100644 index 00000000..8d76c67c --- /dev/null +++ b/testing-scheduler/server/src/step/step_manager.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from src.step.test_step import TestStep +import os +import sys + + +class TestStepManager(object): + def __init__(self, context): + self._context = context + + currentDirPath = os.path.dirname(os.path.abspath(__file__)) + sys.path.append(currentDirPath) + + excludeFiles = ('__init__.py', 'step_manager.py', 'test_step.py') + for fileName in os.listdir(currentDirPath): + if os.path.isfile(os.path.join(currentDirPath, fileName)) and \ + os.path.splitext(fileName)[1] == '.py' and \ + fileName not in excludeFiles: + __import__(os.path.splitext(fileName)[0]) + + def getStepObj(self, type, id, name, service, action, args): + for subclass in TestStep.__subclasses__(): + if type == subclass.__step_type__: + return subclass(id, name, service, action, args, self._context) + + +if __name__ == "__main__": + tsMgr = TestStepManager() + args = {'command': 'greet', 'method': 'POST', 'args': {'name': 'leo'}} + stepObj = tsMgr.getStepObj('test', 1, 'test_cpu', { + 'name': 'greet', 'call': 'REST'}, 'start', args) + print stepObj + print stepObj.__class__.__mro__ diff --git a/testing-scheduler/server/src/step/test_step.py b/testing-scheduler/server/src/step/test_step.py new file mode 100644 index 00000000..363c4800 --- /dev/null +++ b/testing-scheduler/server/src/step/test_step.py @@ -0,0 +1,56 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import os + + +class TestStep(object): + def __init__(self, type, id, name, service, action, args, context): + self._type = type + self._id = id + self._name = name + self._serviceName = service['name'] + self._serviceInterface = service['interface'] + self._action = action + self._args = args + self._context = context + + def getId(self): + return self._id + + def getName(self): + return self._name + + def getServiceName(self): + return self._serviceName + + def getCallFunction(self): + return self._callType + + def getArgs(self): + return self._args + + def action(self): + f = getattr(self, self._action) + f() + + def _argsParse(self): + pass + + def _getCurrentDir(self): + return os.path.dirname(__file__) + + def __str__(self): + return str(self.__dict__) + + +if __name__ == "__main__": + args = {'command': 'start'} + stepObj = TestStep('test_cpu', 'ansible', 'REST', **args) + print stepObj diff --git a/testing-scheduler/server/src/step/workload.py b/testing-scheduler/server/src/step/workload.py new file mode 100644 index 00000000..265b9a6b --- /dev/null +++ b/testing-scheduler/server/src/step/workload.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import json +import os +from src.step.test_step import TestStep + + +class WorkloadStep(TestStep): + __step_type__ = 'workload' + + def __init__(self, id, name, service, action, args): + super(WorkloadStep, self).__init__( + self.__step_type__, id, name, service, action, args) + self._argsParse() + self._action() + + def _argsParse(self): + if self._callType == "REST": + currentDirPath = os.path.dirname(os.path.abspath(__file__)) + envDirPath = os.path.abspath( + os.path.join(currentDirPath, os.pardir, os.pardir, 'env')) + envFilePath = os.path.join( + envDirPath, "%s.json" % self._service['name']) + with open(envFilePath) as f: + propDict = json.load(f) + self._args['ip'] = propDict['ip'] + self._args['port'] = propDict['port'] + self._args['api'] = "%s/%s" % ( + propDict['api_map']['workload'], self._args['command']) + exclude = {'ip', 'port', 'api', 'command', 'method'} + self._args['req_body'] = { + key: value for key, value in + self._args.items() if key not in exclude} + + def _start(self): + print "workload start" + + def _stop(self): + print "workload stop" diff --git a/testing-scheduler/server/src/test_parser.py b/testing-scheduler/server/src/test_parser.py new file mode 100644 index 00000000..7b471517 --- /dev/null +++ b/testing-scheduler/server/src/test_parser.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python + +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import click +import os +import yaml +import json +import collections +from src.step.step_manager import TestStepManager +from src.conductor_processor.workflow import WorkflowFile +from conductorclient.run_new_workflow import WorkflowMgr + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +CONDUCTOR_SERVER_ADDR = "http://conductor_conductor-server_1:8080" +STORE_TASK_PATH = "/tmp/generate_task.json" +STORE_WF_PATH = "/tmp/generate_workflow.json" + + +@click.command() +@click.option("--filepath", help="file path of test case") +def cmdParse(filepath): + parse(filepath) + + +def parse(filepath): + filePrefix, fileName = os.path.split(filepath) + print '------------ start to parse the test case:' + \ + '%s ----------------' % fileName + with open(filepath) as f: + yaml_file = yaml.load(f) + parseTestcase(yaml_file['schema'], fileName) + + workflowId = runWorkFlow() + print '------------------- parse executes end -------------------------' + + return workflowId + + +def parseTestcase(schema, tcName='testcase0'): + if schema is None: + return parseLog(False, reason='schema not found.') + steps = schema['steps'] + if steps is None: + return parseLog(False, reason='steps is invalid.') + flows = schema['flows'] + if flows is None: + return parseLog(False, reasion='flows is invalid.') + # steps is a list, step is dict. no json here. + # steps = sorted(steps, sortById) + + # load context + contextDict = {} + contextDir = os.path.join(BASE_DIR, "env", "context", "context.yaml") + with open(contextDir, "r") as f: + contextDict = yaml.load(f) + # + testStepMgr = TestStepManager(contextDict) + + stepObjArr = [] + for step in steps: + if 'args' not in step: + step['args'] = {} + # type and action can be extended, default couple is 'test' & 'start'. + if 'type' not in step: + step['type'] = 'test' + step['action'] = 'start' + + stepObj = testStepMgr.getStepObj( + step['type'], step['id'], step['name'], step['service'], + step['action'], step['args']) + stepObjArr.append(stepObj) + + # generate workflow by 'flow' and 'step' + tcName = os.path.splitext(tcName)[0] + wfFileObj = WorkflowFile(tcName) + workflowDict, taskMetaList = wfFileObj.generateMetaData(flows, stepObjArr) + + with open(STORE_TASK_PATH, 'w') as f: + f.write(json.dumps({'task_group': taskMetaList}, indent=True)) + with open(STORE_WF_PATH, 'w') as f: + f.write(json.dumps(workflowDict, indent=True)) + + +def parseWebTestcase(webTestcase): + print 'parseWebTestcase----------------------------' + + stepList = webTestcase['stepList'] + mainOrdersList = webTestcase['mainOrdersList'] + subflowList = webTestcase['subflowList'] + + parseData = collections.OrderedDict() + parseData['schema'] = collections.OrderedDict() + parseData['schema']['steps'] = [] + parseData['schema']['flows'] = [] + + parseStepList = parseData['schema']['steps'] + parseFlowList = parseData['schema']['flows'] + stepIndexDict = {} + # parse stepList + for index in range(len(stepList)): + stepItem = stepList[index] + parseStep = collections.OrderedDict() + + parseStep['id'] = index + 1 + parseStep['name'] = stepItem['name'] + parseStep['service'] = collections.OrderedDict() + parseStep['service']['name'] = stepItem['service'] + parseStep['service']['interface'] = stepItem['action'] + parseStep['action'] = 'start' + parseStep['args'] = {} + for paramItem in stepItem['params']: + parseStep['args'][paramItem['key']] = transParamString( + paramItem['value']) + + parseStepList.append(parseStep) + stepIndexDict[parseStep['name']] = parseStep['id'] + # parse flows + # parse mainflow + print stepIndexDict + typeDict = {1: 'normal', 2: 'switch', 3: 'parallel'} + mainFlow = collections.OrderedDict() + mainFlow['name'] = 'main' + mainFlow['orders'] = [] + mainFlow['orders'] = parseOrderList( + mainOrdersList, stepIndexDict, typeDict) + parseFlowList.append(mainFlow) + + # parse subflow + for subflowItem in subflowList: + replaceSubflow = collections.OrderedDict() + replaceSubflow['name'] = subflowItem['name'] + replaceSubflow['orders'] = parseOrderList( + subflowItem['orderList'], stepIndexDict, typeDict) + parseFlowList.append(replaceSubflow) + + print 'END parseWebTestcase----------------------------' + return parseData + + +# parse orderlist from web edition to server edition +def parseOrderList(orderList, stepIndexDict, typeDict): + replaceList = [] + for orderItem in orderList: + replaceOrder = collections.OrderedDict() + orderType = typeDict[orderItem['type']] + replaceOrder['type'] = orderType + if orderType == 'normal': + stepId = stepIndexDict[orderItem['step']] + replaceOrder['step'] = stepId + elif orderType == 'switch': + replaceOrder['value'] = orderItem['value'] + replaceOrder['cases'] = collections.OrderedDict() + for caseItem in orderItem['cases']: + caseValue = caseItem['value'] + caseOrderType = caseItem['orderType'] + caseOrderValue = caseItem['orderValue'] + if caseOrderType == "step": + orderInCase = collections.OrderedDict() + orderInCase['type'] = 'normal' + orderInCase['step'] = stepIndexDict[caseOrderValue] + replaceOrder['cases'][caseValue] = [orderInCase] + else: + replaceOrder['cases'][caseValue] = caseOrderValue + else: + replaceOrder['parallel'] = collections.OrderedDict() + pIndex = 1 + for branchItem in orderItem['branches']: + pKey = 'p' + str(pIndex) + branchOrderType = branchItem['orderType'] + branchOrderValue = branchItem['orderValue'] + if branchOrderType == "step": + replaceBranchItem = collections.OrderedDict() + replaceBranchItem['type'] = 'normal' + replaceBranchItem['step'] = stepIndexDict[branchOrderValue] + replaceOrder['parallel'][pKey] = [replaceBranchItem] + else: + replaceOrder['parallel'][pKey] = branchOrderValue + pIndex += 1 + replaceList.append(replaceOrder) + return replaceList + + +def transParamString(val): + if type(val) != str: + return val + if '.' not in val: + if val.isdigit(): + return int(val) + try: + f = float(val) + return f + except ValueError: + return val + + +def getWebTestcase(originTcDict): + print "getWebTestcase----------------------------------" + webTcDict = { + "stepList": [], + "mainOrdersList": [], + "subflowList": [] + } + stepList = webTcDict['stepList'] + subflowList = webTcDict['subflowList'] + if originTcDict is None: + return webTcDict + originContent = originTcDict['schema'] + originSteps = originContent['steps'] + stepIndexDict = {} + # transform steps to stepList + for stepItem in originSteps: + replaceStep = {} + replaceStep['name'] = stepItem['name'] + replaceStep['service'] = stepItem['service']['name'] + replaceStep['action'] = stepItem['service']['interface'] + replaceStep['params'] = [] + if 'args' in stepItem: + for (key, value) in stepItem['args'].items(): + replaceParam = {} + replaceParam['key'] = key + replaceParam['value'] = value + replaceStep['params'].append(replaceParam) + stepList.append(replaceStep) + stepIndexDict[stepItem['id']] = stepItem['name'] + + # transform main flow + originFlows = originContent['flows'] + originMainflow = {} + for flowIndex in range(len(originFlows)): + flowItem = originFlows[flowIndex] + if flowItem['name'] == 'main': + originMainflow = flowItem + originFlows.pop(flowIndex) + break + typeDict = {'normal': 1, 'switch': 2, 'parallel': 3} + webTcDict['mainOrdersList'] = getOrderList( + originMainflow['orders'], stepIndexDict, typeDict) + + # transform subflows + for originSubflow in originFlows: + replaceSubflow = {} + replaceSubflow['name'] = originSubflow['name'] + replaceSubflow['orderList'] = getOrderList( + originSubflow['orders'], stepIndexDict, typeDict) + subflowList.append(replaceSubflow) + + # return web edition of testcase + print "END getWebTestcase----------------------------------" + return webTcDict + + +def getOrderList(originOrderList, stepIndexDict, typeDict): + replaceOrderList = [] + for orderItem in originOrderList: + replaceOrderItem = {} + orderType = orderItem['type'] + replaceOrderItem['type'] = typeDict[orderType] + if orderType == 'normal': + stepName = stepIndexDict[orderItem['step']] + replaceOrderItem['step'] = stepName + elif orderType == 'switch': + replaceOrderItem['value'] = orderItem['value'] + replaceOrderItem['cases'] = [] + for (caseValue, ordersInCase) in orderItem['cases'].items(): + replaceCase = {} + replaceCase['value'] = caseValue + if type(ordersInCase) == list: + replaceCase['orderType'] = 'step' + caseStepName = stepIndexDict[ordersInCase[0]['step']] + replaceCase['orderValue'] = caseStepName + else: + replaceCase['orderType'] = 'flow' + replaceCase['orderValue'] = ordersInCase + replaceOrderItem['cases'].append(replaceCase) + else: + replaceOrderItem['branches'] = [] + for paraIndex in orderItem['parallel']: + paraItem = orderItem['parallel'][paraIndex] + replaceBranch = {} + if type(paraItem) == list: + replaceBranch['orderType'] = 'step' + branchStepName = stepIndexDict[paraItem[0]['step']] + replaceBranch['orderValue'] = branchStepName + else: + replaceBranch['orderType'] = 'flow' + replaceBranch['orderValue'] = paraItem + replaceOrderItem['branches'].append(replaceBranch) + replaceOrderList.append(replaceOrderItem) + + return replaceOrderList + + +def runWorkFlow(): + wfMgr = WorkflowMgr(CONDUCTOR_SERVER_ADDR) + wfMgr.setTaskDefFromFile(STORE_TASK_PATH) + wfMgr.setWorkflowFromFile(STORE_WF_PATH) + inputParam = {'input': 'fake'} + workflowId = wfMgr.startWorkflow(inputParam) + return workflowId + + +def parseLog(flag, **msg): + return {'result': flag, 'message': msg} + + +if __name__ == "__main__": + cmdParse() diff --git a/testing-scheduler/server/test/__init__.py b/testing-scheduler/server/test/__init__.py new file mode 100644 index 00000000..e8198009 --- /dev/null +++ b/testing-scheduler/server/test/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/testing-scheduler/server/test/test_case/logic/tc_logic_00.yaml b/testing-scheduler/server/test/test_case/logic/tc_logic_00.yaml new file mode 100644 index 00000000..6a4eeea9 --- /dev/null +++ b/testing-scheduler/server/test/test_case/logic/tc_logic_00.yaml @@ -0,0 +1,30 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +# description: test conductor basic logic +# this testcase contains a single step for scheduling. + +schema: + steps: + - + id: 1 + name: make_a_greet + service: + name: greet + interface: greet + args: + name: leo + flows: + - + name: main + orders: + - + type: normal + step: 1 \ No newline at end of file diff --git a/testing-scheduler/server/test/test_case/logic/tc_logic_01.yaml b/testing-scheduler/server/test/test_case/logic/tc_logic_01.yaml new file mode 100644 index 00000000..3f8f4f41 --- /dev/null +++ b/testing-scheduler/server/test/test_case/logic/tc_logic_01.yaml @@ -0,0 +1,73 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +# description: test conductor basic logic +# 5 steps while only 2 in them will be executed. + +schema: + steps: + - + id: 1 + name: make_a_greet + service: + name: greet + interface: greet + args: + name: leo + + - + id: 2 + name: response_the_greet + service: + name: greet + interface: answer + args: + ping: ping_from_leo + + - + id: 3 + name: make_a_greet_3 + service: + name: greet + interface: greet + args: + name: leo_@ + + - + id: 4 + name: make_a_greet_4 + service: + name: greet + interface: greet + args: + name: leo_@ + + - + id: 5 + name: make_a_greet_5 + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + + flows: + - + name: main + orders: + - + type: normal + step: 1 + - + type: normal + step: 2 diff --git a/testing-scheduler/server/test/test_case/logic/tc_logic_02.yaml b/testing-scheduler/server/test/test_case/logic/tc_logic_02.yaml new file mode 100644 index 00000000..ce7197d1 --- /dev/null +++ b/testing-scheduler/server/test/test_case/logic/tc_logic_02.yaml @@ -0,0 +1,71 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +# description: test conductor basic logic of switch +# switch will determine the next case to execute based +# on the "value", and "value" can be assigned by the +# previous step's output. +# The testcase below is an example. +# To referring the step's output, using ((x.y)). +# x is the id of the step, +# y is the output key of the step. + +schema: + steps: + - + id: 1 + name: choose_greet_type + type: test + service: + name: greet + interface: switch + action: start + + - + id: 2 + name: response_the_greet + type: test + service: + name: greet + interface: answer + action: start + args: + ping: ping_from_leo + + - + id: 3 + name: make_a_greet_2 + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + flows: + - + name: main + orders: + - + type: normal + step: 1 + - + type: switch + value: ((1.result)) + cases: + A: + - + type: normal + step: 2 + B: + - + type: normal + step: 3 diff --git a/testing-scheduler/server/test/test_case/logic/tc_logic_03.yaml b/testing-scheduler/server/test/test_case/logic/tc_logic_03.yaml new file mode 100644 index 00000000..2cb89430 --- /dev/null +++ b/testing-scheduler/server/test/test_case/logic/tc_logic_03.yaml @@ -0,0 +1,86 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +# description: test useless sub workflow +# In this testcase, there is a subflow whose name +# is 'flow-1' which will not be executed because no +# other flow refers to it. +# This testcase is used to test that besides main flow, +# whether subflow can be parsed without exception. + +schema: + steps: + - + id: 1 + name: choose_greet_type + type: test + service: + name: greet + interface: switch + action: start + + - + id: 2 + name: response_the_greet + type: test + service: + name: greet + interface: answer + action: start + args: + ping: ping_from_leo + + - + id: 3 + name: make_a_greet_2 + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + - + id: 4 + name: make_a_greet_3 + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + flows: + - + name: main + orders: + - + type: normal + step: 1 + - + type: switch + value: ((1.result)) + cases: + A: + - + type: normal + step: 2 + B: + - + type: normal + step: 3 + - + name: flow-1 + orders: + - + type: normal + step: 4 diff --git a/testing-scheduler/server/test/test_case/logic/tc_logic_04.yaml b/testing-scheduler/server/test/test_case/logic/tc_logic_04.yaml new file mode 100644 index 00000000..67847ff3 --- /dev/null +++ b/testing-scheduler/server/test/test_case/logic/tc_logic_04.yaml @@ -0,0 +1,70 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +# description: test conductor basic logic of sub flow. + +schema: + steps: + - + id: 1 + name: choose_greet_type + type: test + service: + name: greet + interface: switch + action: start + + - + id: 2 + name: response_the_greet + type: test + service: + name: greet + interface: answer + action: start + args: + ping: ping_from_leo + + - + id: 3 + name: make_a_greet_2 + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + flows: + - + name: main + orders: + - + type: normal + step: 1 + - + type: switch + value: ((1.result)) + cases: + A: flow-1 + B: flow-2 + - + name: flow-1 + orders: + - + type: normal + step: 2 + - + name: flow-2 + orders: + - + type: normal + step: 3 diff --git a/testing-scheduler/server/test/test_case/logic/tc_logic_05.yaml b/testing-scheduler/server/test/test_case/logic/tc_logic_05.yaml new file mode 100644 index 00000000..ac0f81c9 --- /dev/null +++ b/testing-scheduler/server/test/test_case/logic/tc_logic_05.yaml @@ -0,0 +1,100 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +# description: test conductor basic logic of switch. +# switch nested in switch by using subflow. + +schema: + steps: + - + id: 1 + name: choose_greet_type + type: test + service: + name: greet + interface: switch + action: start + + - + id: 2 + name: response_the_greet + type: test + service: + name: greet + interface: answer + action: start + args: + ping: ping_from_leo + + - + id: 3 + name: choose_greet_type_2 + type: test + service: + name: greet + interface: switch_2 + action: start + + - + id: 4 + name: make_a_greet + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + - + id: 5 + name: response_the_greet_2 + type: test + service: + name: greet + interface: answer + action: start + args: + ping: ping_from_leo_2 + + flows: + - + name: main + orders: + - + type: normal + step: 1 + - + type: switch + value: ((1.result)) + cases: + A: + - + type: normal + step: 2 + B: flow-1 + - + name: flow-1 + orders: + - + type: normal + step: 3 + - + type: switch + value: ((3.result)) + cases: + C: + - + type: normal + step: 4 + D: + - + type: normal + step: 5 diff --git a/testing-scheduler/server/test/test_case/logic/tc_logic_06.yaml b/testing-scheduler/server/test/test_case/logic/tc_logic_06.yaml new file mode 100644 index 00000000..7c14d012 --- /dev/null +++ b/testing-scheduler/server/test/test_case/logic/tc_logic_06.yaml @@ -0,0 +1,80 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +# description: test conductor basic logic of parallel + +schema: + steps: + - + id: 1 + name: choose_greet_type + type: test + service: + name: greet + interface: switch + action: start + + - + id: 2 + name: response_the_greet + type: test + service: + name: greet + interface: answer + action: start + args: + ping: ping_from_leo + + - + id: 3 + name: make_a_greet + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + - + id: 4 + name: make_a_greet_2 + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + flows: + - + name: main + orders: + - + type: normal + step: 1 + - + type: parallel + parallel: + p1: flow-1 + p2: + - + type: normal + step: 4 + - + name: flow-1 + orders: + - + type: normal + step: 2 + - + type: normal + step: 3 diff --git a/testing-scheduler/server/test/test_case/logic/tc_logic_07.yaml b/testing-scheduler/server/test/test_case/logic/tc_logic_07.yaml new file mode 100644 index 00000000..5fa71227 --- /dev/null +++ b/testing-scheduler/server/test/test_case/logic/tc_logic_07.yaml @@ -0,0 +1,100 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +# description: test conductor basic logic of parallel + switch + +schema: + steps: + - + id: 1 + name: make_a_greet + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + - + id: 2 + name: choose_greet_type + type: test + service: + name: greet + interface: switch + action: start + + - + id: 3 + name: response_the_greet + type: test + service: + name: greet + interface: answer + action: start + args: + ping: ping_from_leo + + - + id: 4 + name: make_a_greet + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + - + id: 5 + name: make_a_greet_2 + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + + flows: + - + name: main + orders: + - + type: normal + step: 1 + - + type: parallel + parallel: + p1: flow-1 + p2: + - + type: normal + step: 5 + - + name: flow-1 + orders: + - + type: normal + step: 2 + - + type: switch + value: ((2.result)) + cases: + A: + - + type: normal + step: 3 + B: + - + type: normal + step: 4 \ No newline at end of file diff --git a/testing-scheduler/server/test/test_case/logic/tc_logic_08.yaml b/testing-scheduler/server/test/test_case/logic/tc_logic_08.yaml new file mode 100644 index 00000000..e5da14f2 --- /dev/null +++ b/testing-scheduler/server/test/test_case/logic/tc_logic_08.yaml @@ -0,0 +1,114 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +# description: test conductor basic logic of parallel. +# parallel nested in parallel using subflow + +schema: + steps: + - + id: 1 + name: make_a_greet + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + - + id: 2 + name: choose_greet_type + type: test + service: + name: greet + interface: switch + action: start + + - + id: 3 + name: response_the_greet + type: test + service: + name: greet + interface: answer + action: start + args: + ping: ping_from_leo + + - + id: 4 + name: make_a_greet + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + - + id: 5 + name: make_a_greet_2 + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + - + id: 6 + name: make_a_greet_3 + type: test + service: + name: greet + interface: greet + action: start + args: + name: leo_@ + + + flows: + - + name: main + orders: + - + type: normal + step: 1 + - + type: parallel + parallel: + p1: flow-p1 + p2: + - + type: normal + step: 6 + - + name: flow-p1 + orders: + - + type: normal + step: 2 + - + type: parallel + parallel: + p1: + - + type: normal + step: 3 + p2: + - + type: normal + step: 4 + - + type: normal + step: 5 \ No newline at end of file diff --git a/testing-scheduler/server/test/test_case/logic_in_out/tc_cxt_01.yaml b/testing-scheduler/server/test/test_case/logic_in_out/tc_cxt_01.yaml new file mode 100644 index 00000000..583bc1c3 --- /dev/null +++ b/testing-scheduler/server/test/test_case/logic_in_out/tc_cxt_01.yaml @@ -0,0 +1,47 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +# description: test use param of the context +# we can using the context param just like environment variable by +# writting ((context.a.b.c)). +# 'context' is a constant string, +# a, b, c is the key in the context yaml file (src/env/context/context.yaml). + +schema: + steps: + - + id: 1 + name: make_a_greet + service: + name: greet + interface: greet + args: + name: leo + + - + id: 2 + name: response_the_greet + service: + name: greet + interface: answer + args: + ping: ((context.sprout.name)) + + flows: + - + name: main + orders: + - + type: normal + step: 1 + - + type: normal + step: 2 + context: clearwater-conf.yaml diff --git a/testing-scheduler/server/test/test_case/logic_in_out/tc_io_01.yaml b/testing-scheduler/server/test/test_case/logic_in_out/tc_io_01.yaml new file mode 100644 index 00000000..b5ce203f --- /dev/null +++ b/testing-scheduler/server/test/test_case/logic_in_out/tc_io_01.yaml @@ -0,0 +1,63 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +# description: test switch value used step-1's output. +# to referring the step's output, using ((x.y)). +# x is the id of the step, +# y is the output key of the step. + +schema: + steps: + - + id: 1 + name: get-switch + service: + name: greet + interface: switch + args: + name: leo + + - + id: 2 + name: greet1 + service: + name: greet + interface: answer + args: + ping: ping_from_Leo + + - + id: 3 + name: greet2 + service: + name: greet + interface: answer + args: + ping: ping_from_Leo_2 + + flows: + - + name: main + orders: + - + type: normal + step: 1 + - + type: switch + value: ((1.result)) + cases: + A: + - + type: normal + step: 2 + B: + - + type: normal + step: 3 \ No newline at end of file diff --git a/testing-scheduler/server/test/test_case/logic_in_out/tc_io_02.yaml b/testing-scheduler/server/test/test_case/logic_in_out/tc_io_02.yaml new file mode 100644 index 00000000..804a6791 --- /dev/null +++ b/testing-scheduler/server/test/test_case/logic_in_out/tc_io_02.yaml @@ -0,0 +1,60 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +# description: test switch and step-2 used step-1's output + +schema: + steps: + - + id: 1 + name: get-switch + service: + name: greet + interface: switch + args: + name: leo + + - + id: 2 + name: greet1 + service: + name: greet + interface: answer + args: + ping: ((1.result)) + + - + id: 3 + name: greet2 + service: + name: greet + interface: answer + args: + ping: ping_from_Leo_2 + + flows: + - + name: main + orders: + - + type: normal + step: 1 + - + type: switch + value: ((1.result)) + cases: + A: + - + type: normal + step: 2 + B: + - + type: normal + step: 3 \ No newline at end of file diff --git a/testing-scheduler/server/test/test_case/suite_test/tc_suitetest_00.yaml b/testing-scheduler/server/test/test_case/suite_test/tc_suitetest_00.yaml new file mode 100644 index 00000000..06f8c4ef --- /dev/null +++ b/testing-scheduler/server/test/test_case/suite_test/tc_suitetest_00.yaml @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +schema: + steps: + - + id: 1 + name: suitetest_00 + service: + name: greet + interface: ten + args: + name: leo + flows: + - + name: main + orders: + - + type: normal + step: 1 \ No newline at end of file diff --git a/testing-scheduler/server/test/test_case/suite_test/tc_suitetest_01.yaml b/testing-scheduler/server/test/test_case/suite_test/tc_suitetest_01.yaml new file mode 100644 index 00000000..0189e7af --- /dev/null +++ b/testing-scheduler/server/test/test_case/suite_test/tc_suitetest_01.yaml @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +schema: + steps: + - + id: 1 + name: suitetest_01 + service: + name: greet + interface: ten + args: + name: leo + flows: + - + name: main + orders: + - + type: normal + step: 1 \ No newline at end of file diff --git a/testing-scheduler/server/test/test_case/suite_test/tc_suitetest_02.yaml b/testing-scheduler/server/test/test_case/suite_test/tc_suitetest_02.yaml new file mode 100644 index 00000000..bc689248 --- /dev/null +++ b/testing-scheduler/server/test/test_case/suite_test/tc_suitetest_02.yaml @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +schema: + steps: + - + id: 1 + name: suitetest_01 + service: + name: greet + interface: greet + args: + name: leo + flows: + - + name: main + orders: + - + type: normal + step: 1 \ No newline at end of file diff --git a/utils/env_prepare/config_prepare.sh b/utils/env_prepare/config_prepare.sh index b13b5a02..8de60438 100644 --- a/utils/env_prepare/config_prepare.sh +++ b/utils/env_prepare/config_prepare.sh @@ -155,7 +155,7 @@ if [[ ${INSTALLER_TYPE} != "" ]]; then echo "export OS_CACERT=${OS_CACERT}" >> ${OPENRC} cat ${OPENRC} else - error "Couldn't find openstack cacert file: ${OS_CACERT}, please check if the it's been properly provided." + info "Couldn't find openstack cacert file: ${OS_CACERT}, please check if the it's been properly provided." fi else error "Couldn't find openstack rc file: ${OPENRC}, please check if the it's been properly provided." @@ -166,6 +166,6 @@ if [[ ${INSTALLER_TYPE} != "" ]]; then if [[ -f "/tmp/id_rsa" ]]; then info "Path of ssh key file for openstack nodes is /tmp/id_rsa" else - error "Couldn't find the ssh key file for openstack nodes. If you are using user/pwd in pod.yaml, please ignore." + info "Couldn't find the ssh key file for openstack nodes. If you are using user/pwd in pod.yaml, please ignore." fi fi diff --git a/utils/infra_setup/heat/common.py b/utils/infra_setup/heat/common.py index a0d6d83c..f0512b0f 100755 --- a/utils/infra_setup/heat/common.py +++ b/utils/infra_setup/heat/common.py @@ -66,14 +66,16 @@ def get_session_auth(): def get_session(): auth = get_session_auth() - try: - cacert = os.environ['OS_CACERT'] - except KeyError: - return session.Session(auth=auth) - else: - insecure = os.getenv('OS_INSECURE', '').lower() == 'true' - cacert = False if insecure else cacert + if os.getenv('OS_INSECURE', '').lower() == 'true': + cacert = False return session.Session(auth=auth, verify=cacert) + else: + try: + cacert = os.environ['OS_CACERT'] + except KeyError: + return session.Session(auth=auth) + else: + return session.Session(auth=auth, verify=cacert) def get_endpoint(service_type, endpoint_type='publicURL'): diff --git a/utils/k8s_setup/__init__.py b/utils/k8s_setup/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/utils/k8s_setup/golang_install.sh b/utils/k8s_setup/golang_install.sh new file mode 100644 index 00000000..06c54cee --- /dev/null +++ b/utils/k8s_setup/golang_install.sh @@ -0,0 +1,100 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +usage="Script to install and config golang of specific version. + +usage: + bash $(basename "$0") [-h|--help] [-v|--version ] [--debug] + +where: + -h|--help show the help text + -v|--version input the version of golang + --debug debug option switch +examples: + $(basename "$0") -v 1.10.3" + +# Debug option +redirect="/dev/null" + +# Process input variables +while [[ $# > 0 ]] + do + key="$1" + case $key in + -h|--help) + echo "$usage" + exit 0 + shift + ;; + -v|--version) + GOLANG_VERSION="$2" + shift + ;; + --debug) + redirect="/dev/stdout" + shift + ;; + *) + echo "unkown option $1 $2" + exit 1 + ;; + esac + shift +done + +#set -e + +echo "=======Downloading golang of version: ${GOLANG_VERSION}========" + +if [[ -f go${GOLANG_VERSION}.linux-amd64.tar.gz ]]; then + rm go${GOLANG_VERSION}.linux-amd64.tar.gz +fi +curl -O https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-amd64.tar.gz >${redirect} + +echo "Installing golang of version: ${GOLANG_VERSION}" +if [[ -d /usr/local/go ]]; then + rm -rf /usr/local/go +fi + +tar -C /usr/local -xzf go${GOLANG_VERSION}.linux-amd64.tar.gz >${redirect} + +if [[ -d $HOME/go ]]; then + rm -rf ${HOME}/go + mkdir ${HOME}/go + mkdir ${HOME}/go/bin + mkdir ${HOME}/go/src +else + mkdir ${HOME}/go + mkdir ${HOME}/go/bin + mkdir ${HOME}/go/src +fi + +echo "Adding golang env to ~/.bashrc" +GOROOT=/usr/local/go +GOPATH=${HOME}/go + +if [[ $(cat ${HOME}/.bashrc | grep GOROOT) ]]; then + echo "golang env alreay in ${HOME}/.bashrc" +else + cat <> ${HOME}/.bashrc + +export GOROOT=/usr/local/go +export GOPATH=${HOME}/go +export PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin +EOF +fi + +export GOROOT=/usr/local/go +export GOPATH=${HOME}/go +export PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin + +echo "Running go version command:" +go version + +echo "=======Installation of golang-${GOLANG_VERSION} complete=======" + diff --git a/utils/k8s_setup/k8s_config_pre.sh b/utils/k8s_setup/k8s_config_pre.sh new file mode 100644 index 00000000..f41ba78d --- /dev/null +++ b/utils/k8s_setup/k8s_config_pre.sh @@ -0,0 +1,66 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2018 Huawei Tech and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +K8S_CONFIG="/tmp/k8s_conig" + +usage="Script to prepare kubenetes test configurations. + +usage: + bash $(basename "$0") [-h|--help] [-i|--installer ] [-c|--config ] + +where: + -h|--help show the help text + -i|--installer specify the installer for the system to be monitored + + one of the following: + (compass) +examples: + $(basename "$0") -i compass" + + +info () { + logger -s -t "BOTTLENECKS INFO" "$*" +} + +error () { + logger -s -t "BOTTLENECKS ERROR" "$*" + exit 1 +} + +# Process input variables +while [[ $# > 0 ]] + do + key="$1" + case $key in + -h|--help) + echo "$usage" + exit 0 + shift + ;; + -i|--installer) + INSTALLER_TYPE="$2" + shift + ;; + -c|--config) + K8S_CONFIG="$2" + shift + ;; + *) + error "unkown input options $1 $2" + exit 1 + ;; + esac + shift +done + +if [[ ${INSTALLER_TYPE} == 'compass' ]]; then + sshpass -p root scp root@192.16.1.222:~/.kube/config ${K8S_CONFIG} +else + echo "BOTTLENECKS EROOR: unrecognized installer" +fi diff --git a/utils/k8s_setup/k8s_env.sh b/utils/k8s_setup/k8s_env.sh new file mode 100644 index 00000000..855dea2f --- /dev/null +++ b/utils/k8s_setup/k8s_env.sh @@ -0,0 +1,13 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +export GOROOT=/usr/local/go +export GOPATH=${HOME}/go +export PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin + diff --git a/utils/k8s_setup/k8s_utils.py b/utils/k8s_setup/k8s_utils.py new file mode 100644 index 00000000..7195bf23 --- /dev/null +++ b/utils/k8s_setup/k8s_utils.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import os +import utils.logger as log +from kubernetes import client, watch + + +LOG = log.Logger(__name__).getLogger() +INSTALLER_TYPE = os.getenv("INSTALLER_TYPE") + + +def get_config_path(INSTALLER_TYPE=None, CONFIG_PATH="/tmp/k8s_config"): + if INSTALLER_TYPE: + CMD = "bash k8s_config_pre.sh -i " + INSTALLER_TYPE + \ + " -c " + CONFIG_PATH + LOG.info("Executing command: " + CMD) + CONFIG_PATH = os.popen(CMD) + else: + if not os.path.exists(CONFIG_PATH): + raise Exception("Must at least specify the path \ +of k8s config!") + return CONFIG_PATH + + +def get_core_api(version='v1'): + if version.lower() == 'v1': + API = client.CoreV1Api() + LOG.info(API) + else: + raise Exception("Must input a validate verison!") + return API + + +def watch_namespace(namespace, count=3, stop=None, request_timeout=0): + w = watch.Watch() + LOG.debug("Watch object generated: {}".format(w)) + LOG.info("Watch stream generated: {}".format( + w.stream(namespace, _request_timeout=request_timeout))) + for event in w.stream(namespace, _request_timeout=request_timeout): + LOG.info("Event: %s %s" % + (event['type'], event['object'].metadata.name)) + if event['object'].metadata.name == stop: + LOG.info("Namesapce successfully added.\n") + w.stop() + count -= 1 + if not count: + LOG.info("Ended.\n") + w.stop() diff --git a/utils/k8s_setup/kubectl_install.sh b/utils/k8s_setup/kubectl_install.sh new file mode 100644 index 00000000..14f97f2b --- /dev/null +++ b/utils/k8s_setup/kubectl_install.sh @@ -0,0 +1,33 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2018 Huawei Tech and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +OS_TYPE=$(uname -a) +OS_UBUNTU=$(echo $OS_TYPE | grep ubuntu) + +if [[ $OS_UBUNTU ]]; then + apt-get update && apt-get install -y apt-transport-https + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + touch /etc/apt/sources.list.d/kubernetes.list + echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list + apt-get update + apt-get install -y kubectl +else + cat < /etc/yum.repos.d/kubernetes.repo +[kubernetes] +name=Kubernetes +baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +EOF + yum install -y kubectl +fi +