testing-scheduler/server/src/test_parser.py

   1 #!/usr/bin/env python\r
   2 \r
   3 ##############################################################################\r
   4 # Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others.\r
   5 #\r
   6 # All rights reserved. This program and the accompanying materials\r
   7 # are made available under the terms of the Apache License, Version 2.0\r
   8 # which accompanies this distribution, and is available at\r
   9 # http://www.apache.org/licenses/LICENSE-2.0\r
  10 ##############################################################################\r
  11 \r
  12 import click\r
  13 import os\r
  14 import yaml\r
  15 import json\r
  16 import collections\r
  17 from src.step.step_manager import TestStepManager\r
  18 from src.conductor_processor.workflow import WorkflowFile\r
  19 from conductorclient.run_new_workflow import WorkflowMgr\r
  20 \r
  21 BASE_DIR = os.path.dirname(os.path.abspath(__file__))\r
  22 CONDUCTOR_SERVER_ADDR = "http://conductor_conductor-server_1:8080"\r
  23 STORE_TASK_PATH = "/tmp/generate_task.json"\r
  24 STORE_WF_PATH = "/tmp/generate_workflow.json"\r
  25 \r
  26 \r
  27 @click.command()\r
  28 @click.option("--filepath", help="file path of test case")\r
  29 def cmdParse(filepath):\r
  30     parse(filepath)\r
  31 \r
  32 \r
  33 def parse(filepath):\r
  34     filePrefix, fileName = os.path.split(filepath)\r
  35     print '------------ start to parse the test case:' + \\r
  36         '%s ----------------' % fileName\r
  37     with open(filepath) as f:\r
  38         yaml_file = yaml.load(f)\r
  39         parseTestcase(yaml_file['schema'], fileName)\r
  40 \r
  41     workflowId = runWorkFlow()\r
  42     print '------------------- parse executes end -------------------------'\r
  43 \r
  44     return workflowId\r
  45 \r
  46 \r
  47 def parseTestcase(schema, tcName='testcase0'):\r
  48     if schema is None:\r
  49         return parseLog(False, reason='schema not found.')\r
  50     steps = schema['steps']\r
  51     if steps is None:\r
  52         return parseLog(False, reason='steps is invalid.')\r
  53     flows = schema['flows']\r
  54     if flows is None:\r
  55         return parseLog(False, reasion='flows is invalid.')\r
  56     # steps is a list, step is dict. no json here.\r
  57     # steps = sorted(steps, sortById)\r
  58 \r
  59     # load context\r
  60     contextDict = {}\r
  61     contextDir = os.path.join(BASE_DIR, "env", "context", "context.yaml")\r
  62     with open(contextDir, "r") as f:\r
  63         contextDict = yaml.load(f)\r
  64     #\r
  65     testStepMgr = TestStepManager(contextDict)\r
  66 \r
  67     stepObjArr = []\r
  68     for step in steps:\r
  69         if 'args' not in step:\r
  70             step['args'] = {}\r
  71         # type and action can be extended, default couple is 'test' & 'start'.\r
  72         if 'type' not in step:\r
  73             step['type'] = 'test'\r
  74             step['action'] = 'start'\r
  75 \r
  76         stepObj = testStepMgr.getStepObj(\r
  77             step['type'], step['id'], step['name'], step['service'],\r
  78             step['action'], step['args'])\r
  79         stepObjArr.append(stepObj)\r
  80 \r
  81     # generate workflow by 'flow' and 'step'\r
  82     tcName = os.path.splitext(tcName)[0]\r
  83     wfFileObj = WorkflowFile(tcName)\r
  84     workflowDict, taskMetaList = wfFileObj.generateMetaData(flows, stepObjArr)\r
  85 \r
  86     with open(STORE_TASK_PATH, 'w') as f:\r
  87         f.write(json.dumps({'task_group': taskMetaList}, indent=True))\r
  88     with open(STORE_WF_PATH, 'w') as f:\r
  89         f.write(json.dumps(workflowDict, indent=True))\r
  90 \r
  91 \r
  92 def parseWebTestcase(webTestcase):\r
  93     print 'parseWebTestcase----------------------------'\r
  94 \r
  95     stepList = webTestcase['stepList']\r
  96     mainOrdersList = webTestcase['mainOrdersList']\r
  97     subflowList = webTestcase['subflowList']\r
  98 \r
  99     parseData = collections.OrderedDict()\r
 100     parseData['schema'] = collections.OrderedDict()\r
 101     parseData['schema']['steps'] = []\r
 102     parseData['schema']['flows'] = []\r
 103 \r
 104     parseStepList = parseData['schema']['steps']\r
 105     parseFlowList = parseData['schema']['flows']\r
 106     stepIndexDict = {}\r
 107     # parse stepList\r
 108     for index in range(len(stepList)):\r
 109         stepItem = stepList[index]\r
 110         parseStep = collections.OrderedDict()\r
 111 \r
 112         parseStep['id'] = index + 1\r
 113         parseStep['name'] = stepItem['name']\r
 114         parseStep['service'] = collections.OrderedDict()\r
 115         parseStep['service']['name'] = stepItem['service']\r
 116         parseStep['service']['interface'] = stepItem['action']\r
 117         parseStep['action'] = 'start'\r
 118         parseStep['args'] = {}\r
 119         for paramItem in stepItem['params']:\r
 120             parseStep['args'][paramItem['key']] = transParamString(\r
 121                 paramItem['value'])\r
 122 \r
 123         parseStepList.append(parseStep)\r
 124         stepIndexDict[parseStep['name']] = parseStep['id']\r
 125     # parse flows\r
 126     # parse mainflow\r
 127     print stepIndexDict\r
 128     typeDict = {1: 'normal', 2: 'switch', 3: 'parallel'}\r
 129     mainFlow = collections.OrderedDict()\r
 130     mainFlow['name'] = 'main'\r
 131     mainFlow['orders'] = []\r
 132     mainFlow['orders'] = parseOrderList(\r
 133         mainOrdersList, stepIndexDict, typeDict)\r
 134     parseFlowList.append(mainFlow)\r
 135 \r
 136     # parse subflow\r
 137     for subflowItem in subflowList:\r
 138         replaceSubflow = collections.OrderedDict()\r
 139         replaceSubflow['name'] = subflowItem['name']\r
 140         replaceSubflow['orders'] = parseOrderList(\r
 141             subflowItem['orderList'], stepIndexDict, typeDict)\r
 142         parseFlowList.append(replaceSubflow)\r
 143 \r
 144     print 'END parseWebTestcase----------------------------'\r
 145     return parseData\r
 146 \r
 147 \r
 148 # parse orderlist from web edition to server edition\r
 149 def parseOrderList(orderList, stepIndexDict, typeDict):\r
 150     replaceList = []\r
 151     for orderItem in orderList:\r
 152         replaceOrder = collections.OrderedDict()\r
 153         orderType = typeDict[orderItem['type']]\r
 154         replaceOrder['type'] = orderType\r
 155         if orderType == 'normal':\r
 156             stepId = stepIndexDict[orderItem['step']]\r
 157             replaceOrder['step'] = stepId\r
 158         elif orderType == 'switch':\r
 159             replaceOrder['value'] = orderItem['value']\r
 160             replaceOrder['cases'] = collections.OrderedDict()\r
 161             for caseItem in orderItem['cases']:\r
 162                 caseValue = caseItem['value']\r
 163                 caseOrderType = caseItem['orderType']\r
 164                 caseOrderValue = caseItem['orderValue']\r
 165                 if caseOrderType == "step":\r
 166                     orderInCase = collections.OrderedDict()\r
 167                     orderInCase['type'] = 'normal'\r
 168                     orderInCase['step'] = stepIndexDict[caseOrderValue]\r
 169                     replaceOrder['cases'][caseValue] = [orderInCase]\r
 170                 else:\r
 171                     replaceOrder['cases'][caseValue] = caseOrderValue\r
 172         else:\r
 173             replaceOrder['parallel'] = collections.OrderedDict()\r
 174             pIndex = 1\r
 175             for branchItem in orderItem['branches']:\r
 176                 pKey = 'p' + str(pIndex)\r
 177                 branchOrderType = branchItem['orderType']\r
 178                 branchOrderValue = branchItem['orderValue']\r
 179                 if branchOrderType == "step":\r
 180                     replaceBranchItem = collections.OrderedDict()\r
 181                     replaceBranchItem['type'] = 'normal'\r
 182                     replaceBranchItem['step'] = stepIndexDict[branchOrderValue]\r
 183                     replaceOrder['parallel'][pKey] = [replaceBranchItem]\r
 184                 else:\r
 185                     replaceOrder['parallel'][pKey] = branchOrderValue\r
 186                 pIndex += 1\r
 187         replaceList.append(replaceOrder)\r
 188     return replaceList\r
 189 \r
 190 \r
 191 def transParamString(val):\r
 192     if type(val) != str:\r
 193         return val\r
 194     if '.' not in val:\r
 195         if val.isdigit():\r
 196             return int(val)\r
 197     try:\r
 198         f = float(val)\r
 199         return f\r
 200     except ValueError:\r
 201         return val\r
 202 \r
 203 \r
 204 def getWebTestcase(originTcDict):\r
 205     print "getWebTestcase----------------------------------"\r
 206     webTcDict = {\r
 207         "stepList": [],\r
 208         "mainOrdersList": [],\r
 209         "subflowList": []\r
 210     }\r
 211     stepList = webTcDict['stepList']\r
 212     subflowList = webTcDict['subflowList']\r
 213     if originTcDict is None:\r
 214         return webTcDict\r
 215     originContent = originTcDict['schema']\r
 216     originSteps = originContent['steps']\r
 217     stepIndexDict = {}\r
 218     # transform steps to stepList\r
 219     for stepItem in originSteps:\r
 220         replaceStep = {}\r
 221         replaceStep['name'] = stepItem['name']\r
 222         replaceStep['service'] = stepItem['service']['name']\r
 223         replaceStep['action'] = stepItem['service']['interface']\r
 224         replaceStep['params'] = []\r
 225         if 'args' in stepItem:\r
 226             for (key, value) in stepItem['args'].items():\r
 227                 replaceParam = {}\r
 228                 replaceParam['key'] = key\r
 229                 replaceParam['value'] = value\r
 230                 replaceStep['params'].append(replaceParam)\r
 231         stepList.append(replaceStep)\r
 232         stepIndexDict[stepItem['id']] = stepItem['name']\r
 233 \r
 234     # transform main flow\r
 235     originFlows = originContent['flows']\r
 236     originMainflow = {}\r
 237     for flowIndex in range(len(originFlows)):\r
 238         flowItem = originFlows[flowIndex]\r
 239         if flowItem['name'] == 'main':\r
 240             originMainflow = flowItem\r
 241             originFlows.pop(flowIndex)\r
 242             break\r
 243     typeDict = {'normal': 1, 'switch': 2, 'parallel': 3}\r
 244     webTcDict['mainOrdersList'] = getOrderList(\r
 245         originMainflow['orders'], stepIndexDict, typeDict)\r
 246 \r
 247     # transform subflows\r
 248     for originSubflow in originFlows:\r
 249         replaceSubflow = {}\r
 250         replaceSubflow['name'] = originSubflow['name']\r
 251         replaceSubflow['orderList'] = getOrderList(\r
 252             originSubflow['orders'], stepIndexDict, typeDict)\r
 253         subflowList.append(replaceSubflow)\r
 254 \r
 255     # return web edition of testcase\r
 256     print "END getWebTestcase----------------------------------"\r
 257     return webTcDict\r
 258 \r
 259 \r
 260 def getOrderList(originOrderList, stepIndexDict, typeDict):\r
 261     replaceOrderList = []\r
 262     for orderItem in originOrderList:\r
 263         replaceOrderItem = {}\r
 264         orderType = orderItem['type']\r
 265         replaceOrderItem['type'] = typeDict[orderType]\r
 266         if orderType == 'normal':\r
 267             stepName = stepIndexDict[orderItem['step']]\r
 268             replaceOrderItem['step'] = stepName\r
 269         elif orderType == 'switch':\r
 270             replaceOrderItem['value'] = orderItem['value']\r
 271             replaceOrderItem['cases'] = []\r
 272             for (caseValue, ordersInCase) in orderItem['cases'].items():\r
 273                 replaceCase = {}\r
 274                 replaceCase['value'] = caseValue\r
 275                 if type(ordersInCase) == list:\r
 276                     replaceCase['orderType'] = 'step'\r
 277                     caseStepName = stepIndexDict[ordersInCase[0]['step']]\r
 278                     replaceCase['orderValue'] = caseStepName\r
 279                 else:\r
 280                     replaceCase['orderType'] = 'flow'\r
 281                     replaceCase['orderValue'] = ordersInCase\r
 282                 replaceOrderItem['cases'].append(replaceCase)\r
 283         else:\r
 284             replaceOrderItem['branches'] = []\r
 285             for paraIndex in orderItem['parallel']:\r
 286                 paraItem = orderItem['parallel'][paraIndex]\r
 287                 replaceBranch = {}\r
 288                 if type(paraItem) == list:\r
 289                     replaceBranch['orderType'] = 'step'\r
 290                     branchStepName = stepIndexDict[paraItem[0]['step']]\r
 291                     replaceBranch['orderValue'] = branchStepName\r
 292                 else:\r
 293                     replaceBranch['orderType'] = 'flow'\r
 294                     replaceBranch['orderValue'] = paraItem\r
 295                 replaceOrderItem['branches'].append(replaceBranch)\r
 296         replaceOrderList.append(replaceOrderItem)\r
 297 \r
 298     return replaceOrderList\r
 299 \r
 300 \r
 301 def runWorkFlow():\r
 302     wfMgr = WorkflowMgr(CONDUCTOR_SERVER_ADDR)\r
 303     wfMgr.setTaskDefFromFile(STORE_TASK_PATH)\r
 304     wfMgr.setWorkflowFromFile(STORE_WF_PATH)\r
 305     inputParam = {'input': 'fake'}\r
 306     workflowId = wfMgr.startWorkflow(inputParam)\r
 307     return workflowId\r
 308 \r
 309 \r
 310 def parseLog(flag, **msg):\r
 311     return {'result': flag, 'message': msg}\r
 312 \r
 313 \r
 314 if __name__ == "__main__":\r
 315     cmdParse()\r