Add conductor UI to the result page of test scheduler
[bottlenecks.git] / utils / parser.py
1 #!/usr/bin/env python
2 ##############################################################################
3 # Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
4 #
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 ##############################################################################
10 '''This file realize the function of how to parser a config file.
11 This contain Two part:
12 Frist is Init some variables the will be used.
13 Second is reading config file.'''
14
15 import os
16 import yaml
17 import json
18 import time
19 from pyroute2 import IPDB
20 import utils.infra_setup.runner.docker_env as docker_env
21
22
23 class Parser():
24
25     bottlenecks_config = {}
26     bottlenecks_test = {}
27
28     @classmethod
29     def config_init(cls):
30         cls.code_dir = os.path.dirname(os.path.abspath(__file__))
31         cls.root_dir = os.path.dirname(cls.code_dir)
32         cls.test_dir = os.path.join(cls.root_dir, 'testsuites')
33         config_dir = os.path.join(
34             cls.root_dir,
35             'config',
36             'config.yaml')
37
38         with open(config_dir) as file:
39             config_info = yaml.load(file)
40             common_config = config_info['common_config']
41             cls.bottlenecks_config["releng_dir"] = common_config["releng_dir"]
42             cls.bottlenecks_config["fetch_os"] = common_config["fetch_os_file"]
43             cls.bottlenecks_config["log_dir"] = common_config['log_dir']
44             cls.bottlenecks_config["rc_dir"] = common_config['rc_dir']
45             cls.bottlenecks_config["pod_info"] = common_config['pod_info']
46             cls.bottlenecks_config["yardstick_rc_dir"] = \
47                 common_config['yardstick_rc_dir']
48             cls.bottlenecks_config["yardstick_image_dir"] = \
49                 common_config['yardstick_image_dir']
50             cls.bottlenecks_config["image_url"] = common_config['image_url']
51             cls.bottlenecks_config["yardstick_image_name"] = \
52                 common_config['yardstick_image_name']
53             cls.config_dir_check(cls.bottlenecks_config["log_dir"])
54
55     @classmethod
56     def story_read(cls, testcase, story_name):
57         story_dir = os.path.join(
58             cls.test_dir,
59             testcase,
60             'testsuite_story',
61             story_name + '.yaml')
62         with open(story_dir) as file:
63             story_parser = yaml.load(file)
64         for case_name in story_parser['testcase']:
65             Parser.testcase_read(testcase, case_name)
66
67         return cls.bottlenecks_test
68
69     @classmethod
70     def testcase_read(cls, testcase, testcase_name):
71
72         testcase_dir = os.path.join(
73             cls.test_dir,
74             testcase,
75             'testcase_cfg',
76             testcase_name)
77         testcase_local = testcase_dir + ".yaml"
78         with open(testcase_local) as f:
79             cls.bottlenecks_test[testcase_name] = yaml.load(f)
80
81         return cls.bottlenecks_test
82
83     @classmethod
84     def config_dir_check(cls, dirname):
85         if dirname is None:
86             dirname = '/tmp/'
87         if not os.path.exists(dirname):
88             os.makedirs(dirname)
89
90     @classmethod
91     def testcase_out_dir(cls, testcase):
92         file_name = os.getenv("OUTPUT_FILE")
93         if file_name is None:
94             file_suffix = time.strftime('%H_%M', time.localtime(time.time()))
95             suffix_name = "_" + str(file_suffix)
96             out_name = cls.bottlenecks_config["log_dir"] + testcase
97             outfile_name = out_name + suffix_name + ".out"
98         else:
99             out_name = str(file_name)
100             outfile_name = cls.bottlenecks_config["log_dir"] + out_name
101         return outfile_name
102
103     @staticmethod
104     def config_parser(testcase_cfg, parameters):
105         test_cfg = testcase_cfg['test_config']
106         stack_cfg = testcase_cfg['stack_config']
107         # TO-DO add cli parameters to stack_config.
108         return test_cfg, stack_cfg
109
110     @staticmethod
111     def convert_docker_env(config, ip_type):
112         if ip_type is "dashboard":
113             config["contexts"]["dashboard_ip"] = \
114                 docker_env.ELK_info["ip"] + ":9200"
115         elif ip_type is "yardstick":
116             config["contexts"]["yardstick_ip"] = \
117                 docker_env.yardstick_info["ip"] + ":8888"
118
119     @staticmethod
120     def ip_parser(ip_type):
121         with IPDB() as ip:
122             GATEWAY_IP = ip.routes['default'].gateway
123         if ip_type is "dashboard":
124             TEST_IP = GATEWAY_IP + ":9200"
125         elif ip_type is "yardstick_test_ip":
126             TEST_IP = GATEWAY_IP + ":8888"
127         return TEST_IP
128
129     @staticmethod
130     def result_to_file(data, file_name):
131         with open(file_name, "a") as f:
132             f.write(json.dumps(data, f))
133             f.write("\n")
134
135     @staticmethod
136     def str_to_list(str_org):
137         try:
138             data = str_org.split(',')
139         except AttributeError:
140             data = []
141             data.append(str_org)
142         return data
143
144
145 class HeatTemplate_Parser():
146     """parser a Heat template and a method to deploy template to a stack"""
147
148     def __init__(self):
149         self.heat_date = {}
150         self.heat_date["resources"] = {}
151         self.heat_date["outputs"] = {}
152         self.heat_date["heat_template_version"] = "2013-05-23"
153         self.heat_date["description"] = {"Stack built by the bottlenecks"
154                                          " framework for root."}
155
156     def add_security_group(self, name):
157         """add to the template a Neutron SecurityGroup"""
158         security_name = name + "-security_group"
159         self.heat_date["resources"][security_name] = {
160             "type": "OS::Neutron::SecurityGroup",
161             "properties": {
162                 "name": security_name,
163                 "description": "Group allowing icmp and upd/tcp on all ports",
164                 "rules": [
165                     {"remote_ip_prefix": "0.0.0.0/0",
166                      "protocol": "tcp",
167                      "port_range_min": "1",
168                      "port_range_max": "65535"},
169                     {"remote_ip_prefix": "0.0.0.0/0",
170                      "protocol": "udp",
171                      "port_range_min": "1",
172                      "port_range_max": "65535"},
173                     {"remote_ip_prefix": "0.0.0.0/0",
174                      "protocol": "icmp"}
175                 ]
176             }
177         }
178
179         self.heat_date["outputs"][security_name] = {
180             "description": "ID of Security Group",
181             "value": {"get_resource": security_name}
182         }
183
184     def add_keypair(self, name):
185         """add to the template a Nova KeyPair"""
186         key_name = name + "key"
187         with open(Parser.root_dir +
188                   "utils/infra_setup/"
189                   "bottlenecks_key/bottlenecks_key.pub") as f:
190             key_content = f.read()
191             self.heat_date["resources"][key_name] = {
192                 "type": "OS::Nova::KeyPair",
193                 "properties": {
194                     "name": key_name,
195                     "public_key": key_content
196                 }
197             }
198
199     def add_network(self, name):
200         """add to the template a Neutron Net"""
201         network_name = name + "-net"
202         self.heat_date["resources"][network_name] = {
203             "type": "OS::Neutron::Net",
204             "properties": {"name": network_name}
205         }
206
207     def add_subnet(self, name, cidr):
208         """add to the template a Neutron Subnet"""
209         network_name = name + "-net"
210         subnet_name = name + "-subnet"
211         self.heat_date["resources"][subnet_name] = {
212             "type": "OS::Neutron::Subnet",
213             "depends_on": network_name,
214             "properties": {
215                 "name": subnet_name,
216                 "cidr": cidr,
217                 "network_id": {"get_resource": network_name}
218             }
219         }
220
221         self.heat_date["outputs"][subnet_name] = {
222             "description": "subnet %s ID" % subnet_name,
223             "value": {"get_resource": subnet_name}
224         }
225
226     def add_router(self, name, ext_gw_net):
227         """add to the template a Neutron Router and interface"""
228         router_name = name + "-route"
229         subnet_name = name + "-subnet"
230
231         self.heat_date["resources"][router_name] = {
232             "type": "OS::Neutron::Router",
233             "depends_on": [subnet_name],
234             "properties": {
235                 "name": router_name,
236                 "external_gateway_info": {
237                     "network": ext_gw_net
238                 }
239             }
240         }
241
242     def add_router_interface(self, name):
243         """add to the template a Neutron RouterInterface and interface"""
244         router_name = name + "-route"
245         subnet_name = name + "-subnet"
246         router_if_name = name + "-interface"
247
248         self.heat_date["resources"][router_if_name] = {
249             "type": "OS::Neutron::RouterInterface",
250             "depends_on": [router_name, subnet_name],
251             "properties": {
252                 "router_id": {"get_resource": router_name},
253                 "subnet_id": {"get_resource": subnet_name}
254             }
255         }
256
257     def add_server(self, name, image, flavor, user, ports=None):
258         """add to the template a Nova Server"""
259
260         key_name = "bottlenecks-poscakey"
261         port_name = name + "-port"
262         self.heat_date["resources"][name] = {
263             "type": "OS::Nova::Server",
264             "properties": {
265                 "name": name,
266                 "flavor": flavor,
267                 "image": image,
268                 "key_name": {"get_resource": key_name},
269                 "admin_user": user,
270                 "networks": [{
271                     "port": {"get_resource": port_name}
272                 }]
273             }
274         }
275
276         self.heat_date["outputs"][name] = {
277             "description": "VM UUID",
278             "value": {"get_resource": name}
279         }
280
281     def add_port(self, name, stack_name=None):
282         """add to the template a named Neutron Port"""
283         network_name = stack_name + "-net"
284         subnet_name = stack_name + "-subnet"
285         port_name = name + "-port"
286         security_name = stack_name + "-security_group"
287
288         self.heat_date["resources"][port_name] = {
289             "type": "OS::Neutron::Port",
290             "depends_on": [subnet_name],
291             "properties": {
292                 "name": port_name,
293                 "fixed_ips": [{"subnet": {"get_resource": subnet_name}}],
294                 "network_id": {"get_resource": network_name},
295                 "replacement_policy": "AUTO",
296                 "security_groups": [{"get_resource": security_name}]
297             }
298         }
299
300         self.heat_date["outputs"][port_name] = {
301             "description": "Address for interface %s" % port_name,
302             "value": {"get_attr": [port_name, "fixed_ips", 0, "ip_address"]}
303         }
304
305     def add_floating_ip(self, name, stack_name, network_ext):
306         """add to the template a Nova FloatingIP resource
307         see: https://bugs.launchpad.net/heat/+bug/1299259
308         """
309         port_name = name + "-port"
310         floating_ip_name = name + "-floating"
311         router_if_name = stack_name + "-interface"
312
313         self.heat_date["resources"][floating_ip_name] = {
314             "depends_on": [router_if_name, port_name],
315             "type": "OS::Nova::FloatingIP",
316             "properties": {
317                 "pool": network_ext,
318             }
319         }
320         self.heat_date['outputs'][floating_ip_name] = {
321             'description': 'floating ip %s' % name,
322             'value': {'get_attr': [name, 'ip']}
323         }
324
325     def add_floating_ip_ass(self, name):
326         """add to the template a Nova FloatingIP resource
327         see: https://bugs.launchpad.net/heat/+bug/1299259
328         """
329         port_name = name + "-port"
330         floating_ip_name = name + "-floating"
331         floating_ass = name + "-floating_ass"
332
333         self.heat_date["resources"][floating_ass] = {
334             "type": 'OS::Neutron::FloatingIPAssociation',
335             "depends_on": [port_name, floating_ip_name],
336             "properties": {
337                 "floatingip_id": {'get_resource': floating_ip_name},
338                 "port_id": {"get_resource": port_name}
339             }
340         }
341
342     def get_template_date(self):
343         return self.heat_date