utils/parser.py

   1 #!/usr/bin/env python
   2 ##############################################################################
   3 # Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ##############################################################################
  10 '''This file realize the function of how to parser a config file.
  11 This contain Two part:
  12 Frist is Init some variables the will be used.
  13 Second is reading config file.'''
  14
  15 import os
  16 import yaml
  17 import json
  18 import time
  19 from pyroute2 import IPDB
  20 import utils.infra_setup.runner.docker_env as docker_env
  21
  22
  23 class Parser():
  24
  25     bottlenecks_config = {}
  26     bottlenecks_test = {}
  27
  28     @classmethod
  29     def config_init(cls):
  30         cls.code_dir = os.path.dirname(os.path.abspath(__file__))
  31         cls.root_dir = os.path.dirname(cls.code_dir)
  32         cls.test_dir = os.path.join(cls.root_dir, 'testsuites')
  33         config_dir = os.path.join(
  34             cls.root_dir,
  35             'config',
  36             'config.yaml')
  37
  38         with open(config_dir) as file:
  39             config_info = yaml.load(file)
  40             common_config = config_info['common_config']
  41             cls.bottlenecks_config["releng_dir"] = common_config["releng_dir"]
  42             cls.bottlenecks_config["fetch_os"] = common_config["fetch_os_file"]
  43             cls.bottlenecks_config["log_dir"] = common_config['log_dir']
  44             cls.bottlenecks_config["rc_dir"] = common_config['rc_dir']
  45             cls.bottlenecks_config["pod_info"] = common_config['pod_info']
  46             cls.bottlenecks_config["yardstick_rc_dir"] = \
  47                 common_config['yardstick_rc_dir']
  48             cls.bottlenecks_config["yardstick_image_dir"] = \
  49                 common_config['yardstick_image_dir']
  50             cls.bottlenecks_config["image_url"] = common_config['image_url']
  51             cls.bottlenecks_config["yardstick_image_name"] = \
  52                 common_config['yardstick_image_name']
  53             cls.config_dir_check(cls.bottlenecks_config["log_dir"])
  54
  55     @classmethod
  56     def story_read(cls, testcase, story_name):
  57         story_dir = os.path.join(
  58             cls.test_dir,
  59             testcase,
  60             'testsuite_story',
  61             story_name + '.yaml')
  62         with open(story_dir) as file:
  63             story_parser = yaml.load(file)
  64         for case_name in story_parser['testcase']:
  65             Parser.testcase_read(testcase, case_name)
  66
  67         return cls.bottlenecks_test
  68
  69     @classmethod
  70     def testcase_read(cls, testcase, testcase_name):
  71
  72         testcase_dir = os.path.join(
  73             cls.test_dir,
  74             testcase,
  75             'testcase_cfg',
  76             testcase_name)
  77         testcase_local = testcase_dir + ".yaml"
  78         with open(testcase_local) as f:
  79             cls.bottlenecks_test[testcase_name] = yaml.load(f)
  80
  81         return cls.bottlenecks_test
  82
  83     @classmethod
  84     def config_dir_check(cls, dirname):
  85         if dirname is None:
  86             dirname = '/tmp/'
  87         if not os.path.exists(dirname):
  88             os.makedirs(dirname)
  89
  90     @classmethod
  91     def testcase_out_dir(cls, testcase):
  92         file_name = os.getenv("OUTPUT_FILE")
  93         if file_name is None:
  94             file_suffix = time.strftime('%H_%M', time.localtime(time.time()))
  95             suffix_name = "_" + str(file_suffix)
  96             out_name = cls.bottlenecks_config["log_dir"] + testcase
  97             outfile_name = out_name + suffix_name + ".out"
  98         else:
  99             out_name = str(file_name)
 100             outfile_name = cls.bottlenecks_config["log_dir"] + out_name
 101         return outfile_name
 102
 103     @staticmethod
 104     def config_parser(testcase_cfg, parameters):
 105         test_cfg = testcase_cfg['test_config']
 106         stack_cfg = testcase_cfg['stack_config']
 107         # TO-DO add cli parameters to stack_config.
 108         return test_cfg, stack_cfg
 109
 110     @staticmethod
 111     def convert_docker_env(config, ip_type):
 112         if ip_type is "dashboard":
 113             config["contexts"]["dashboard_ip"] = \
 114                 docker_env.ELK_info["ip"] + ":9200"
 115         elif ip_type is "yardstick":
 116             config["contexts"]["yardstick_ip"] = \
 117                 docker_env.yardstick_info["ip"] + ":8888"
 118
 119     @staticmethod
 120     def ip_parser(ip_type):
 121         with IPDB() as ip:
 122             GATEWAY_IP = ip.routes['default'].gateway
 123         if ip_type is "dashboard":
 124             TEST_IP = GATEWAY_IP + ":9200"
 125         elif ip_type is "yardstick_test_ip":
 126             TEST_IP = GATEWAY_IP + ":8888"
 127         return TEST_IP
 128
 129     @staticmethod
 130     def result_to_file(data, file_name):
 131         with open(file_name, "a") as f:
 132             f.write(json.dumps(data, f))
 133             f.write("\n")
 134
 135     @staticmethod
 136     def str_to_list(str_org):
 137         try:
 138             data = str_org.split(',')
 139         except AttributeError:
 140             data = []
 141             data.append(str_org)
 142         return data
 143
 144
 145 class HeatTemplate_Parser():
 146     """parser a Heat template and a method to deploy template to a stack"""
 147
 148     def __init__(self):
 149         self.heat_date = {}
 150         self.heat_date["resources"] = {}
 151         self.heat_date["outputs"] = {}
 152         self.heat_date["heat_template_version"] = "2013-05-23"
 153         self.heat_date["description"] = {"Stack built by the bottlenecks"
 154                                          " framework for root."}
 155
 156     def add_security_group(self, name):
 157         """add to the template a Neutron SecurityGroup"""
 158         security_name = name + "-security_group"
 159         self.heat_date["resources"][security_name] = {
 160             "type": "OS::Neutron::SecurityGroup",
 161             "properties": {
 162                 "name": security_name,
 163                 "description": "Group allowing icmp and upd/tcp on all ports",
 164                 "rules": [
 165                     {"remote_ip_prefix": "0.0.0.0/0",
 166                      "protocol": "tcp",
 167                      "port_range_min": "1",
 168                      "port_range_max": "65535"},
 169                     {"remote_ip_prefix": "0.0.0.0/0",
 170                      "protocol": "udp",
 171                      "port_range_min": "1",
 172                      "port_range_max": "65535"},
 173                     {"remote_ip_prefix": "0.0.0.0/0",
 174                      "protocol": "icmp"}
 175                 ]
 176             }
 177         }
 178
 179         self.heat_date["outputs"][security_name] = {
 180             "description": "ID of Security Group",
 181             "value": {"get_resource": security_name}
 182         }
 183
 184     def add_keypair(self, name):
 185         """add to the template a Nova KeyPair"""
 186         key_name = name + "key"
 187         with open(Parser.root_dir +
 188                   "utils/infra_setup/"
 189                   "bottlenecks_key/bottlenecks_key.pub") as f:
 190             key_content = f.read()
 191             self.heat_date["resources"][key_name] = {
 192                 "type": "OS::Nova::KeyPair",
 193                 "properties": {
 194                     "name": key_name,
 195                     "public_key": key_content
 196                 }
 197             }
 198
 199     def add_network(self, name):
 200         """add to the template a Neutron Net"""
 201         network_name = name + "-net"
 202         self.heat_date["resources"][network_name] = {
 203             "type": "OS::Neutron::Net",
 204             "properties": {"name": network_name}
 205         }
 206
 207     def add_subnet(self, name, cidr):
 208         """add to the template a Neutron Subnet"""
 209         network_name = name + "-net"
 210         subnet_name = name + "-subnet"
 211         self.heat_date["resources"][subnet_name] = {
 212             "type": "OS::Neutron::Subnet",
 213             "depends_on": network_name,
 214             "properties": {
 215                 "name": subnet_name,
 216                 "cidr": cidr,
 217                 "network_id": {"get_resource": network_name}
 218             }
 219         }
 220
 221         self.heat_date["outputs"][subnet_name] = {
 222             "description": "subnet %s ID" % subnet_name,
 223             "value": {"get_resource": subnet_name}
 224         }
 225
 226     def add_router(self, name, ext_gw_net):
 227         """add to the template a Neutron Router and interface"""
 228         router_name = name + "-route"
 229         subnet_name = name + "-subnet"
 230
 231         self.heat_date["resources"][router_name] = {
 232             "type": "OS::Neutron::Router",
 233             "depends_on": [subnet_name],
 234             "properties": {
 235                 "name": router_name,
 236                 "external_gateway_info": {
 237                     "network": ext_gw_net
 238                 }
 239             }
 240         }
 241
 242     def add_router_interface(self, name):
 243         """add to the template a Neutron RouterInterface and interface"""
 244         router_name = name + "-route"
 245         subnet_name = name + "-subnet"
 246         router_if_name = name + "-interface"
 247
 248         self.heat_date["resources"][router_if_name] = {
 249             "type": "OS::Neutron::RouterInterface",
 250             "depends_on": [router_name, subnet_name],
 251             "properties": {
 252                 "router_id": {"get_resource": router_name},
 253                 "subnet_id": {"get_resource": subnet_name}
 254             }
 255         }
 256
 257     def add_server(self, name, image, flavor, user, ports=None):
 258         """add to the template a Nova Server"""
 259
 260         key_name = "bottlenecks-poscakey"
 261         port_name = name + "-port"
 262         self.heat_date["resources"][name] = {
 263             "type": "OS::Nova::Server",
 264             "properties": {
 265                 "name": name,
 266                 "flavor": flavor,
 267                 "image": image,
 268                 "key_name": {"get_resource": key_name},
 269                 "admin_user": user,
 270                 "networks": [{
 271                     "port": {"get_resource": port_name}
 272                 }]
 273             }
 274         }
 275
 276         self.heat_date["outputs"][name] = {
 277             "description": "VM UUID",
 278             "value": {"get_resource": name}
 279         }
 280
 281     def add_port(self, name, stack_name=None):
 282         """add to the template a named Neutron Port"""
 283         network_name = stack_name + "-net"
 284         subnet_name = stack_name + "-subnet"
 285         port_name = name + "-port"
 286         security_name = stack_name + "-security_group"
 287
 288         self.heat_date["resources"][port_name] = {
 289             "type": "OS::Neutron::Port",
 290             "depends_on": [subnet_name],
 291             "properties": {
 292                 "name": port_name,
 293                 "fixed_ips": [{"subnet": {"get_resource": subnet_name}}],
 294                 "network_id": {"get_resource": network_name},
 295                 "replacement_policy": "AUTO",
 296                 "security_groups": [{"get_resource": security_name}]
 297             }
 298         }
 299
 300         self.heat_date["outputs"][port_name] = {
 301             "description": "Address for interface %s" % port_name,
 302             "value": {"get_attr": [port_name, "fixed_ips", 0, "ip_address"]}
 303         }
 304
 305     def add_floating_ip(self, name, stack_name, network_ext):
 306         """add to the template a Nova FloatingIP resource
 307         see: https://bugs.launchpad.net/heat/+bug/1299259
 308         """
 309         port_name = name + "-port"
 310         floating_ip_name = name + "-floating"
 311         router_if_name = stack_name + "-interface"
 312
 313         self.heat_date["resources"][floating_ip_name] = {
 314             "depends_on": [router_if_name, port_name],
 315             "type": "OS::Nova::FloatingIP",
 316             "properties": {
 317                 "pool": network_ext,
 318             }
 319         }
 320         self.heat_date['outputs'][floating_ip_name] = {
 321             'description': 'floating ip %s' % name,
 322             'value': {'get_attr': [name, 'ip']}
 323         }
 324
 325     def add_floating_ip_ass(self, name):
 326         """add to the template a Nova FloatingIP resource
 327         see: https://bugs.launchpad.net/heat/+bug/1299259
 328         """
 329         port_name = name + "-port"
 330         floating_ip_name = name + "-floating"
 331         floating_ass = name + "-floating_ass"
 332
 333         self.heat_date["resources"][floating_ass] = {
 334             "type": 'OS::Neutron::FloatingIPAssociation',
 335             "depends_on": [port_name, floating_ip_name],
 336             "properties": {
 337                 "floatingip_id": {'get_resource': floating_ip_name},
 338                 "port_id": {"get_resource": port_name}
 339             }
 340         }
 341
 342     def get_template_date(self):
 343         return self.heat_date