api/resources/env_action.py

   1 ##############################################################################
   2 # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9 from __future__ import absolute_import
  10
  11 import errno
  12 import logging
  13 import os
  14 import subprocess
  15 import threading
  16 import time
  17 import uuid
  18 import glob
  19
  20 from six.moves import configparser
  21 from oslo_serialization import jsonutils
  22
  23 from api import conf as api_conf
  24 from api.database.handler import AsyncTaskHandler
  25 from api.utils import influx
  26 from api.utils.common import result_handler
  27 from docker import Client
  28 from yardstick.common import constants as config
  29 from yardstick.common import utils as yardstick_utils
  30 from yardstick.common.httpClient import HttpClient
  31
  32 logger = logging.getLogger(__name__)
  33 logger.setLevel(logging.DEBUG)
  34
  35
  36 def createGrafanaContainer(args):
  37     task_id = str(uuid.uuid4())
  38
  39     thread = threading.Thread(target=_create_grafana, args=(task_id,))
  40     thread.start()
  41
  42     return result_handler('success', {'task_id': task_id})
  43
  44
  45 def _create_grafana(task_id):
  46     _create_task(task_id)
  47
  48     client = Client(base_url=config.DOCKER_URL)
  49
  50     try:
  51         if not _check_image_exist(client, '%s:%s' % (config.GRAFANA_IMAGE,
  52                                                      config.GRAFANA_TAGS)):
  53             client.pull(config.GRAFANA_IMAGE, config.GRAFANA_TAGS)
  54
  55         _create_grafana_container(client)
  56
  57         time.sleep(5)
  58
  59         _create_data_source()
  60
  61         _create_dashboard()
  62
  63         _update_task_status(task_id)
  64     except Exception as e:
  65         _update_task_error(task_id, str(e))
  66         logger.debug('Error: %s', e)
  67
  68
  69 def _create_dashboard():
  70     url = 'http://admin:admin@%s:3000/api/dashboards/db' % api_conf.GATEWAY_IP
  71     path = os.path.join(config.YARDSTICK_REPOS_DIR, 'dashboard',
  72                         '*dashboard.json')
  73
  74     for i in sorted(glob.iglob(path)):
  75         with open(i) as f:
  76             data = jsonutils.load(f)
  77         HttpClient().post(url, data)
  78
  79
  80 def _create_data_source():
  81     url = 'http://admin:admin@%s:3000/api/datasources' % api_conf.GATEWAY_IP
  82     data = {
  83         "name": "yardstick",
  84         "type": "influxdb",
  85         "access": "proxy",
  86         "url": "http://%s:8086" % api_conf.GATEWAY_IP,
  87         "password": "root",
  88         "user": "root",
  89         "database": "yardstick",
  90         "basicAuth": True,
  91         "basicAuthUser": "admin",
  92         "basicAuthPassword": "admin",
  93         "isDefault": False,
  94     }
  95     HttpClient().post(url, data)
  96
  97
  98 def _create_grafana_container(client):
  99     ports = [3000]
 100     port_bindings = {k: k for k in ports}
 101     host_config = client.create_host_config(port_bindings=port_bindings)
 102
 103     container = client.create_container(image='%s:%s' % (config.GRAFANA_IMAGE,
 104                                                          config.GRAFANA_TAGS),
 105                                         ports=ports,
 106                                         detach=True,
 107                                         tty=True,
 108                                         host_config=host_config)
 109     client.start(container)
 110
 111
 112 def _check_image_exist(client, t):
 113     return any(t in a['RepoTags'][0] for a in client.images() if a['RepoTags'])
 114
 115
 116 def createInfluxDBContainer(args):
 117     task_id = str(uuid.uuid4())
 118
 119     thread = threading.Thread(target=_create_influxdb, args=(task_id,))
 120     thread.start()
 121
 122     return result_handler('success', {'task_id': task_id})
 123
 124
 125 def _create_influxdb(task_id):
 126     _create_task(task_id)
 127
 128     client = Client(base_url=config.DOCKER_URL)
 129
 130     try:
 131         _change_output_to_influxdb()
 132
 133         if not _check_image_exist(client, '%s:%s' % (config.INFLUXDB_IMAGE,
 134                                                      config.INFLUXDB_TAG)):
 135             client.pull(config.INFLUXDB_IMAGE, tag=config.INFLUXDB_TAG)
 136
 137         _create_influxdb_container(client)
 138
 139         time.sleep(5)
 140
 141         _config_influxdb()
 142
 143         _update_task_status(task_id)
 144     except Exception as e:
 145         _update_task_error(task_id, str(e))
 146         logger.debug('Error: %s', e)
 147
 148
 149 def _create_influxdb_container(client):
 150
 151     ports = [8083, 8086]
 152     port_bindings = {k: k for k in ports}
 153     host_config = client.create_host_config(port_bindings=port_bindings)
 154
 155     container = client.create_container(image='%s:%s' % (config.INFLUXDB_IMAGE,
 156                                                          config.INFLUXDB_TAG),
 157                                         ports=ports,
 158                                         detach=True,
 159                                         tty=True,
 160                                         host_config=host_config)
 161     client.start(container)
 162
 163
 164 def _config_influxdb():
 165     try:
 166         client = influx.get_data_db_client()
 167         client.create_user(config.USER, config.PASSWORD, config.DATABASE)
 168         client.create_database(config.DATABASE)
 169         logger.info('Success to config influxDB')
 170     except Exception as e:
 171         logger.debug('Failed to config influxDB: %s', e)
 172
 173
 174 def _change_output_to_influxdb():
 175     yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR)
 176
 177     parser = configparser.ConfigParser()
 178     parser.read(config.YARDSTICK_CONFIG_SAMPLE_FILE)
 179
 180     parser.set('DEFAULT', 'dispatcher', 'influxdb')
 181     parser.set('dispatcher_influxdb', 'target',
 182                'http://%s:8086' % api_conf.GATEWAY_IP)
 183
 184     with open(config.YARDSTICK_CONFIG_FILE, 'w') as f:
 185         parser.write(f)
 186
 187
 188 def prepareYardstickEnv(args):
 189     task_id = str(uuid.uuid4())
 190
 191     thread = threading.Thread(target=_prepare_env_daemon, args=(task_id,))
 192     thread.start()
 193
 194     return result_handler('success', {'task_id': task_id})
 195
 196
 197 def _prepare_env_daemon(task_id):
 198     _create_task(task_id)
 199
 200     installer_ip = os.environ.get('INSTALLER_IP', 'undefined')
 201     installer_type = os.environ.get('INSTALLER_TYPE', 'undefined')
 202
 203     try:
 204         _check_variables(installer_ip, installer_type)
 205
 206         _create_directories()
 207
 208         rc_file = config.OPENSTACK_RC_FILE
 209
 210         _get_remote_rc_file(rc_file, installer_ip, installer_type)
 211
 212         _source_file(rc_file)
 213
 214         _append_external_network(rc_file)
 215
 216         # update the external_network
 217         _source_file(rc_file)
 218
 219         _clean_images()
 220
 221         _load_images()
 222
 223         _update_task_status(task_id)
 224     except Exception as e:
 225         _update_task_error(task_id, str(e))
 226         logger.debug('Error: %s', e)
 227
 228
 229 def _check_variables(installer_ip, installer_type):
 230
 231     if installer_ip == 'undefined':
 232         raise SystemExit('Missing INSTALLER_IP')
 233
 234     if installer_type == 'undefined':
 235         raise SystemExit('Missing INSTALLER_TYPE')
 236     elif installer_type not in config.INSTALLERS:
 237         raise SystemExit('INSTALLER_TYPE is not correct')
 238
 239
 240 def _create_directories():
 241     yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR)
 242
 243
 244 def _source_file(rc_file):
 245     yardstick_utils.source_env(rc_file)
 246
 247
 248 def _get_remote_rc_file(rc_file, installer_ip, installer_type):
 249
 250     os_fetch_script = os.path.join(config.RELENG_DIR, config.OS_FETCH_SCRIPT)
 251
 252     try:
 253         cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type,
 254                '-a', installer_ip]
 255         p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 256         p.communicate()
 257
 258         if p.returncode != 0:
 259             logger.debug('Failed to fetch credentials from installer')
 260     except OSError as e:
 261         if e.errno != errno.EEXIST:
 262             raise
 263
 264
 265 def _append_external_network(rc_file):
 266     neutron_client = yardstick_utils.get_neutron_client()
 267     networks = neutron_client.list_networks()['networks']
 268     try:
 269         ext_network = next(n['name'] for n in networks if n['router:external'])
 270     except StopIteration:
 271         logger.warning("Can't find external network")
 272     else:
 273         cmd = 'export EXTERNAL_NETWORK=%s' % ext_network
 274         try:
 275             with open(rc_file, 'a') as f:
 276                 f.write(cmd + '\n')
 277         except OSError as e:
 278             if e.errno != errno.EEXIST:
 279                 raise
 280
 281
 282 def _clean_images():
 283     cmd = [config.CLEAN_IMAGES_SCRIPT]
 284     p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 285                          cwd=config.YARDSTICK_REPOS_DIR)
 286     output = p.communicate()[0]
 287     logger.debug('The result is: %s', output)
 288
 289
 290 def _load_images():
 291     cmd = [config.LOAD_IMAGES_SCRIPT]
 292     p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 293                          cwd=config.YARDSTICK_REPOS_DIR)
 294     output = p.communicate()[0]
 295     logger.debug('The result is: %s', output)
 296
 297
 298 def _create_task(task_id):
 299     async_handler = AsyncTaskHandler()
 300     task_dict = {
 301         'task_id': task_id,
 302         'status': 0
 303     }
 304     async_handler.insert(task_dict)
 305
 306
 307 def _update_task_status(task_id):
 308     async_handler = AsyncTaskHandler()
 309
 310     task = async_handler.get_task_by_taskid(task_id)
 311     async_handler.update_status(task, 1)
 312
 313
 314 def _update_task_error(task_id, error):
 315     async_handler = AsyncTaskHandler()
 316
 317     task = async_handler.get_task_by_taskid(task_id)
 318     async_handler.update_status(task, 2)
 319     async_handler.update_error(task, error)