api/resources/env_action.py

   1 ##############################################################################
   2 # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9 from __future__ import absolute_import
  10
  11 import errno
  12 import logging
  13 import os
  14 import subprocess
  15 import threading
  16 import time
  17 import uuid
  18 import glob
  19
  20 from six.moves import configparser
  21 from oslo_serialization import jsonutils
  22
  23 from api import conf as api_conf
  24 from api.database.handler import AsyncTaskHandler
  25 from api.utils import influx
  26 from api.utils.common import result_handler
  27 from docker import Client
  28 from yardstick.common import constants as config
  29 from yardstick.common import utils as yardstick_utils
  30 from yardstick.common import openstack_utils
  31 from yardstick.common.httpClient import HttpClient
  32
  33 logger = logging.getLogger(__name__)
  34 logger.setLevel(logging.DEBUG)
  35
  36
  37 def createGrafanaContainer(args):
  38     task_id = str(uuid.uuid4())
  39
  40     thread = threading.Thread(target=_create_grafana, args=(task_id,))
  41     thread.start()
  42
  43     return result_handler('success', {'task_id': task_id})
  44
  45
  46 def _create_grafana(task_id):
  47     _create_task(task_id)
  48
  49     client = Client(base_url=config.DOCKER_URL)
  50
  51     try:
  52         if not _check_image_exist(client, '%s:%s' % (config.GRAFANA_IMAGE,
  53                                                      config.GRAFANA_TAGS)):
  54             client.pull(config.GRAFANA_IMAGE, config.GRAFANA_TAGS)
  55
  56         _create_grafana_container(client)
  57
  58         time.sleep(5)
  59
  60         _create_data_source()
  61
  62         _create_dashboard()
  63
  64         _update_task_status(task_id)
  65     except Exception as e:
  66         _update_task_error(task_id, str(e))
  67         logger.debug('Error: %s', e)
  68
  69
  70 def _create_dashboard():
  71     url = 'http://admin:admin@%s:3000/api/dashboards/db' % api_conf.GATEWAY_IP
  72     path = os.path.join(config.YARDSTICK_REPOS_DIR, 'dashboard',
  73                         '*dashboard.json')
  74
  75     for i in sorted(glob.iglob(path)):
  76         with open(i) as f:
  77             data = jsonutils.load(f)
  78         HttpClient().post(url, data)
  79
  80
  81 def _create_data_source():
  82     url = 'http://admin:admin@%s:3000/api/datasources' % api_conf.GATEWAY_IP
  83     data = {
  84         "name": "yardstick",
  85         "type": "influxdb",
  86         "access": "proxy",
  87         "url": "http://%s:8086" % api_conf.GATEWAY_IP,
  88         "password": "root",
  89         "user": "root",
  90         "database": "yardstick",
  91         "basicAuth": True,
  92         "basicAuthUser": "admin",
  93         "basicAuthPassword": "admin",
  94         "isDefault": False,
  95     }
  96     HttpClient().post(url, data)
  97
  98
  99 def _create_grafana_container(client):
 100     ports = [3000]
 101     port_bindings = {k: k for k in ports}
 102     host_config = client.create_host_config(port_bindings=port_bindings)
 103
 104     container = client.create_container(image='%s:%s' % (config.GRAFANA_IMAGE,
 105                                                          config.GRAFANA_TAGS),
 106                                         ports=ports,
 107                                         detach=True,
 108                                         tty=True,
 109                                         host_config=host_config)
 110     client.start(container)
 111
 112
 113 def _check_image_exist(client, t):
 114     return any(t in a['RepoTags'][0] for a in client.images() if a['RepoTags'])
 115
 116
 117 def createInfluxDBContainer(args):
 118     task_id = str(uuid.uuid4())
 119
 120     thread = threading.Thread(target=_create_influxdb, args=(task_id,))
 121     thread.start()
 122
 123     return result_handler('success', {'task_id': task_id})
 124
 125
 126 def _create_influxdb(task_id):
 127     _create_task(task_id)
 128
 129     client = Client(base_url=config.DOCKER_URL)
 130
 131     try:
 132         _change_output_to_influxdb()
 133
 134         if not _check_image_exist(client, '%s:%s' % (config.INFLUXDB_IMAGE,
 135                                                      config.INFLUXDB_TAG)):
 136             client.pull(config.INFLUXDB_IMAGE, tag=config.INFLUXDB_TAG)
 137
 138         _create_influxdb_container(client)
 139
 140         time.sleep(5)
 141
 142         _config_influxdb()
 143
 144         _update_task_status(task_id)
 145     except Exception as e:
 146         _update_task_error(task_id, str(e))
 147         logger.debug('Error: %s', e)
 148
 149
 150 def _create_influxdb_container(client):
 151
 152     ports = [8083, 8086]
 153     port_bindings = {k: k for k in ports}
 154     host_config = client.create_host_config(port_bindings=port_bindings)
 155
 156     container = client.create_container(image='%s:%s' % (config.INFLUXDB_IMAGE,
 157                                                          config.INFLUXDB_TAG),
 158                                         ports=ports,
 159                                         detach=True,
 160                                         tty=True,
 161                                         host_config=host_config)
 162     client.start(container)
 163
 164
 165 def _config_influxdb():
 166     try:
 167         client = influx.get_data_db_client()
 168         client.create_user(config.USER, config.PASSWORD, config.DATABASE)
 169         client.create_database(config.DATABASE)
 170         logger.info('Success to config influxDB')
 171     except Exception as e:
 172         logger.debug('Failed to config influxDB: %s', e)
 173
 174
 175 def _change_output_to_influxdb():
 176     yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR)
 177
 178     parser = configparser.ConfigParser()
 179     parser.read(config.YARDSTICK_CONFIG_SAMPLE_FILE)
 180
 181     parser.set('DEFAULT', 'dispatcher', 'influxdb')
 182     parser.set('dispatcher_influxdb', 'target',
 183                'http://%s:8086' % api_conf.GATEWAY_IP)
 184
 185     with open(config.YARDSTICK_CONFIG_FILE, 'w') as f:
 186         parser.write(f)
 187
 188
 189 def prepareYardstickEnv(args):
 190     task_id = str(uuid.uuid4())
 191
 192     thread = threading.Thread(target=_prepare_env_daemon, args=(task_id,))
 193     thread.start()
 194
 195     return result_handler('success', {'task_id': task_id})
 196
 197
 198 def _prepare_env_daemon(task_id):
 199     _create_task(task_id)
 200
 201     installer_ip = os.environ.get('INSTALLER_IP', 'undefined')
 202     installer_type = os.environ.get('INSTALLER_TYPE', 'undefined')
 203
 204     try:
 205         _check_variables(installer_ip, installer_type)
 206
 207         _create_directories()
 208
 209         rc_file = config.OPENSTACK_RC_FILE
 210
 211         _get_remote_rc_file(rc_file, installer_ip, installer_type)
 212
 213         _source_file(rc_file)
 214
 215         _append_external_network(rc_file)
 216
 217         # update the external_network
 218         _source_file(rc_file)
 219
 220         _clean_images()
 221
 222         _load_images()
 223
 224         _update_task_status(task_id)
 225     except Exception as e:
 226         _update_task_error(task_id, str(e))
 227         logger.debug('Error: %s', e)
 228
 229
 230 def _check_variables(installer_ip, installer_type):
 231
 232     if installer_ip == 'undefined':
 233         raise SystemExit('Missing INSTALLER_IP')
 234
 235     if installer_type == 'undefined':
 236         raise SystemExit('Missing INSTALLER_TYPE')
 237     elif installer_type not in config.INSTALLERS:
 238         raise SystemExit('INSTALLER_TYPE is not correct')
 239
 240
 241 def _create_directories():
 242     yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR)
 243
 244
 245 def _source_file(rc_file):
 246     yardstick_utils.source_env(rc_file)
 247
 248
 249 def _get_remote_rc_file(rc_file, installer_ip, installer_type):
 250
 251     os_fetch_script = os.path.join(config.RELENG_DIR, config.OS_FETCH_SCRIPT)
 252
 253     try:
 254         cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type,
 255                '-a', installer_ip]
 256         p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 257         p.communicate()
 258
 259         if p.returncode != 0:
 260             logger.debug('Failed to fetch credentials from installer')
 261     except OSError as e:
 262         if e.errno != errno.EEXIST:
 263             raise
 264
 265
 266 def _append_external_network(rc_file):
 267     neutron_client = openstack_utils.get_neutron_client()
 268     networks = neutron_client.list_networks()['networks']
 269     try:
 270         ext_network = next(n['name'] for n in networks if n['router:external'])
 271     except StopIteration:
 272         logger.warning("Can't find external network")
 273     else:
 274         cmd = 'export EXTERNAL_NETWORK=%s' % ext_network
 275         try:
 276             with open(rc_file, 'a') as f:
 277                 f.write(cmd + '\n')
 278         except OSError as e:
 279             if e.errno != errno.EEXIST:
 280                 raise
 281
 282
 283 def _clean_images():
 284     cmd = [config.CLEAN_IMAGES_SCRIPT]
 285     p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 286                          cwd=config.YARDSTICK_REPOS_DIR)
 287     output = p.communicate()[0]
 288     logger.debug('The result is: %s', output)
 289
 290
 291 def _load_images():
 292     cmd = [config.LOAD_IMAGES_SCRIPT]
 293     p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 294                          cwd=config.YARDSTICK_REPOS_DIR)
 295     output = p.communicate()[0]
 296     logger.debug('The result is: %s', output)
 297
 298
 299 def _create_task(task_id):
 300     async_handler = AsyncTaskHandler()
 301     task_dict = {
 302         'task_id': task_id,
 303         'status': 0
 304     }
 305     async_handler.insert(task_dict)
 306
 307
 308 def _update_task_status(task_id):
 309     async_handler = AsyncTaskHandler()
 310
 311     task = async_handler.get_task_by_taskid(task_id)
 312     async_handler.update_status(task, 1)
 313
 314
 315 def _update_task_error(task_id, error):
 316     async_handler = AsyncTaskHandler()
 317
 318     task = async_handler.get_task_by_taskid(task_id)
 319     async_handler.update_status(task, 2)
 320     async_handler.update_error(task, error)