api/resources/env_action.py

   1 ##############################################################################
   2 # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9 from __future__ import absolute_import
  10
  11 import errno
  12 import logging
  13 import os
  14 import subprocess
  15 import threading
  16 import time
  17 import uuid
  18 import glob
  19
  20 from six.moves import configparser
  21 from oslo_serialization import jsonutils
  22
  23 from api import conf as api_conf
  24 from api.database.handler import AsyncTaskHandler
  25 from api.utils import influx
  26 from api.utils.common import result_handler
  27 from docker import Client
  28 from yardstick.common import constants as config
  29 from yardstick.common import utils as yardstick_utils
  30 from yardstick.common import openstack_utils
  31 from yardstick.common.httpClient import HttpClient
  32
  33 logger = logging.getLogger(__name__)
  34 logger.setLevel(logging.DEBUG)
  35
  36
  37 def createGrafanaContainer(args):
  38     task_id = str(uuid.uuid4())
  39
  40     thread = threading.Thread(target=_create_grafana, args=(task_id,))
  41     thread.start()
  42
  43     return result_handler('success', {'task_id': task_id})
  44
  45
  46 def _create_grafana(task_id):
  47     _create_task(task_id)
  48
  49     client = Client(base_url=config.DOCKER_URL)
  50
  51     try:
  52         if not _check_image_exist(client, '%s:%s' % (config.GRAFANA_IMAGE,
  53                                                      config.GRAFANA_TAGS)):
  54             client.pull(config.GRAFANA_IMAGE, config.GRAFANA_TAGS)
  55
  56         _create_grafana_container(client)
  57
  58         time.sleep(5)
  59
  60         _create_data_source()
  61
  62         _create_dashboard()
  63
  64         _update_task_status(task_id)
  65     except Exception as e:
  66         _update_task_error(task_id, str(e))
  67         logger.debug('Error: %s', e)
  68
  69
  70 def _create_dashboard():
  71     url = 'http://admin:admin@%s:3000/api/dashboards/db' % api_conf.GATEWAY_IP
  72     path = os.path.join(config.YARDSTICK_REPOS_DIR, 'dashboard',
  73                         '*dashboard.json')
  74
  75     for i in sorted(glob.iglob(path)):
  76         with open(i) as f:
  77             data = jsonutils.load(f)
  78         HttpClient().post(url, data)
  79
  80
  81 def _create_data_source():
  82     url = 'http://admin:admin@%s:3000/api/datasources' % api_conf.GATEWAY_IP
  83     data = {
  84         "name": "yardstick",
  85         "type": "influxdb",
  86         "access": "proxy",
  87         "url": "http://%s:8086" % api_conf.GATEWAY_IP,
  88         "password": "root",
  89         "user": "root",
  90         "database": "yardstick",
  91         "basicAuth": True,
  92         "basicAuthUser": "admin",
  93         "basicAuthPassword": "admin",
  94         "isDefault": False,
  95     }
  96     HttpClient().post(url, data)
  97
  98
  99 def _create_grafana_container(client):
 100     ports = [3000]
 101     port_bindings = {k: k for k in ports}
 102     host_config = client.create_host_config(port_bindings=port_bindings)
 103
 104     container = client.create_container(image='%s:%s' % (config.GRAFANA_IMAGE,
 105                                                          config.GRAFANA_TAGS),
 106                                         ports=ports,
 107                                         detach=True,
 108                                         tty=True,
 109                                         host_config=host_config)
 110     client.start(container)
 111
 112
 113 def _check_image_exist(client, t):
 114     return any(t in a['RepoTags'][0] for a in client.images() if a['RepoTags'])
 115
 116
 117 def createInfluxDBContainer(args):
 118     task_id = str(uuid.uuid4())
 119
 120     thread = threading.Thread(target=_create_influxdb, args=(task_id,))
 121     thread.start()
 122
 123     return result_handler('success', {'task_id': task_id})
 124
 125
 126 def _create_influxdb(task_id):
 127     _create_task(task_id)
 128
 129     client = Client(base_url=config.DOCKER_URL)
 130
 131     try:
 132         _change_output_to_influxdb()
 133
 134         if not _check_image_exist(client, '%s:%s' % (config.INFLUXDB_IMAGE,
 135                                                      config.INFLUXDB_TAG)):
 136             client.pull(config.INFLUXDB_IMAGE, tag=config.INFLUXDB_TAG)
 137
 138         _create_influxdb_container(client)
 139
 140         time.sleep(5)
 141
 142         _config_influxdb()
 143
 144         _update_task_status(task_id)
 145     except Exception as e:
 146         _update_task_error(task_id, str(e))
 147         logger.debug('Error: %s', e)
 148
 149
 150 def _create_influxdb_container(client):
 151
 152     ports = [8083, 8086]
 153     port_bindings = {k: k for k in ports}
 154     host_config = client.create_host_config(port_bindings=port_bindings)
 155
 156     container = client.create_container(image='%s:%s' % (config.INFLUXDB_IMAGE,
 157                                                          config.INFLUXDB_TAG),
 158                                         ports=ports,
 159                                         detach=True,
 160                                         tty=True,
 161                                         host_config=host_config)
 162     client.start(container)
 163
 164
 165 def _config_influxdb():
 166     try:
 167         client = influx.get_data_db_client()
 168         client.create_user(config.USER, config.PASSWORD, config.DATABASE)
 169         client.create_database(config.DATABASE)
 170         logger.info('Success to config influxDB')
 171     except Exception as e:
 172         logger.debug('Failed to config influxDB: %s', e)
 173
 174
 175 def _change_output_to_influxdb():
 176     yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR)
 177
 178     parser = configparser.ConfigParser()
 179     parser.read(config.YARDSTICK_CONFIG_SAMPLE_FILE)
 180
 181     parser.set('DEFAULT', 'dispatcher', 'influxdb')
 182     parser.set('dispatcher_influxdb', 'target',
 183                'http://%s:8086' % api_conf.GATEWAY_IP)
 184
 185     with open(config.YARDSTICK_CONFIG_FILE, 'w') as f:
 186         parser.write(f)
 187
 188
 189 def prepareYardstickEnv(args):
 190     task_id = str(uuid.uuid4())
 191
 192     thread = threading.Thread(target=_prepare_env_daemon, args=(task_id,))
 193     thread.start()
 194
 195     return result_handler('success', {'task_id': task_id})
 196
 197
 198 def _already_source_openrc():
 199     """Check if openrc is sourced already"""
 200     return all(os.environ.get(k) for k in ['OS_AUTH_URL', 'OS_USERNAME',
 201                                            'OS_PASSWORD', 'OS_TENANT_NAME',
 202                                            'EXTERNAL_NETWORK'])
 203
 204
 205 def _prepare_env_daemon(task_id):
 206     _create_task(task_id)
 207
 208     installer_ip = os.environ.get('INSTALLER_IP', 'undefined')
 209     installer_type = os.environ.get('INSTALLER_TYPE', 'undefined')
 210
 211     try:
 212         _check_variables(installer_ip, installer_type)
 213
 214         _create_directories()
 215
 216         rc_file = config.OPENSTACK_RC_FILE
 217
 218         if not _already_source_openrc():
 219             _get_remote_rc_file(rc_file, installer_ip, installer_type)
 220             _source_file(rc_file)
 221             _append_external_network(rc_file)
 222
 223         # update the external_network
 224         _source_file(rc_file)
 225
 226         _clean_images()
 227
 228         _load_images()
 229
 230         _update_task_status(task_id)
 231     except Exception as e:
 232         _update_task_error(task_id, str(e))
 233         logger.debug('Error: %s', e)
 234
 235
 236 def _check_variables(installer_ip, installer_type):
 237
 238     if installer_ip == 'undefined':
 239         raise SystemExit('Missing INSTALLER_IP')
 240
 241     if installer_type == 'undefined':
 242         raise SystemExit('Missing INSTALLER_TYPE')
 243     elif installer_type not in config.INSTALLERS:
 244         raise SystemExit('INSTALLER_TYPE is not correct')
 245
 246
 247 def _create_directories():
 248     yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR)
 249
 250
 251 def _source_file(rc_file):
 252     yardstick_utils.source_env(rc_file)
 253
 254
 255 def _get_remote_rc_file(rc_file, installer_ip, installer_type):
 256
 257     os_fetch_script = os.path.join(config.RELENG_DIR, config.OS_FETCH_SCRIPT)
 258
 259     try:
 260         cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type,
 261                '-a', installer_ip]
 262         p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 263         p.communicate()
 264
 265         if p.returncode != 0:
 266             logger.debug('Failed to fetch credentials from installer')
 267     except OSError as e:
 268         if e.errno != errno.EEXIST:
 269             raise
 270
 271
 272 def _append_external_network(rc_file):
 273     neutron_client = openstack_utils.get_neutron_client()
 274     networks = neutron_client.list_networks()['networks']
 275     try:
 276         ext_network = next(n['name'] for n in networks if n['router:external'])
 277     except StopIteration:
 278         logger.warning("Can't find external network")
 279     else:
 280         cmd = 'export EXTERNAL_NETWORK=%s' % ext_network
 281         try:
 282             with open(rc_file, 'a') as f:
 283                 f.write(cmd + '\n')
 284         except OSError as e:
 285             if e.errno != errno.EEXIST:
 286                 raise
 287
 288
 289 def _clean_images():
 290     cmd = [config.CLEAN_IMAGES_SCRIPT]
 291     p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 292                          cwd=config.YARDSTICK_REPOS_DIR)
 293     output = p.communicate()[0]
 294     logger.debug('The result is: %s', output)
 295
 296
 297 def _load_images():
 298     cmd = [config.LOAD_IMAGES_SCRIPT]
 299     p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 300                          cwd=config.YARDSTICK_REPOS_DIR)
 301     output = p.communicate()[0]
 302     logger.debug('The result is: %s', output)
 303
 304
 305 def _create_task(task_id):
 306     async_handler = AsyncTaskHandler()
 307     task_dict = {
 308         'task_id': task_id,
 309         'status': 0
 310     }
 311     async_handler.insert(task_dict)
 312
 313
 314 def _update_task_status(task_id):
 315     async_handler = AsyncTaskHandler()
 316
 317     task = async_handler.get_task_by_taskid(task_id)
 318     async_handler.update_status(task, 1)
 319
 320
 321 def _update_task_error(task_id, error):
 322     async_handler = AsyncTaskHandler()
 323
 324     task = async_handler.get_task_by_taskid(task_id)
 325     async_handler.update_status(task, 2)
 326     async_handler.update_error(task, error)