api/resources/env_action.py

   1 ##############################################################################
   2 # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9 from __future__ import absolute_import
  10
  11 import errno
  12 import logging
  13 import os
  14 import subprocess
  15 import threading
  16 import time
  17 import uuid
  18 import glob
  19
  20 from six.moves import configparser
  21 from oslo_serialization import jsonutils
  22 from docker import Client
  23
  24 from api.database.handler import AsyncTaskHandler
  25 from api.utils import influx
  26 from api.utils.common import result_handler
  27 from yardstick.common import constants as consts
  28 from yardstick.common import utils as yardstick_utils
  29 from yardstick.common import openstack_utils
  30 from yardstick.common.httpClient import HttpClient
  31
  32
  33 logger = logging.getLogger(__name__)
  34 logger.setLevel(logging.DEBUG)
  35
  36
  37 def createGrafanaContainer(args):
  38     task_id = str(uuid.uuid4())
  39
  40     thread = threading.Thread(target=_create_grafana, args=(task_id,))
  41     thread.start()
  42
  43     return result_handler('success', {'task_id': task_id})
  44
  45
  46 def _create_grafana(task_id):
  47     _create_task(task_id)
  48
  49     client = Client(base_url=consts.DOCKER_URL)
  50
  51     try:
  52         image = '{}:{}'.format(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG)
  53         if not _check_image_exist(client, image):
  54             client.pull(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG)
  55
  56         _create_grafana_container(client)
  57
  58         time.sleep(5)
  59
  60         _create_data_source()
  61
  62         _create_dashboard()
  63
  64         _update_task_status(task_id)
  65     except Exception as e:
  66         _update_task_error(task_id, str(e))
  67         logger.exception('Error: %s', e)
  68
  69
  70 def _create_dashboard():
  71     url = 'http://admin:admin@%s:3000/api/dashboards/db' % consts.GRAFANA_IP
  72     path = os.path.join(consts.REPOS_DIR, 'dashboard', '*dashboard.json')
  73
  74     for i in sorted(glob.iglob(path)):
  75         with open(i) as f:
  76             data = jsonutils.load(f)
  77         HttpClient().post(url, data)
  78
  79
  80 def _create_data_source():
  81     url = 'http://admin:admin@%s:3000/api/datasources' % consts.GRAFANA_IP
  82     data = {
  83         "name": "yardstick",
  84         "type": "influxdb",
  85         "access": "proxy",
  86         "url": "http://%s:8086" % consts.INFLUXDB_IP,
  87         "password": "root",
  88         "user": "root",
  89         "database": "yardstick",
  90         "basicAuth": True,
  91         "basicAuthUser": "admin",
  92         "basicAuthPassword": "admin",
  93         "isDefault": False,
  94     }
  95     HttpClient().post(url, data)
  96
  97
  98 def _create_grafana_container(client):
  99     ports = [3000]
 100     port_bindings = {k: k for k in ports}
 101     host_config = client.create_host_config(port_bindings=port_bindings)
 102
 103     container = client.create_container(image='%s:%s' % (consts.GRAFANA_IMAGE,
 104                                                          consts.GRAFANA_TAG),
 105                                         ports=ports,
 106                                         detach=True,
 107                                         tty=True,
 108                                         host_config=host_config)
 109     client.start(container)
 110
 111
 112 def _check_image_exist(client, t):
 113     return any(t in a['RepoTags'][0] for a in client.images() if a['RepoTags'])
 114
 115
 116 def createInfluxDBContainer(args):
 117     task_id = str(uuid.uuid4())
 118
 119     thread = threading.Thread(target=_create_influxdb, args=(task_id,))
 120     thread.start()
 121
 122     return result_handler('success', {'task_id': task_id})
 123
 124
 125 def _create_influxdb(task_id):
 126     _create_task(task_id)
 127
 128     client = Client(base_url=consts.DOCKER_URL)
 129
 130     try:
 131         _change_output_to_influxdb()
 132
 133         if not _check_image_exist(client, '%s:%s' % (consts.INFLUXDB_IMAGE,
 134                                                      consts.INFLUXDB_TAG)):
 135             client.pull(consts.INFLUXDB_IMAGE, tag=consts.INFLUXDB_TAG)
 136
 137         _create_influxdb_container(client)
 138
 139         time.sleep(5)
 140
 141         _config_influxdb()
 142
 143         _update_task_status(task_id)
 144     except Exception as e:
 145         _update_task_error(task_id, str(e))
 146         logger.debug('Error: %s', e)
 147
 148
 149 def _create_influxdb_container(client):
 150
 151     ports = [8083, 8086]
 152     port_bindings = {k: k for k in ports}
 153     host_config = client.create_host_config(port_bindings=port_bindings)
 154
 155     container = client.create_container(image='%s:%s' % (consts.INFLUXDB_IMAGE,
 156                                                          consts.INFLUXDB_TAG),
 157                                         ports=ports,
 158                                         detach=True,
 159                                         tty=True,
 160                                         host_config=host_config)
 161     client.start(container)
 162
 163
 164 def _config_influxdb():
 165     try:
 166         client = influx.get_data_db_client()
 167         client.create_user(consts.INFLUXDB_USER,
 168                            consts.INFLUXDB_PASS,
 169                            consts.INFLUXDB_DB_NAME)
 170         client.create_database(consts.INFLUXDB_DB_NAME)
 171         logger.info('Success to config influxDB')
 172     except Exception as e:
 173         logger.debug('Failed to config influxDB: %s', e)
 174
 175
 176 def _change_output_to_influxdb():
 177     yardstick_utils.makedirs(consts.CONF_DIR)
 178
 179     parser = configparser.ConfigParser()
 180     parser.read(consts.CONF_SAMPLE_FILE)
 181
 182     parser.set('DEFAULT', 'dispatcher', 'influxdb')
 183     parser.set('dispatcher_influxdb', 'target',
 184                'http://%s:8086' % consts.INFLUXDB_IP)
 185
 186     with open(consts.CONF_FILE, 'w') as f:
 187         parser.write(f)
 188
 189
 190 def prepareYardstickEnv(args):
 191     task_id = str(uuid.uuid4())
 192
 193     thread = threading.Thread(target=_prepare_env_daemon, args=(task_id,))
 194     thread.start()
 195
 196     return result_handler('success', {'task_id': task_id})
 197
 198
 199 def _already_source_openrc():
 200     """Check if openrc is sourced already"""
 201     return all(os.environ.get(k) for k in ['OS_AUTH_URL', 'OS_USERNAME',
 202                                            'OS_PASSWORD', 'EXTERNAL_NETWORK'])
 203
 204
 205 def _prepare_env_daemon(task_id):
 206     _create_task(task_id)
 207
 208     try:
 209         _create_directories()
 210
 211         rc_file = consts.OPENRC
 212
 213         if not _already_source_openrc():
 214             if not os.path.exists(rc_file):
 215                 installer_ip = os.environ.get('INSTALLER_IP', '192.168.200.2')
 216                 installer_type = os.environ.get('INSTALLER_TYPE', 'compass')
 217                 _get_remote_rc_file(rc_file, installer_ip, installer_type)
 218                 _source_file(rc_file)
 219                 _append_external_network(rc_file)
 220             _source_file(rc_file)
 221
 222         _clean_images()
 223
 224         _load_images()
 225
 226         _update_task_status(task_id)
 227     except Exception as e:
 228         _update_task_error(task_id, str(e))
 229         logger.debug('Error: %s', e)
 230
 231
 232 def _create_directories():
 233     yardstick_utils.makedirs(consts.CONF_DIR)
 234
 235
 236 def _source_file(rc_file):
 237     yardstick_utils.source_env(rc_file)
 238
 239
 240 def _get_remote_rc_file(rc_file, installer_ip, installer_type):
 241
 242     os_fetch_script = os.path.join(consts.RELENG_DIR, consts.FETCH_SCRIPT)
 243
 244     try:
 245         cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type,
 246                '-a', installer_ip]
 247         p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 248         p.communicate()
 249
 250         if p.returncode != 0:
 251             logger.debug('Failed to fetch credentials from installer')
 252     except OSError as e:
 253         if e.errno != errno.EEXIST:
 254             raise
 255
 256
 257 def _append_external_network(rc_file):
 258     neutron_client = openstack_utils.get_neutron_client()
 259     networks = neutron_client.list_networks()['networks']
 260     try:
 261         ext_network = next(n['name'] for n in networks if n['router:external'])
 262     except StopIteration:
 263         logger.warning("Can't find external network")
 264     else:
 265         cmd = 'export EXTERNAL_NETWORK=%s' % ext_network
 266         try:
 267             with open(rc_file, 'a') as f:
 268                 f.write(cmd + '\n')
 269         except OSError as e:
 270             if e.errno != errno.EEXIST:
 271                 raise
 272
 273
 274 def _clean_images():
 275     cmd = [consts.CLEAN_IMAGES_SCRIPT]
 276     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR)
 277     output = p.communicate()[0]
 278     logger.debug('The result is: %s', output)
 279
 280
 281 def _load_images():
 282     cmd = [consts.LOAD_IMAGES_SCRIPT]
 283     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR)
 284     output = p.communicate()[0]
 285     logger.debug('The result is: %s', output)
 286
 287
 288 def _create_task(task_id):
 289     async_handler = AsyncTaskHandler()
 290     task_dict = {
 291         'task_id': task_id,
 292         'status': 0
 293     }
 294     async_handler.insert(task_dict)
 295
 296
 297 def _update_task_status(task_id):
 298     async_handler = AsyncTaskHandler()
 299
 300     task = async_handler.get_task_by_taskid(task_id)
 301     async_handler.update_status(task, 1)
 302
 303
 304 def _update_task_error(task_id, error):
 305     async_handler = AsyncTaskHandler()
 306
 307     task = async_handler.get_task_by_taskid(task_id)
 308     async_handler.update_status(task, 2)
 309     async_handler.update_error(task, error)