api/resources/env_action.py

   1 ##############################################################################
   2 # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9 from __future__ import absolute_import
  10
  11 import errno
  12 import logging
  13 import os
  14 import subprocess
  15 import threading
  16 import time
  17 import uuid
  18 import glob
  19
  20 from six.moves import configparser
  21 from oslo_serialization import jsonutils
  22 from docker import Client
  23 from docker.utils import create_host_config
  24
  25 from api.database.handler import AsyncTaskHandler
  26 from api.utils import influx
  27 from api.utils.common import result_handler
  28 from yardstick.common import constants as consts
  29 from yardstick.common import utils as yardstick_utils
  30 from yardstick.common import openstack_utils
  31 from yardstick.common.httpClient import HttpClient
  32
  33
  34 logger = logging.getLogger(__name__)
  35 logger.setLevel(logging.DEBUG)
  36
  37
  38 def createGrafanaContainer(args):
  39     task_id = str(uuid.uuid4())
  40
  41     thread = threading.Thread(target=_create_grafana, args=(task_id,))
  42     thread.start()
  43
  44     return result_handler('success', {'task_id': task_id})
  45
  46
  47 def _create_grafana(task_id):
  48     _create_task(task_id)
  49
  50     client = Client(base_url=consts.DOCKER_URL)
  51
  52     try:
  53         image = '{}:{}'.format(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG)
  54         if not _check_image_exist(client, image):
  55             client.pull(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG)
  56
  57         _create_grafana_container(client)
  58
  59         time.sleep(5)
  60
  61         _create_data_source()
  62
  63         _create_dashboard()
  64
  65         _update_task_status(task_id)
  66     except Exception as e:
  67         _update_task_error(task_id, str(e))
  68         logger.exception('Error: %s', e)
  69
  70
  71 def _create_dashboard():
  72     url = 'http://admin:admin@%s:3000/api/dashboards/db' % consts.GRAFANA_IP
  73     path = os.path.join(consts.REPOS_DIR, 'dashboard', '*dashboard.json')
  74
  75     for i in sorted(glob.iglob(path)):
  76         with open(i) as f:
  77             data = jsonutils.load(f)
  78         HttpClient().post(url, data)
  79
  80
  81 def _create_data_source():
  82     url = 'http://admin:admin@%s:3000/api/datasources' % consts.GRAFANA_IP
  83     data = {
  84         "name": "yardstick",
  85         "type": "influxdb",
  86         "access": "proxy",
  87         "url": "http://%s:8086" % consts.INFLUXDB_IP,
  88         "password": "root",
  89         "user": "root",
  90         "database": "yardstick",
  91         "basicAuth": True,
  92         "basicAuthUser": "admin",
  93         "basicAuthPassword": "admin",
  94         "isDefault": False,
  95     }
  96     HttpClient().post(url, data)
  97
  98
  99 def _create_grafana_container(client):
 100     ports = [3000]
 101     port_bindings = {k: k for k in ports}
 102     restart_policy = {"MaximumRetryCount": 0, "Name": "always"}
 103     host_config = client.create_host_config(port_bindings=port_bindings,
 104                                             restart_policy=restart_policy)
 105
 106     container = client.create_container(image='%s:%s' % (consts.GRAFANA_IMAGE,
 107                                                          consts.GRAFANA_TAG),
 108                                         ports=ports,
 109                                         detach=True,
 110                                         tty=True,
 111                                         host_config=host_config)
 112     client.start(container)
 113
 114
 115 def _check_image_exist(client, t):
 116     return any(t in a['RepoTags'][0] for a in client.images() if a['RepoTags'])
 117
 118
 119 def createInfluxDBContainer(args):
 120     task_id = str(uuid.uuid4())
 121
 122     thread = threading.Thread(target=_create_influxdb, args=(task_id,))
 123     thread.start()
 124
 125     return result_handler('success', {'task_id': task_id})
 126
 127
 128 def _create_influxdb(task_id):
 129     _create_task(task_id)
 130
 131     client = Client(base_url=consts.DOCKER_URL)
 132
 133     try:
 134         _change_output_to_influxdb()
 135
 136         if not _check_image_exist(client, '%s:%s' % (consts.INFLUXDB_IMAGE,
 137                                                      consts.INFLUXDB_TAG)):
 138             client.pull(consts.INFLUXDB_IMAGE, tag=consts.INFLUXDB_TAG)
 139
 140         _create_influxdb_container(client)
 141
 142         time.sleep(5)
 143
 144         _config_influxdb()
 145
 146         _update_task_status(task_id)
 147     except Exception as e:
 148         _update_task_error(task_id, str(e))
 149         logger.debug('Error: %s', e)
 150
 151
 152 def _create_influxdb_container(client):
 153
 154     ports = [8083, 8086]
 155     port_bindings = {k: k for k in ports}
 156     restart_policy = {"MaximumRetryCount": 0, "Name": "always"}
 157     host_config = client.create_host_config(port_bindings=port_bindings,
 158                                             restart_policy=restart_policy)
 159
 160     container = client.create_container(image='%s:%s' % (consts.INFLUXDB_IMAGE,
 161                                                          consts.INFLUXDB_TAG),
 162                                         ports=ports,
 163                                         detach=True,
 164                                         tty=True,
 165                                         host_config=host_config)
 166     client.start(container)
 167
 168
 169 def _config_influxdb():
 170     try:
 171         client = influx.get_data_db_client()
 172         client.create_user(consts.INFLUXDB_USER,
 173                            consts.INFLUXDB_PASS,
 174                            consts.INFLUXDB_DB_NAME)
 175         client.create_database(consts.INFLUXDB_DB_NAME)
 176         logger.info('Success to config influxDB')
 177     except Exception as e:
 178         logger.debug('Failed to config influxDB: %s', e)
 179
 180
 181 def _change_output_to_influxdb():
 182     yardstick_utils.makedirs(consts.CONF_DIR)
 183
 184     parser = configparser.ConfigParser()
 185     parser.read(consts.CONF_SAMPLE_FILE)
 186
 187     parser.set('DEFAULT', 'dispatcher', 'influxdb')
 188     parser.set('dispatcher_influxdb', 'target',
 189                'http://%s:8086' % consts.INFLUXDB_IP)
 190
 191     with open(consts.CONF_FILE, 'w') as f:
 192         parser.write(f)
 193
 194
 195 def prepareYardstickEnv(args):
 196     task_id = str(uuid.uuid4())
 197
 198     thread = threading.Thread(target=_prepare_env_daemon, args=(task_id,))
 199     thread.start()
 200
 201     return result_handler('success', {'task_id': task_id})
 202
 203
 204 def _already_source_openrc():
 205     """Check if openrc is sourced already"""
 206     return all(os.environ.get(k) for k in ['OS_AUTH_URL', 'OS_USERNAME',
 207                                            'OS_PASSWORD', 'EXTERNAL_NETWORK'])
 208
 209
 210 def _prepare_env_daemon(task_id):
 211     _create_task(task_id)
 212
 213     try:
 214         _create_directories()
 215
 216         rc_file = consts.OPENRC
 217
 218         if not _already_source_openrc():
 219             if not os.path.exists(rc_file):
 220                 installer_ip = os.environ.get('INSTALLER_IP', '192.168.200.2')
 221                 installer_type = os.environ.get('INSTALLER_TYPE', 'compass')
 222                 _get_remote_rc_file(rc_file, installer_ip, installer_type)
 223                 _source_file(rc_file)
 224                 _append_external_network(rc_file)
 225             _source_file(rc_file)
 226
 227         _clean_images()
 228
 229         _load_images()
 230
 231         _update_task_status(task_id)
 232     except Exception as e:
 233         _update_task_error(task_id, str(e))
 234         logger.debug('Error: %s', e)
 235
 236
 237 def _create_directories():
 238     yardstick_utils.makedirs(consts.CONF_DIR)
 239
 240
 241 def _source_file(rc_file):
 242     yardstick_utils.source_env(rc_file)
 243
 244
 245 def _get_remote_rc_file(rc_file, installer_ip, installer_type):
 246
 247     os_fetch_script = os.path.join(consts.RELENG_DIR, consts.FETCH_SCRIPT)
 248
 249     try:
 250         cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type,
 251                '-a', installer_ip]
 252         p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 253         p.communicate()
 254
 255         if p.returncode != 0:
 256             logger.debug('Failed to fetch credentials from installer')
 257     except OSError as e:
 258         if e.errno != errno.EEXIST:
 259             raise
 260
 261
 262 def _append_external_network(rc_file):
 263     neutron_client = openstack_utils.get_neutron_client()
 264     networks = neutron_client.list_networks()['networks']
 265     try:
 266         ext_network = next(n['name'] for n in networks if n['router:external'])
 267     except StopIteration:
 268         logger.warning("Can't find external network")
 269     else:
 270         cmd = 'export EXTERNAL_NETWORK=%s' % ext_network
 271         try:
 272             with open(rc_file, 'a') as f:
 273                 f.write(cmd + '\n')
 274         except OSError as e:
 275             if e.errno != errno.EEXIST:
 276                 raise
 277
 278
 279 def _clean_images():
 280     cmd = [consts.CLEAN_IMAGES_SCRIPT]
 281     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR)
 282     output = p.communicate()[0]
 283     logger.debug('The result is: %s', output)
 284
 285
 286 def _load_images():
 287     cmd = [consts.LOAD_IMAGES_SCRIPT]
 288     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR)
 289     output = p.communicate()[0]
 290     logger.debug('The result is: %s', output)
 291
 292
 293 def _create_task(task_id):
 294     async_handler = AsyncTaskHandler()
 295     task_dict = {
 296         'task_id': task_id,
 297         'status': 0
 298     }
 299     async_handler.insert(task_dict)
 300
 301
 302 def _update_task_status(task_id):
 303     async_handler = AsyncTaskHandler()
 304
 305     task = async_handler.get_task_by_taskid(task_id)
 306     async_handler.update_status(task, 1)
 307
 308
 309 def _update_task_error(task_id, error):
 310     async_handler = AsyncTaskHandler()
 311
 312     task = async_handler.get_task_by_taskid(task_id)
 313     async_handler.update_status(task, 2)
 314     async_handler.update_error(task, error)