api/resources/v2/containers.py

   1 ##############################################################################
   2 # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9 from __future__ import absolute_import
  10
  11 import logging
  12 import threading
  13 import time
  14 import uuid
  15 import os
  16 import glob
  17
  18 from six.moves import configparser
  19 from oslo_serialization import jsonutils
  20 from docker import Client
  21
  22 from api import ApiResource
  23 from api.utils import influx
  24 from api.database.v2.handlers import V2ContainerHandler
  25 from api.database.v2.handlers import V2EnvironmentHandler
  26 from yardstick.common import constants as consts
  27 from yardstick.common import utils
  28 from yardstick.common.utils import result_handler
  29 from yardstick.common.utils import get_free_port
  30 from yardstick.common.httpClient import HttpClient
  31
  32
  33 LOG = logging.getLogger(__name__)
  34 LOG.setLevel(logging.DEBUG)
  35
  36 environment_handler = V2EnvironmentHandler()
  37 container_handler = V2ContainerHandler()
  38
  39
  40 class V2Containers(ApiResource):
  41
  42     def post(self):
  43         return self._dispatch_post()
  44
  45     def create_influxdb(self, args):
  46         try:
  47             environment_id = args['environment_id']
  48         except KeyError:
  49             return result_handler(consts.API_ERROR, 'environment_id must be provided')
  50
  51         try:
  52             uuid.UUID(environment_id)
  53         except ValueError:
  54             return result_handler(consts.API_ERROR, 'invalid environment id')
  55
  56         try:
  57             environment = environment_handler.get_by_uuid(environment_id)
  58         except ValueError:
  59             return result_handler(consts.API_ERROR, 'no such environment id')
  60
  61         container_info = environment.container_id
  62         container_info = jsonutils.loads(container_info) if container_info else {}
  63
  64         if container_info.get('influxdb'):
  65             return result_handler(consts.API_ERROR, 'influxdb container already exist')
  66
  67         name = 'influxdb-{}'.format(environment_id[:8])
  68         port = get_free_port(consts.SERVER_IP)
  69         container_id = str(uuid.uuid4())
  70         LOG.info('%s will launch on : %s', name, port)
  71
  72         LOG.info('launch influxdb background')
  73         args = (name, port, container_id)
  74         thread = threading.Thread(target=self._create_influxdb, args=args)
  75         thread.start()
  76
  77         LOG.info('record container in database')
  78         container_init_data = {
  79             'uuid': container_id,
  80             'environment_id': environment_id,
  81             'name': name,
  82             'port': port,
  83             'status': 0
  84         }
  85         container_handler.insert(container_init_data)
  86
  87         LOG.info('update container in environment')
  88         container_info['influxdb'] = container_id
  89         environment_info = {'container_id': jsonutils.dumps(container_info)}
  90         environment_handler.update_attr(environment_id, environment_info)
  91
  92         return result_handler(consts.API_SUCCESS, {'uuid': container_id})
  93
  94     def _check_image_exist(self, client, t):
  95         return any(t in a['RepoTags'][0]
  96                    for a in client.images() if a['RepoTags'])
  97
  98     def _create_influxdb(self, name, port, container_id):
  99         client = Client(base_url=consts.DOCKER_URL)
 100
 101         try:
 102             LOG.info('Checking if influxdb image exist')
 103             if not self._check_image_exist(client, '%s:%s' %
 104                                            (consts.INFLUXDB_IMAGE,
 105                                             consts.INFLUXDB_TAG)):
 106                 LOG.info('Influxdb image not exist, start pulling')
 107                 client.pull(consts.INFLUXDB_IMAGE, tag=consts.INFLUXDB_TAG)
 108
 109             LOG.info('Createing influxdb container')
 110             container = self._create_influxdb_container(client, name, port)
 111             LOG.info('Influxdb container is created')
 112
 113             time.sleep(5)
 114
 115             container = client.inspect_container(container['Id'])
 116             ip = container['NetworkSettings']['Networks']['bridge']['IPAddress']
 117             LOG.debug('container ip is: %s', ip)
 118
 119             LOG.info('Changing output to influxdb')
 120             self._change_output_to_influxdb(ip)
 121
 122             LOG.info('Config influxdb')
 123             self._config_influxdb()
 124
 125             container_handler.update_attr(container_id, {'status': 1})
 126
 127             LOG.info('Finished')
 128         except Exception:
 129             container_handler.update_attr(container_id, {'status': 2})
 130             LOG.exception('Creating influxdb failed')
 131
 132     def _create_influxdb_container(self, client, name, port):
 133
 134         ports = [port]
 135         port_bindings = {8086: port}
 136         restart_policy = {"MaximumRetryCount": 0, "Name": "always"}
 137         host_config = client.create_host_config(port_bindings=port_bindings,
 138                                                 restart_policy=restart_policy)
 139
 140         LOG.info('Creating container')
 141         container = client.create_container(image='%s:%s' %
 142                                             (consts.INFLUXDB_IMAGE,
 143                                              consts.INFLUXDB_TAG),
 144                                             ports=ports,
 145                                             name=name,
 146                                             detach=True,
 147                                             tty=True,
 148                                             host_config=host_config)
 149         LOG.info('Starting container')
 150         client.start(container)
 151         return container
 152
 153     def _config_influxdb(self):
 154         try:
 155             client = influx.get_data_db_client()
 156             client.create_user(consts.INFLUXDB_USER,
 157                                consts.INFLUXDB_PASS,
 158                                consts.INFLUXDB_DB_NAME)
 159             client.create_database(consts.INFLUXDB_DB_NAME)
 160             LOG.info('Success to config influxDB')
 161         except Exception:
 162             LOG.exception('Config influxdb failed')
 163
 164     def _change_output_to_influxdb(self, ip):
 165         utils.makedirs(consts.CONF_DIR)
 166
 167         parser = configparser.ConfigParser()
 168         LOG.info('Reading output sample configuration')
 169         parser.read(consts.CONF_SAMPLE_FILE)
 170
 171         LOG.info('Set dispatcher to influxdb')
 172         parser.set('DEFAULT', 'dispatcher', 'influxdb')
 173         parser.set('dispatcher_influxdb', 'target',
 174                    'http://{}:{}'.format(ip, 8086))
 175
 176         LOG.info('Writing to %s', consts.CONF_FILE)
 177         with open(consts.CONF_FILE, 'w') as f:
 178             parser.write(f)
 179
 180     def create_grafana(self, args):
 181         try:
 182             environment_id = args['environment_id']
 183         except KeyError:
 184             return result_handler(consts.API_ERROR, 'environment_id must be provided')
 185
 186         try:
 187             uuid.UUID(environment_id)
 188         except ValueError:
 189             return result_handler(consts.API_ERROR, 'invalid environment id')
 190
 191         try:
 192             environment = environment_handler.get_by_uuid(environment_id)
 193         except ValueError:
 194             return result_handler(consts.API_ERROR, 'no such environment id')
 195
 196         container_info = environment.container_id
 197         container_info = jsonutils.loads(container_info) if container_info else {}
 198
 199         if not container_info.get('influxdb'):
 200             return result_handler(consts.API_ERROR, 'influxdb not set')
 201
 202         if container_info.get('grafana'):
 203             return result_handler(consts.API_ERROR, 'grafana container already exists')
 204
 205         name = 'grafana-{}'.format(environment_id[:8])
 206         port = get_free_port(consts.SERVER_IP)
 207         container_id = str(uuid.uuid4())
 208
 209         args = (name, port, container_id)
 210         thread = threading.Thread(target=self._create_grafana, args=args)
 211         thread.start()
 212
 213         container_init_data = {
 214             'uuid': container_id,
 215             'environment_id': environment_id,
 216             'name': name,
 217             'port': port,
 218             'status': 0
 219         }
 220         container_handler.insert(container_init_data)
 221
 222         container_info['grafana'] = container_id
 223         environment_info = {'container_id': jsonutils.dumps(container_info)}
 224         environment_handler.update_attr(environment_id, environment_info)
 225
 226         return result_handler(consts.API_SUCCESS, {'uuid': container_id})
 227
 228     def _create_grafana(self, name, port, container_id):
 229         client = Client(base_url=consts.DOCKER_URL)
 230
 231         try:
 232             LOG.info('Checking if grafana image exist')
 233             image = '{}:{}'.format(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG)
 234             if not self._check_image_exist(client, image):
 235                 LOG.info('Grafana image not exist, start pulling')
 236                 client.pull(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG)
 237
 238             LOG.info('Createing grafana container')
 239             container = self._create_grafana_container(client, name, port)
 240             LOG.info('Grafana container is created')
 241
 242             time.sleep(5)
 243
 244             container = client.inspect_container(container['Id'])
 245             ip = container['NetworkSettings']['Networks']['bridge']['IPAddress']
 246             LOG.debug('container ip is: %s', ip)
 247
 248             LOG.info('Creating data source for grafana')
 249             self._create_data_source(ip)
 250
 251             LOG.info('Creating dashboard for grafana')
 252             self._create_dashboard(ip)
 253
 254             container_handler.update_attr(container_id, {'status': 1})
 255             LOG.info('Finished')
 256         except Exception:
 257             container_handler.update_attr(container_id, {'status': 2})
 258             LOG.exception('Create grafana failed')
 259
 260     def _create_dashboard(self, ip):
 261         url = 'http://admin:admin@{}:{}/api/dashboards/db'.format(ip, 3000)
 262         path = os.path.join(consts.REPOS_DIR, 'dashboard', '*dashboard.json')
 263
 264         for i in sorted(glob.iglob(path)):
 265             with open(i) as f:
 266                 data = jsonutils.load(f)
 267             try:
 268                 HttpClient().post(url, data)
 269             except Exception:
 270                 LOG.exception('Create dashboard %s failed', i)
 271                 raise
 272
 273     def _create_data_source(self, ip):
 274         url = 'http://admin:admin@{}:{}/api/datasources'.format(ip, 3000)
 275
 276         influx_conf = utils.parse_ini_file(consts.CONF_FILE)
 277         try:
 278             influx_url = influx_conf['dispatcher_influxdb']['target']
 279         except KeyError:
 280             LOG.exception('influxdb url not set in yardstick.conf')
 281             raise
 282
 283         data = {
 284             "name": "yardstick",
 285             "type": "influxdb",
 286             "access": "proxy",
 287             "url": influx_url,
 288             "password": "root",
 289             "user": "root",
 290             "database": "yardstick",
 291             "basicAuth": True,
 292             "basicAuthUser": "admin",
 293             "basicAuthPassword": "admin",
 294             "isDefault": False,
 295         }
 296         try:
 297             HttpClient().post(url, data)
 298         except Exception:
 299             LOG.exception('Create datasources failed')
 300             raise
 301
 302     def _create_grafana_container(self, client, name, port):
 303         ports = [3000]
 304         port_bindings = {3000: port}
 305         restart_policy = {"MaximumRetryCount": 0, "Name": "always"}
 306         host_config = client.create_host_config(port_bindings=port_bindings,
 307                                                 restart_policy=restart_policy)
 308
 309         LOG.info('Creating container')
 310         container = client.create_container(image='%s:%s' %
 311                                             (consts.GRAFANA_IMAGE,
 312                                              consts.GRAFANA_TAG),
 313                                             name=name,
 314                                             ports=ports,
 315                                             detach=True,
 316                                             tty=True,
 317                                             host_config=host_config)
 318         LOG.info('Starting container')
 319         client.start(container)
 320         return container