api/resources/v2/containers.py

   1 ##############################################################################
   2 # Copyright (c) 2017 Huawei Technologies Co.,Ltd.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9 from __future__ import absolute_import
  10
  11 import logging
  12 import threading
  13 import time
  14 import uuid
  15 import os
  16 import glob
  17
  18 from six.moves import configparser
  19 from oslo_serialization import jsonutils
  20 from docker import Client
  21
  22 from api import ApiResource
  23 from api.utils import influx
  24 from api.database.v2.handlers import V2ContainerHandler
  25 from api.database.v2.handlers import V2EnvironmentHandler
  26 from yardstick.common import constants as consts
  27 from yardstick.common import utils
  28 from yardstick.common.utils import result_handler
  29 from yardstick.common.utils import get_free_port
  30 from yardstick.common.httpClient import HttpClient
  31
  32
  33 LOG = logging.getLogger(__name__)
  34 LOG.setLevel(logging.DEBUG)
  35
  36 environment_handler = V2EnvironmentHandler()
  37 container_handler = V2ContainerHandler()
  38
  39
  40 class V2Containers(ApiResource):
  41
  42     def post(self):
  43         return self._dispatch_post()
  44
  45     def create_influxdb(self, args):
  46         try:
  47             environment_id = args['environment_id']
  48         except KeyError:
  49             return result_handler(consts.API_ERROR, 'environment_id must be provided')
  50
  51         try:
  52             uuid.UUID(environment_id)
  53         except ValueError:
  54             return result_handler(consts.API_ERROR, 'invalid environment id')
  55
  56         try:
  57             environment = environment_handler.get_by_uuid(environment_id)
  58         except ValueError:
  59             return result_handler(consts.API_ERROR, 'no such environment id')
  60
  61         container_info = environment.container_id
  62         container_info = jsonutils.loads(container_info) if container_info else {}
  63
  64         if container_info.get('influxdb'):
  65             return result_handler(consts.API_ERROR, 'influxdb container already exist')
  66
  67         name = 'influxdb-{}'.format(environment_id[:8])
  68         port = get_free_port(consts.SERVER_IP)
  69         container_id = str(uuid.uuid4())
  70         LOG.info('%s will launch on : %s', name, port)
  71
  72         LOG.info('launch influxdb background')
  73         args = (name, port, container_id)
  74         thread = threading.Thread(target=self._create_influxdb, args=args)
  75         thread.start()
  76
  77         LOG.info('record container in database')
  78         container_init_data = {
  79             'uuid': container_id,
  80             'environment_id': environment_id,
  81             'name': name,
  82             'port': port,
  83             'status': 0
  84         }
  85         container_handler.insert(container_init_data)
  86
  87         LOG.info('update container in environment')
  88         container_info['influxdb'] = container_id
  89         environment_info = {'container_id': jsonutils.dumps(container_info)}
  90         environment_handler.update_attr(environment_id, environment_info)
  91
  92         return result_handler(consts.API_SUCCESS, {'uuid': container_id})
  93
  94     def _check_image_exist(self, client, t):
  95         return any(t in a['RepoTags'][0]
  96                    for a in client.images() if a['RepoTags'])
  97
  98     def _create_influxdb(self, name, port, container_id):
  99         client = Client(base_url=consts.DOCKER_URL)
 100
 101         try:
 102             LOG.info('Checking if influxdb image exist')
 103             if not self._check_image_exist(client, '%s:%s' %
 104                                            (consts.INFLUXDB_IMAGE,
 105                                             consts.INFLUXDB_TAG)):
 106                 LOG.info('Influxdb image not exist, start pulling')
 107                 client.pull(consts.INFLUXDB_IMAGE, tag=consts.INFLUXDB_TAG)
 108
 109             LOG.info('Createing influxdb container')
 110             container = self._create_influxdb_container(client, name, port)
 111             LOG.info('Influxdb container is created')
 112
 113             time.sleep(5)
 114
 115             container = client.inspect_container(container['Id'])
 116             ip = container['NetworkSettings']['Networks']['bridge']['IPAddress']
 117             LOG.debug('container ip is: %s', ip)
 118
 119             LOG.info('Changing output to influxdb')
 120             self._change_output_to_influxdb(ip)
 121
 122             LOG.info('Config influxdb')
 123             self._config_influxdb()
 124
 125             container_handler.update_attr(container_id, {'status': 1})
 126
 127             LOG.info('Finished')
 128         except Exception:
 129             container_handler.update_attr(container_id, {'status': 2})
 130             LOG.exception('Creating influxdb failed')
 131
 132     def _create_influxdb_container(self, client, name, port):
 133
 134         ports = [port]
 135         port_bindings = {8086: port}
 136         restart_policy = {"MaximumRetryCount": 0, "Name": "always"}
 137         host_config = client.create_host_config(port_bindings=port_bindings,
 138                                                 restart_policy=restart_policy)
 139
 140         LOG.info('Creating container')
 141         container = client.create_container(image='%s:%s' %
 142                                             (consts.INFLUXDB_IMAGE,
 143                                              consts.INFLUXDB_TAG),
 144                                             ports=ports,
 145                                             name=name,
 146                                             detach=True,
 147                                             tty=True,
 148                                             host_config=host_config)
 149         LOG.info('Starting container')
 150         client.start(container)
 151         return container
 152
 153     def _config_influxdb(self):
 154         try:
 155             client = influx.get_data_db_client()
 156             client.create_user(consts.INFLUXDB_USER,
 157                                consts.INFLUXDB_PASS,
 158                                consts.INFLUXDB_DB_NAME)
 159             client.create_database(consts.INFLUXDB_DB_NAME)
 160             LOG.info('Success to config influxDB')
 161         except Exception:
 162             LOG.exception('Config influxdb failed')
 163
 164     def _change_output_to_influxdb(self, ip):
 165         utils.makedirs(consts.CONF_DIR)
 166
 167         parser = configparser.ConfigParser()
 168         LOG.info('Reading output sample configuration')
 169         parser.read(consts.CONF_SAMPLE_FILE)
 170
 171         LOG.info('Set dispatcher to influxdb')
 172         parser.set('DEFAULT', 'dispatcher', 'influxdb')
 173         parser.set('dispatcher_influxdb', 'target',
 174                    'http://{}:{}'.format(ip, 8086))
 175
 176         LOG.info('Writing to %s', consts.CONF_FILE)
 177         with open(consts.CONF_FILE, 'w') as f:
 178             parser.write(f)
 179
 180     def create_grafana(self, args):
 181         try:
 182             environment_id = args['environment_id']
 183         except KeyError:
 184             return result_handler(consts.API_ERROR, 'environment_id must be provided')
 185
 186         try:
 187             uuid.UUID(environment_id)
 188         except ValueError:
 189             return result_handler(consts.API_ERROR, 'invalid environment id')
 190
 191         try:
 192             environment = environment_handler.get_by_uuid(environment_id)
 193         except ValueError:
 194             return result_handler(consts.API_ERROR, 'no such environment id')
 195
 196         container_info = environment.container_id
 197         container_info = jsonutils.loads(container_info) if container_info else {}
 198
 199         if not container_info.get('influxdb'):
 200             return result_handler(consts.API_ERROR, 'influxdb not set')
 201
 202         if container_info.get('grafana'):
 203             return result_handler(consts.API_ERROR, 'grafana container already exists')
 204
 205         name = 'grafana-{}'.format(environment_id[:8])
 206         port = get_free_port(consts.SERVER_IP)
 207         container_id = str(uuid.uuid4())
 208
 209         args = (name, port, container_id)
 210         thread = threading.Thread(target=self._create_grafana, args=args)
 211         thread.start()
 212
 213         container_init_data = {
 214             'uuid': container_id,
 215             'environment_id': environment_id,
 216             'name': name,
 217             'port': port,
 218             'status': 0
 219         }
 220         container_handler.insert(container_init_data)
 221
 222         container_info['grafana'] = container_id
 223         environment_info = {'container_id': jsonutils.dumps(container_info)}
 224         environment_handler.update_attr(environment_id, environment_info)
 225
 226         return result_handler(consts.API_SUCCESS, {'uuid': container_id})
 227
 228     def _create_grafana(self, name, port, container_id):
 229         client = Client(base_url=consts.DOCKER_URL)
 230
 231         try:
 232             LOG.info('Checking if grafana image exist')
 233             image = '{}:{}'.format(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG)
 234             if not self._check_image_exist(client, image):
 235                 LOG.info('Grafana image not exist, start pulling')
 236                 client.pull(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG)
 237
 238             LOG.info('Createing grafana container')
 239             container = self._create_grafana_container(client, name, port)
 240             LOG.info('Grafana container is created')
 241
 242             time.sleep(5)
 243
 244             container = client.inspect_container(container['Id'])
 245             ip = container['NetworkSettings']['Networks']['bridge']['IPAddress']
 246             LOG.debug('container ip is: %s', ip)
 247
 248             LOG.info('Creating data source for grafana')
 249             self._create_data_source(ip)
 250
 251             LOG.info('Creating dashboard for grafana')
 252             self._create_dashboard(ip)
 253
 254             container_handler.update_attr(container_id, {'status': 1})
 255             LOG.info('Finished')
 256         except Exception:
 257             container_handler.update_attr(container_id, {'status': 2})
 258             LOG.exception('Create grafana failed')
 259
 260     def _create_dashboard(self, ip):
 261         url = 'http://admin:admin@{}:{}/api/dashboards/db'.format(ip, 3000)
 262         path = os.path.join(consts.REPOS_DIR, 'dashboard', 'opnfv_yardstick_tc*.json')
 263
 264         for i in sorted(glob.iglob(path)):
 265             with open(i) as f:
 266                 data = jsonutils.load(f)
 267             try:
 268                 HttpClient().post(url, {'dashboard': data})
 269             except Exception:
 270                 LOG.exception('Create dashboard %s failed', i)
 271                 raise
 272
 273     def _create_data_source(self, ip):
 274         url = 'http://admin:admin@{}:{}/api/datasources'.format(ip, 3000)
 275         influx_conf = utils.parse_ini_file(consts.CONF_FILE).get('dispatcher_influxdb', {})
 276
 277         data = {
 278             "name": "yardstick",
 279             "type": "influxdb",
 280             "access": "proxy",
 281             "url": influx_conf.get('target', ''),
 282             "password": influx_conf.get('password', ''),
 283             "user": influx_conf.get('username', ''),
 284             "database": "yardstick",
 285             "basicAuth": True,
 286             "basicAuthUser": "admin",
 287             "basicAuthPassword": "admin",
 288             "isDefault": False,
 289         }
 290         try:
 291             HttpClient().post(url, data)
 292         except Exception:
 293             LOG.exception('Create datasources failed')
 294             raise
 295
 296     def _create_grafana_container(self, client, name, port):
 297         ports = [3000]
 298         port_bindings = {3000: port}
 299         restart_policy = {"MaximumRetryCount": 0, "Name": "always"}
 300         host_config = client.create_host_config(port_bindings=port_bindings,
 301                                                 restart_policy=restart_policy)
 302
 303         LOG.info('Creating container')
 304         container = client.create_container(image='%s:%s' %
 305                                             (consts.GRAFANA_IMAGE,
 306                                              consts.GRAFANA_TAG),
 307                                             name=name,
 308                                             ports=ports,
 309                                             detach=True,
 310                                             tty=True,
 311                                             host_config=host_config)
 312         LOG.info('Starting container')
 313         client.start(container)
 314         return container
 315
 316
 317 class V2Container(ApiResource):
 318
 319     def get(self, container_id):
 320         try:
 321             uuid.UUID(container_id)
 322         except ValueError:
 323             return result_handler(consts.API_ERROR, 'invalid container id')
 324
 325         try:
 326             container = container_handler.get_by_uuid(container_id)
 327         except ValueError:
 328             return result_handler(consts.API_ERROR, 'no such container id')
 329
 330         name = container.name
 331         client = Client(base_url=consts.DOCKER_URL)
 332         info = client.inspect_container(name)
 333
 334         data = {
 335             'name': name,
 336             'status': info.get('State', {}).get('Status', 'error'),
 337             'time': info.get('Created'),
 338             'port': container.port
 339         }
 340
 341         return result_handler(consts.API_SUCCESS, {'container': data})
 342
 343     def delete(self, container_id):
 344         try:
 345             uuid.UUID(container_id)
 346         except ValueError:
 347             return result_handler(consts.API_ERROR, 'invalid container id')
 348
 349         try:
 350             container = container_handler.get_by_uuid(container_id)
 351         except ValueError:
 352             return result_handler(consts.API_ERROR, 'no such container id')
 353
 354         environment_id = container.environment_id
 355
 356         client = Client(base_url=consts.DOCKER_URL)
 357         LOG.info('delete container: %s', container.name)
 358         try:
 359             client.remove_container(container.name, force=True)
 360         except Exception:
 361             LOG.exception('delete container failed')
 362             return result_handler(consts.API_ERROR, 'delete container failed')
 363
 364         LOG.info('delete container in database')
 365         container_handler.delete_by_uuid(container_id)
 366
 367         LOG.info('update container in environment')
 368         environment = environment_handler.get_by_uuid(environment_id)
 369         container_info = jsonutils.loads(environment.container_id)
 370         key = next((k for k, v in container_info.items() if v == container_id))
 371         container_info.pop(key)
 372         environment_delete_data = {
 373             'container_id': jsonutils.dumps(container_info)
 374         }
 375         environment_handler.update_attr(environment_id, environment_delete_data)
 376
 377         return result_handler(consts.API_SUCCESS, {'container': container_id})