deploy/cloud/deployment.py

   1 ###############################################################################
   2 # Copyright (c) 2015 Ericsson AB and others.
   3 # szilard.cserey@ericsson.com
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ###############################################################################
   9
  10
  11 import time
  12 import re
  13
  14 from common import (
  15     N,
  16     E,
  17     exec_cmd,
  18     run_proc,
  19     parse,
  20     err,
  21     log,
  22     delete,
  23 )
  24
  25 SEARCH_TEXT = '(err)'
  26 LOG_FILE = '/var/log/puppet.log'
  27 GREP_LINES_OF_LEADING_CONTEXT = 100
  28 GREP_LINES_OF_TRAILING_CONTEXT = 100
  29 LIST_OF_CHAR_TO_BE_ESCAPED = ['[', ']', '"']
  30
  31 class Deployment(object):
  32
  33     def __init__(self, dea, yaml_config_dir, env_id, node_id_roles_dict,
  34                  no_health_check):
  35         self.dea = dea
  36         self.yaml_config_dir = yaml_config_dir
  37         self.env_id = env_id
  38         self.node_id_roles_dict = node_id_roles_dict
  39         self.no_health_check = no_health_check
  40         self.pattern = re.compile(
  41             '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')
  42
  43     def collect_error_logs(self):
  44         for node_id, roles_blade in self.node_id_roles_dict.iteritems():
  45             log_list = []
  46             cmd = ('ssh -q node-%s grep \'"%s"\' %s'
  47                    % (node_id, SEARCH_TEXT, LOG_FILE))
  48             results, _ = exec_cmd(cmd, False)
  49             for result in results.splitlines():
  50                 log_msg = ''
  51
  52                 sub_cmd = '"%s" %s' % (result, LOG_FILE)
  53                 for c in LIST_OF_CHAR_TO_BE_ESCAPED:
  54                     sub_cmd = sub_cmd.replace(c, '\%s' % c)
  55                 grep_cmd = ('grep -B%s %s'
  56                             % (GREP_LINES_OF_LEADING_CONTEXT, sub_cmd))
  57                 cmd = ('ssh -q node-%s "%s"' % (node_id, grep_cmd))
  58
  59                 details, _ = exec_cmd(cmd, False)
  60                 details_list = details.splitlines()
  61
  62                 found_prev_log = False
  63                 for i in range(len(details_list) - 2, -1, -1):
  64                     if self.pattern.match(details_list[i]):
  65                         found_prev_log = True
  66                         break
  67                 if found_prev_log:
  68                     log_msg += '\n'.join(details_list[i:-1]) + '\n'
  69
  70                 grep_cmd = ('grep -A%s %s'
  71                             % (GREP_LINES_OF_TRAILING_CONTEXT, sub_cmd))
  72                 cmd = ('ssh -q node-%s "%s"' % (node_id, grep_cmd))
  73
  74                 details, _ = exec_cmd(cmd, False)
  75                 details_list = details.splitlines()
  76
  77                 found_next_log = False
  78                 for i in range(1, len(details_list)):
  79                     if self.pattern.match(details_list[i]):
  80                         found_next_log = True
  81                         break
  82                 if found_next_log:
  83                     log_msg += '\n'.join(details_list[:i])
  84                 else:
  85                     log_msg += details
  86
  87                 if log_msg:
  88                    log_list.append(log_msg)
  89
  90             if log_list:
  91                 role = ('controller' if 'controller' in roles_blade[0]
  92                         else 'compute host')
  93                 log('_' * 40 + 'Errors in node-%s %s' % (node_id, role)
  94                     + '_' * 40)
  95                 for log_msg in log_list:
  96                     print(log_msg + '\n')
  97
  98     def run_deploy(self):
  99         WAIT_LOOP = 180
 100         SLEEP_TIME = 60
 101         LOG_FILE = 'cloud.log'
 102
 103         log('Starting deployment of environment %s' % self.env_id)
 104         run_proc('fuel --env %s deploy-changes | strings | tee %s'
 105                  % (self.env_id, LOG_FILE))
 106
 107         ready = False
 108         for i in range(WAIT_LOOP):
 109             env = parse(exec_cmd('fuel env --env %s' % self.env_id))
 110             log('Environment status: %s' % env[0][E['status']])
 111             r, _ = exec_cmd('tail -2 %s | head -1' % LOG_FILE, False)
 112             if r:
 113                 log(r)
 114             if env[0][E['status']] == 'operational':
 115                 ready = True
 116                 break
 117             elif (env[0][E['status']] == 'error'
 118                   or env[0][E['status']] == 'stopped'):
 119                 break
 120             else:
 121                 time.sleep(SLEEP_TIME)
 122         delete(LOG_FILE)
 123
 124         if ready:
 125             log('Environment %s successfully deployed' % self.env_id)
 126         else:
 127             self.collect_error_logs()
 128             err('Deployment failed, environment %s is not operational'
 129                 % self.env_id)
 130
 131     def verify_node_status(self):
 132         node_list = parse(exec_cmd('fuel node list'))
 133         failed_nodes = []
 134         for node in node_list:
 135             if node[N['status']] != 'ready' and node[N['cluster']] != 'None':
 136                 failed_nodes.append((node[N['id']], node[N['status']]))
 137
 138         if failed_nodes:
 139             summary = ''
 140             for node, status in failed_nodes:
 141                 summary += '[node %s, status %s]\n' % (node, status)
 142             err('Deployment failed: %s' % summary)
 143
 144     def health_check(self):
 145         log('Now running sanity and smoke health checks')
 146         r = exec_cmd('fuel health --env %s --check sanity,smoke --force'
 147                      % self.env_id)
 148         log(r)
 149         if 'failure' in r:
 150             err('Healthcheck failed!')
 151
 152     def deploy(self):
 153         self.run_deploy()
 154         self.verify_node_status()
 155         if not self.no_health_check:
 156             self.health_check()