1 ###############################################################################
2 # Copyright (c) 2015 Ericsson AB and others.
3 # szilard.cserey@ericsson.com
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ###############################################################################
18 run_proc_wait_terminated,
27 LOG_FILE = '/var/log/puppet.log'
28 GREP_LINES_OF_LEADING_CONTEXT = 100
29 GREP_LINES_OF_TRAILING_CONTEXT = 100
30 LIST_OF_CHAR_TO_BE_ESCAPED = ['[', ']', '"']
32 class Deployment(object):
35 def __init__(self, dea, yaml_config_dir, env_id, node_id_roles_dict,
36 no_health_check, deploy_timeout):
38 self.yaml_config_dir = yaml_config_dir
40 self.node_id_roles_dict = node_id_roles_dict
41 self.no_health_check = no_health_check
42 self.deploy_timeout = deploy_timeout
43 self.pattern = re.compile(
44 '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')
47 def collect_error_logs(self):
48 for node_id, roles_blade in self.node_id_roles_dict.iteritems():
50 cmd = ('ssh -q node-%s grep \'"%s"\' %s'
51 % (node_id, SEARCH_TEXT, LOG_FILE))
52 results, _ = exec_cmd(cmd, False)
53 for result in results.splitlines():
56 sub_cmd = '"%s" %s' % (result, LOG_FILE)
57 for c in LIST_OF_CHAR_TO_BE_ESCAPED:
58 sub_cmd = sub_cmd.replace(c, '\%s' % c)
59 grep_cmd = ('grep -B%s %s'
60 % (GREP_LINES_OF_LEADING_CONTEXT, sub_cmd))
61 cmd = ('ssh -q node-%s "%s"' % (node_id, grep_cmd))
63 details, _ = exec_cmd(cmd, False)
64 details_list = details.splitlines()
66 found_prev_log = False
67 for i in range(len(details_list) - 2, -1, -1):
68 if self.pattern.match(details_list[i]):
72 log_msg += '\n'.join(details_list[i:-1]) + '\n'
74 grep_cmd = ('grep -A%s %s'
75 % (GREP_LINES_OF_TRAILING_CONTEXT, sub_cmd))
76 cmd = ('ssh -q node-%s "%s"' % (node_id, grep_cmd))
78 details, _ = exec_cmd(cmd, False)
79 details_list = details.splitlines()
81 found_next_log = False
82 for i in range(1, len(details_list)):
83 if self.pattern.match(details_list[i]):
87 log_msg += '\n'.join(details_list[:i])
92 log_list.append(log_msg)
95 role = ('controller' if 'controller' in roles_blade[0]
97 log('_' * 40 + 'Errors in node-%s %s' % (node_id, role)
99 for log_msg in log_list:
100 print(log_msg + '\n')
103 def run_deploy(self):
105 LOG_FILE = 'cloud.log'
107 log('Starting deployment of environment %s' % self.env_id)
108 deploy_proc = run_proc('fuel --env %s deploy-changes | strings > %s'
109 % (self.env_id, LOG_FILE))
112 for i in range(int(self.deploy_timeout)):
113 env = parse(exec_cmd('fuel env --env %s' % self.env_id))
114 log('Environment status: %s' % env[0][E['status']])
115 r, _ = exec_cmd('tail -2 %s | head -1' % LOG_FILE, False)
118 if env[0][E['status']] == 'operational':
121 elif (env[0][E['status']] == 'error'
122 or env[0][E['status']] == 'stopped'):
125 time.sleep(SLEEP_TIME)
127 if (env[0][E['status']] <> 'operational'
128 and env[0][E['status']] <> 'error'
129 and env[0][E['status']] <> 'stopped'):
130 err('Deployment timed out, environment %s is not operational, snapshot will not be performed'
131 % self.env_id, self.collect_logs)
133 run_proc_wait_terminated(deploy_proc)
137 log('Environment %s successfully deployed' % self.env_id)
139 self.collect_error_logs()
140 err('Deployment failed, environment %s is not operational'
141 % self.env_id, self.collect_logs)
144 def collect_logs(self):
145 log('Cleaning out any previous deployment logs')
146 exec_cmd('rm -f /var/log/remote/fuel-snapshot-*', False)
147 exec_cmd('rm -f /root/deploy-*', False)
148 log('Generating Fuel deploy snap-shot')
149 if exec_cmd('fuel snapshot < /dev/null &> snapshot.log', False)[1] <> 0:
150 log('Could not create a Fuel snapshot')
152 exec_cmd('mv /root/fuel-snapshot* /var/log/remote/', False)
154 log('Collecting all Fuel Snapshot & deploy log files')
155 r, _ = exec_cmd('tar -czhf /root/deploy-%s.log.tar.gz /var/log/remote' % time.strftime("%Y%m%d-%H%M%S"), False)
159 def verify_node_status(self):
160 node_list = parse(exec_cmd('fuel --env %s node' % self.env_id))
162 for node in node_list:
163 if node[N['status']] != 'ready':
164 failed_nodes.append((node[N['id']], node[N['status']]))
168 for node, status in failed_nodes:
169 summary += '[node %s, status %s]\n' % (node, status)
170 err('Deployment failed: %s' % summary, self.collect_logs)
173 def health_check(self):
174 log('Now running sanity and smoke health checks')
175 r = exec_cmd('fuel health --env %s --check sanity,smoke --force' % self.env_id)
178 err('Healthcheck failed!', self.collect_logs)
183 self.verify_node_status()
184 if not self.no_health_check: