X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=deploy%2Fcloud%2Fdeployment.py;h=f8e1617f8e80f5a3335c56fd2048320c8d7a5bd1;hb=b86d8e65f9a8a3e52a7f06f8ac9d32963c9df6d4;hp=12c1f897861f14208ffaf6de0c8da6ea0baac8fd;hpb=24a95306d2564b272b5320e9149d9aea70b4061c;p=fuel.git

diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py
index 12c1f8978..f8e1617f8 100644
--- a/deploy/cloud/deployment.py
+++ b/deploy/cloud/deployment.py
@@ -7,7 +7,6 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ###############################################################################
 
-
 import time
 import re
 
@@ -16,13 +15,15 @@ from common import (
     E,
     exec_cmd,
     run_proc,
+    run_proc_wait_terminated,
+    run_proc_kill,
     parse,
     err,
     log,
     delete,
 )
 
-SEARCH_TEXT = 'Puppet (err)'
+SEARCH_TEXT = '(err)'
 LOG_FILE = '/var/log/puppet.log'
 GREP_LINES_OF_LEADING_CONTEXT = 100
 GREP_LINES_OF_TRAILING_CONTEXT = 100
@@ -30,16 +31,19 @@ LIST_OF_CHAR_TO_BE_ESCAPED = ['[', ']', '"']
 
 class Deployment(object):
 
+
     def __init__(self, dea, yaml_config_dir, env_id, node_id_roles_dict,
-                 no_health_check):
+                 no_health_check, deploy_timeout):
         self.dea = dea
         self.yaml_config_dir = yaml_config_dir
         self.env_id = env_id
         self.node_id_roles_dict = node_id_roles_dict
         self.no_health_check = no_health_check
+        self.deploy_timeout = deploy_timeout
         self.pattern = re.compile(
             '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')
 
+
     def collect_error_logs(self):
         for node_id, roles_blade in self.node_id_roles_dict.iteritems():
             log_list = []
@@ -95,17 +99,17 @@ class Deployment(object):
                 for log_msg in log_list:
                     print(log_msg + '\n')
 
+
     def run_deploy(self):
-        WAIT_LOOP = 180
         SLEEP_TIME = 60
         LOG_FILE = 'cloud.log'
 
         log('Starting deployment of environment %s' % self.env_id)
-        run_proc('fuel --env %s deploy-changes | strings | tee %s'
-                 % (self.env_id, LOG_FILE))
+        deploy_proc = run_proc('fuel --env %s deploy-changes | strings > %s'
+                               % (self.env_id, LOG_FILE))
 
         ready = False
-        for i in range(WAIT_LOOP):
+        for i in range(int(self.deploy_timeout)):
             env = parse(exec_cmd('fuel env --env %s' % self.env_id))
             log('Environment status: %s' % env[0][E['status']])
             r, _ = exec_cmd('tail -2 %s | head -1' % LOG_FILE, False)
@@ -119,6 +123,14 @@ class Deployment(object):
                 break
             else:
                 time.sleep(SLEEP_TIME)
+
+        if (env[0][E['status']] <> 'operational'
+            and env[0][E['status']] <> 'error'
+            and env[0][E['status']] <> 'stopped'):
+            err('Deployment timed out, environment %s is not operational, snapshot will not be performed'
+                % self.env_id, self.collect_logs)
+
+        run_proc_wait_terminated(deploy_proc)
         delete(LOG_FILE)
 
         if ready:
@@ -126,31 +138,49 @@ class Deployment(object):
         else:
             self.collect_error_logs()
             err('Deployment failed, environment %s is not operational'
-                % self.env_id)
+                % self.env_id, self.collect_logs)
+
+
+    def collect_logs(self):
+        log('Cleaning out any previous deployment logs')
+        exec_cmd('rm -f /var/log/remote/fuel-snapshot-*', False)
+        exec_cmd('rm -f /root/deploy-*', False)
+        log('Generating Fuel deploy snap-shot')
+        if exec_cmd('fuel snapshot < /dev/null &> snapshot.log', False)[1] <> 0:
+            log('Could not create a Fuel snapshot')
+        else:
+            exec_cmd('mv /root/fuel-snapshot* /var/log/remote/', False)
+
+        log('Collecting all Fuel Snapshot & deploy log files')
+        r, _ = exec_cmd('tar -czhf /root/deploy-%s.log.tar.gz /var/log/remote' % time.strftime("%Y%m%d-%H%M%S"), False)
+        log(r)
+
 
     def verify_node_status(self):
         node_list = parse(exec_cmd('fuel node list'))
         failed_nodes = []
         for node in node_list:
-            if node[N['status']] != 'ready':
+            if node[N['status']] != 'ready' and node[N['cluster']] != 'None':
                 failed_nodes.append((node[N['id']], node[N['status']]))
 
         if failed_nodes:
             summary = ''
             for node, status in failed_nodes:
                 summary += '[node %s, status %s]\n' % (node, status)
-            err('Deployment failed: %s' % summary)
+            err('Deployment failed: %s' % summary, self.collect_logs)
+
 
     def health_check(self):
         log('Now running sanity and smoke health checks')
-        r = exec_cmd('fuel health --env %s --check sanity,smoke --force'
-                     % self.env_id)
+        r = exec_cmd('fuel health --env %s --check sanity,smoke --force' % self.env_id)
         log(r)
         if 'failure' in r:
-            err('Healthcheck failed!')
+            err('Healthcheck failed!', self.collect_logs)
+
 
     def deploy(self):
         self.run_deploy()
         self.verify_node_status()
         if not self.no_health_check:
             self.health_check()
+        self.collect_logs()