Increase ssh timeout and add retry for Ansible

[apex.git] / apex / common / utils.py

diff --git a/apex/common/utils.py b/apex/common/utils.py
index 848f264..f418d42 100644 (file)
--- a/apex/common/utils.py
+++ b/apex/common/utils.py
@@ -76,7 +76,8 @@ def run_ansible(ansible_vars, playbook, host='localhost', user='root',
     else:
         conn_type = 'smart'
     ansible_command = ['ansible-playbook', '--become', '-i', inv_host,
-                       '-u', user, '-c', conn_type, playbook, '-vvv']
+                       '-u', user, '-c', conn_type, '-T', '30',
+                       playbook, '-vv']
     if dry_run:
         ansible_command.append('--check')
 
@@ -95,13 +96,36 @@ def run_ansible(ansible_vars, playbook, host='localhost', user='root',
             with open(ansible_tmp, 'w') as fh:
                 fh.write("ANSIBLE_HOST_KEY_CHECKING=FALSE {}".format(
                     ' '.join(ansible_command)))
-    try:
-        my_env = os.environ.copy()
-        my_env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
-        logging.info("Executing playbook...this may take some time")
-        logging.debug(subprocess.check_output(ansible_command, env=my_env,
-                      stderr=subprocess.STDOUT).decode('utf-8'))
-    except subprocess.CalledProcessError as e:
-        logging.error("Error executing ansible: {}".format(
-            pprint.pformat(e.output.decode('utf-8'))))
-        raise
+
+    my_env = os.environ.copy()
+    my_env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
+    logging.info("Executing playbook...this may take some time")
+    p = subprocess.Popen(ansible_command,
+                         stdin=subprocess.PIPE,
+                         stdout=subprocess.PIPE,
+                         bufsize=1,
+                         env=my_env,
+                         universal_newlines=True)
+    # read first line
+    x = p.stdout.readline()
+    # initialize task
+    task = ''
+    while x:
+        # append lines to task
+        task += x
+        # log the line and read another
+        x = p.stdout.readline()
+        # deliver the task to info when we get a blank line
+        if not x.strip():
+            task += x
+            logging.info(task.replace('\\n', '\n'))
+            task = ''
+            x = p.stdout.readline()
+    # clean up and get return code
+    p.stdout.close()
+    rc = p.wait()
+    if rc:
+        # raise errors
+        e = "Ansible playbook failed. See Ansible logs for details."
+        logging.error(e)
+        raise Exception(e)