Add workaround for nova vif plugging timeout error

[apex-tripleo-heat-templates.git] / docker / docker-puppet.py

diff --git a/docker/docker-puppet.py b/docker/docker-puppet.py
index 4659cf5..d12e055 100755 (executable)
--- a/docker/docker-puppet.py
+++ b/docker/docker-puppet.py
@@ -26,6 +26,7 @@ import sys
 import subprocess
 import sys
 import tempfile
+import time
 import multiprocessing
 
 logger = None
@@ -59,10 +60,23 @@ def short_hostname():
 
 def pull_image(name):
     log.info('Pulling image: %s' % name)
-    subproc = subprocess.Popen(['/usr/bin/docker', 'pull', name],
-                               stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE)
-    cmd_stdout, cmd_stderr = subproc.communicate()
+    retval = -1
+    count = 0
+    while retval != 0:
+        count += 1
+        subproc = subprocess.Popen(['/usr/bin/docker', 'pull', name],
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE)
+
+        cmd_stdout, cmd_stderr = subproc.communicate()
+        retval = subproc.returncode
+        if retval != 0:
+            time.sleep(3)
+            log.warning('docker pull failed: %s' % cmd_stderr)
+            log.warning('retrying pulling image: %s' % name)
+        if count >= 5:
+            log.error('Failed to pull image: %s' % name)
+            break
     if cmd_stdout:
         log.debug(cmd_stdout)
     if cmd_stderr:
@@ -76,16 +90,17 @@ def match_config_volume(prefix, config):
     config_volume=None
     for v in volumes:
         if v.startswith(prefix):
-            config_volume =  os.path.relpath(
-                v.split(":")[0], prefix).split("/")[0]
+            config_volume = os.path.dirname(v.split(":")[0])
             break
     return config_volume
 
 
-def get_config_hash(prefix, config_volume):
-    hashfile = os.path.join(prefix, "%s.md5sum" % config_volume)
+def get_config_hash(config_volume):
+    hashfile = "%s.md5sum" % config_volume
+    log.debug("Looking for hashfile %s for config_volume %s" % (hashfile, config_volume))
     hash_data = None
     if os.path.isfile(hashfile):
+        log.debug("Got hashfile %s for config_volume %s" % (hashfile, config_volume))
         with open(hashfile) as f:
             hash_data = f.read().rstrip()
     return hash_data
@@ -210,8 +225,14 @@ def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volume
         touch /tmp/the_origin_of_time
         sync
 
+        set +e
         FACTER_hostname=$HOSTNAME FACTER_uuid=docker /usr/bin/puppet apply \
-        --color=false --logdest syslog --logdest console $TAGS /etc/config.pp
+        --detailed-exitcodes --color=false --logdest syslog --logdest console $TAGS /etc/config.pp
+        rc=$?
+        set -e
+        if [ $rc -ne 2 -a $rc -ne 0 ]; then
+            exit $rc
+        fi
 
         # Disables archiving
         if [ -z "$NO_ARCHIVE" ]; then
@@ -234,6 +255,7 @@ def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volume
             # Write a checksum of the config-data dir, this is used as a
             # salt to trigger container restart when the config changes
             tar -c -f - /var/lib/config-data/${NAME} --mtime='1970-01-01' | md5sum | awk '{print $1}' > /var/lib/config-data/${NAME}.md5sum
+            tar -c -f - /var/lib/config-data/puppet-generated/${NAME} --mtime='1970-01-01' | md5sum | awk '{print $1}' > /var/lib/config-data/puppet-generated/${NAME}.md5sum
         fi
         """)
 
@@ -257,7 +279,7 @@ def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volume
                 '--volume', '%s:/etc/config.pp:ro' % tmp_man.name,
                 '--volume', '/etc/puppet/:/tmp/puppet-etc/:ro',
                 '--volume', '/usr/share/openstack-puppet/modules/:/usr/share/openstack-puppet/modules/:ro',
-                '--volume', '/var/lib/config-data/:/var/lib/config-data/:rw',
+                '--volume', '%s:/var/lib/config-data/:rw' % os.environ.get('CONFIG_VOLUME_PREFIX', '/var/lib/config-data'),
                 '--volume', 'tripleo_logs:/var/log/tripleo/',
                 # Syslog socket for puppet logs
                 '--volume', '/dev/log:/dev/log',
@@ -291,7 +313,9 @@ def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volume
         subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE, env=env)
         cmd_stdout, cmd_stderr = subproc.communicate()
-        if subproc.returncode != 0:
+        # puppet with --detailed-exitcodes will return 0 for success and no changes
+        # and 2 for success and resource changes. Other numbers are failures
+        if subproc.returncode not in [0, 2]:
             log.error('Failed running docker-puppet.py for %s' % config_volume)
             if cmd_stdout:
                 log.error(cmd_stdout)
@@ -339,7 +363,7 @@ returncodes = list(p.map(mp_puppet_config, process_map))
 config_volumes = [pm[0] for pm in process_map]
 success = True
 for returncode, config_volume in zip(returncodes, config_volumes):
-    if returncode != 0:
+    if returncode not in [0, 2]:
         log.error('ERROR configuring %s' % config_volume)
         success = False
 
@@ -357,7 +381,7 @@ for infile in infiles:
     for k, v in infile_data.iteritems():
         config_volume = match_config_volume(config_volume_prefix, v)
         if config_volume:
-            config_hash = get_config_hash(config_volume_prefix, config_volume)
+            config_hash = get_config_hash(config_volume)
             if config_hash:
                 env = v.get('environment', [])
                 env.append("TRIPLEO_CONFIG_HASH=%s" % config_hash)
@@ -366,6 +390,7 @@ for infile in infiles:
 
     outfile = os.path.join(os.path.dirname(infile), "hashed-" + os.path.basename(infile))
     with open(outfile, 'w') as out_f:
+        os.chmod(out_f.name, 0600)
         json.dump(infile_data, out_f)
 
 if not success: