import subprocess
import sys
import tempfile
+import time
import multiprocessing
-log = logging.getLogger()
-ch = logging.StreamHandler(sys.stdout)
-if os.environ.get('DEBUG', False):
- log.setLevel(logging.DEBUG)
- ch.setLevel(logging.DEBUG)
-else:
- log.setLevel(logging.INFO)
- ch.setLevel(logging.INFO)
-formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
-ch.setFormatter(formatter)
-log.addHandler(ch)
+logger = None
+
+def get_logger():
+ global logger
+ if logger is None:
+ logger = logging.getLogger()
+ ch = logging.StreamHandler(sys.stdout)
+ if os.environ.get('DEBUG', False):
+ logger.setLevel(logging.DEBUG)
+ ch.setLevel(logging.DEBUG)
+ else:
+ logger.setLevel(logging.INFO)
+ ch.setLevel(logging.INFO)
+ formatter = logging.Formatter('%(asctime)s %(levelname)s: '
+ '%(process)s -- %(message)s')
+ ch.setFormatter(formatter)
+ logger.addHandler(ch)
+ return logger
+
# this is to match what we do in deployed-server
def short_hostname():
def pull_image(name):
log.info('Pulling image: %s' % name)
- subproc = subprocess.Popen(['/usr/bin/docker', 'pull', name],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- cmd_stdout, cmd_stderr = subproc.communicate()
+ retval = -1
+ count = 0
+ while retval != 0:
+ count += 1
+ subproc = subprocess.Popen(['/usr/bin/docker', 'pull', name],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+
+ cmd_stdout, cmd_stderr = subproc.communicate()
+ retval = subproc.returncode
+ if retval != 0:
+ time.sleep(3)
+ log.warning('docker pull failed: %s' % cmd_stderr)
+ log.warning('retrying pulling image: %s' % name)
+ if count >= 5:
+ log.error('Failed to pull image: %s' % name)
+ break
if cmd_stdout:
log.debug(cmd_stdout)
if cmd_stderr:
config_volume=None
for v in volumes:
if v.startswith(prefix):
- config_volume = os.path.relpath(
- v.split(":")[0], prefix).split("/")[0]
+ config_volume = os.path.dirname(v.split(":")[0])
break
return config_volume
-def get_config_hash(prefix, config_volume):
- hashfile = os.path.join(prefix, "%s.md5sum" % config_volume)
+def get_config_hash(config_volume):
+ hashfile = "%s.md5sum" % config_volume
+ log.debug("Looking for hashfile %s for config_volume %s" % (hashfile, config_volume))
hash_data = None
if os.path.isfile(hashfile):
+ log.debug("Got hashfile %s for config_volume %s" % (hashfile, config_volume))
with open(hashfile) as f:
hash_data = f.read().rstrip()
return hash_data
process_count = int(os.environ.get('PROCESS_COUNT',
multiprocessing.cpu_count()))
-
+log = get_logger()
log.info('Running docker-puppet')
config_file = os.environ.get('CONFIG', '/var/lib/docker-puppet/docker-puppet.json')
log.debug('CONFIG: %s' % config_file)
log.info('Service compilation completed.')
def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volumes)):
-
+ log = get_logger()
+ log.info('Started processing puppet configs')
log.debug('config_volume %s' % config_volume)
log.debug('puppet_tags %s' % puppet_tags)
log.debug('manifest %s' % manifest)
touch /tmp/the_origin_of_time
sync
- FACTER_hostname=$HOSTNAME FACTER_uuid=docker /usr/bin/puppet apply --verbose $TAGS /etc/config.pp
+ set +e
+ FACTER_hostname=$HOSTNAME FACTER_uuid=docker /usr/bin/puppet apply \
+ --detailed-exitcodes --color=false --logdest syslog --logdest console $TAGS /etc/config.pp
+ rc=$?
+ set -e
+ if [ $rc -ne 2 -a $rc -ne 0 ]; then
+ exit $rc
+ fi
# Disables archiving
if [ -z "$NO_ARCHIVE" ]; then
- archivedirs=("/etc" "/root" "/opt" "/var/lib/ironic/tftpboot" "/var/lib/ironic/httpboot" "/var/www" "/var/spool/cron")
+ archivedirs=("/etc" "/root" "/opt" "/var/lib/ironic/tftpboot" "/var/lib/ironic/httpboot" "/var/www" "/var/spool/cron" "/var/lib/nova/.ssh")
rsync_srcs=""
for d in "${archivedirs[@]}"; do
if [ -d "$d" ]; then
# Write a checksum of the config-data dir, this is used as a
# salt to trigger container restart when the config changes
tar -c -f - /var/lib/config-data/${NAME} --mtime='1970-01-01' | md5sum | awk '{print $1}' > /var/lib/config-data/${NAME}.md5sum
+ tar -c -f - /var/lib/config-data/puppet-generated/${NAME} --mtime='1970-01-01' | md5sum | awk '{print $1}' > /var/lib/config-data/puppet-generated/${NAME}.md5sum
fi
""")
dcmd = ['/usr/bin/docker', 'run',
'--user', 'root',
'--name', 'docker-puppet-%s' % config_volume,
+ '--health-cmd', '/bin/true',
'--env', 'PUPPET_TAGS=%s' % puppet_tags,
'--env', 'NAME=%s' % config_volume,
'--env', 'HOSTNAME=%s' % short_hostname(),
'--volume', '%s:/etc/config.pp:ro' % tmp_man.name,
'--volume', '/etc/puppet/:/tmp/puppet-etc/:ro',
'--volume', '/usr/share/openstack-puppet/modules/:/usr/share/openstack-puppet/modules/:ro',
- '--volume', '/var/lib/config-data/:/var/lib/config-data/:rw',
+ '--volume', '%s:/var/lib/config-data/:rw' % os.environ.get('CONFIG_VOLUME_PREFIX', '/var/lib/config-data'),
'--volume', 'tripleo_logs:/var/log/tripleo/',
+ # Syslog socket for puppet logs
+ '--volume', '/dev/log:/dev/log',
# OpenSSL trusted CA injection
'--volume', '/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro',
'--volume', '/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro',
subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env)
cmd_stdout, cmd_stderr = subproc.communicate()
- if subproc.returncode != 0:
+ # puppet with --detailed-exitcodes will return 0 for success and no changes
+ # and 2 for success and resource changes. Other numbers are failures
+ if subproc.returncode not in [0, 2]:
log.error('Failed running docker-puppet.py for %s' % config_volume)
if cmd_stdout:
log.error(cmd_stdout)
log.debug(cmd_stderr)
# only delete successful runs, for debugging
rm_container('docker-puppet-%s' % config_volume)
+
+ log.info('Finished processing puppet configs')
return subproc.returncode
# Holds all the information for each process to consume.
config_volumes = [pm[0] for pm in process_map]
success = True
for returncode, config_volume in zip(returncodes, config_volumes):
- if returncode != 0:
+ if returncode not in [0, 2]:
log.error('ERROR configuring %s' % config_volume)
success = False
for k, v in infile_data.iteritems():
config_volume = match_config_volume(config_volume_prefix, v)
if config_volume:
- config_hash = get_config_hash(config_volume_prefix, config_volume)
+ config_hash = get_config_hash(config_volume)
if config_hash:
env = v.get('environment', [])
env.append("TRIPLEO_CONFIG_HASH=%s" % config_hash)
outfile = os.path.join(os.path.dirname(infile), "hashed-" + os.path.basename(infile))
with open(outfile, 'w') as out_f:
+ os.chmod(out_f.name, 0600)
json.dump(infile_data, out_f)
if not success: