3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
15 # Shell script tool to run puppet inside of the given docker container image.
16 # Uses the config file at /var/lib/docker-puppet/docker-puppet.json as a source for a JSON
17 # array of [config_volume, puppet_tags, manifest, config_image, [volumes]] settings
18 # that can be used to generate config files or run ad-hoc puppet modules
19 # inside of a container.
26 import multiprocessing
29 # this is to match what we do in deployed-server
31 subproc = subprocess.Popen(['hostname', '-s'],
32 stdout=subprocess.PIPE,
33 stderr=subprocess.PIPE)
34 cmd_stdout, cmd_stderr = subproc.communicate()
35 return cmd_stdout.rstrip()
39 print('Pulling image: %s' % name)
40 subproc = subprocess.Popen(['/usr/bin/docker', 'pull', name],
41 stdout=subprocess.PIPE,
42 stderr=subprocess.PIPE)
43 cmd_stdout, cmd_stderr = subproc.communicate()
48 def rm_container(name):
49 if os.environ.get('SHOW_DIFF', None):
50 print('Diffing container: %s' % name)
51 subproc = subprocess.Popen(['/usr/bin/docker', 'diff', name],
52 stdout=subprocess.PIPE,
53 stderr=subprocess.PIPE)
54 cmd_stdout, cmd_stderr = subproc.communicate()
58 print('Removing container: %s' % name)
59 subproc = subprocess.Popen(['/usr/bin/docker', 'rm', name],
60 stdout=subprocess.PIPE,
61 stderr=subprocess.PIPE)
62 cmd_stdout, cmd_stderr = subproc.communicate()
65 cmd_stderr != 'Error response from daemon: ' \
66 'No such container: {}\n'.format(name):
69 process_count = int(os.environ.get('PROCESS_COUNT',
70 multiprocessing.cpu_count()))
72 config_file = os.environ.get('CONFIG', '/var/lib/docker-puppet/docker-puppet.json')
73 print('docker-puppet')
74 print('CONFIG: %s' % config_file)
75 with open(config_file) as f:
76 json_data = json.load(f)
78 # To save time we support configuring 'shared' services at the same
79 # time. For example configuring all of the heat services
80 # in a single container pass makes sense and will save some time.
81 # To support this we merge shared settings together here.
83 # We key off of config_volume as this should be the same for a
84 # given group of services. We are also now specifying the container
85 # in which the services should be configured. This should match
86 # in all instances where the volume name is also the same.
90 for service in (json_data or []):
93 if isinstance(service, dict):
95 service.get('config_volume'),
96 service.get('puppet_tags'),
97 service.get('step_config'),
98 service.get('config_image'),
99 service.get('volumes', []),
102 config_volume = service[0] or ''
103 puppet_tags = service[1] or ''
104 manifest = service[2] or ''
105 config_image = service[3] or ''
106 volumes = service[4] if len(service) > 4 else []
108 if not manifest or not config_image:
112 print('config_volume %s' % config_volume)
113 print('puppet_tags %s' % puppet_tags)
114 print('manifest %s' % manifest)
115 print('config_image %s' % config_image)
116 print('volumes %s' % volumes)
117 # We key off of config volume for all configs.
118 if config_volume in configs:
119 # Append puppet tags and manifest.
120 print("Existing service, appending puppet tags and manifest\n")
122 configs[config_volume][1] = '%s,%s' % (configs[config_volume][1],
125 configs[config_volume][2] = '%s\n%s' % (configs[config_volume][2],
127 if configs[config_volume][3] != config_image:
128 print("WARNING: Config containers do not match even though"
129 " shared volumes are the same!\n")
131 print("Adding new service\n")
132 configs[config_volume] = service
134 print('Service compilation completed.\n')
136 def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volumes)):
139 print('config_volume %s' % config_volume)
140 print('puppet_tags %s' % puppet_tags)
141 print('manifest %s' % manifest)
142 print('config_image %s' % config_image)
143 print('volumes %s' % volumes)
144 hostname = short_hostname()
145 sh_script = '/var/lib/docker-puppet/docker-puppet-%s.sh' % config_volume
147 with open(sh_script, 'w') as script_file:
148 os.chmod(script_file.name, 0755)
149 script_file.write("""#!/bin/bash
152 cp -a /tmp/puppet-etc/* /etc/puppet
153 rm -Rf /etc/puppet/ssl # not in use and causes permission errors
154 echo '{"step": %(step)s}' > /etc/puppet/hieradata/docker.json
156 if [ -n "%(puppet_tags)s" ]; then
157 TAGS='--tags "%(puppet_tags)s"'
159 FACTER_hostname=%(hostname)s FACTER_uuid=docker /usr/bin/puppet apply --verbose $TAGS /etc/config.pp
162 if [ -z "%(no_archive)s" ]; then
163 rm -Rf /var/lib/config-data/%(name)s
165 # copying etc should be enough for most services
166 mkdir -p /var/lib/config-data/%(name)s/etc
167 cp -a /etc/* /var/lib/config-data/%(name)s/etc/
169 if [ -d /root/ ]; then
170 cp -a /root/ /var/lib/config-data/%(name)s/root/
172 if [ -d /var/lib/ironic/tftpboot/ ]; then
173 mkdir -p /var/lib/config-data/%(name)s/var/lib/ironic/
174 cp -a /var/lib/ironic/tftpboot/ /var/lib/config-data/%(name)s/var/lib/ironic/tftpboot/
176 if [ -d /var/lib/ironic/httpboot/ ]; then
177 mkdir -p /var/lib/config-data/%(name)s/var/lib/ironic/
178 cp -a /var/lib/ironic/httpboot/ /var/lib/config-data/%(name)s/var/lib/ironic/httpboot/
181 # apache services may files placed in /var/www/
182 if [ -d /var/www/ ]; then
183 mkdir -p /var/lib/config-data/%(name)s/var/www
184 cp -a /var/www/* /var/lib/config-data/%(name)s/var/www/
187 """ % {'puppet_tags': puppet_tags, 'name': config_volume,
188 'hostname': hostname,
189 'no_archive': os.environ.get('NO_ARCHIVE', ''),
190 'step': os.environ.get('STEP', '6')})
192 with tempfile.NamedTemporaryFile() as tmp_man:
193 with open(tmp_man.name, 'w') as man_file:
194 man_file.write('include ::tripleo::packages\n')
195 man_file.write(manifest)
197 rm_container('docker-puppet-%s' % config_volume)
198 pull_image(config_image)
200 dcmd = ['/usr/bin/docker', 'run',
202 '--name', 'docker-puppet-%s' % config_volume,
203 '--volume', '%s:/etc/config.pp:ro' % tmp_man.name,
204 '--volume', '/etc/puppet/:/tmp/puppet-etc/:ro',
205 '--volume', '/usr/share/openstack-puppet/modules/:/usr/share/openstack-puppet/modules/:ro',
206 '--volume', '/var/lib/config-data/:/var/lib/config-data/:rw',
207 '--volume', 'tripleo_logs:/var/log/tripleo/',
208 # OpenSSL trusted CA injection
209 '--volume', '/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro',
210 '--volume', '/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro',
211 '--volume', '/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro',
212 '--volume', '/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro',
214 '--volume', '%s:%s:rw' % (sh_script, sh_script) ]
216 for volume in volumes:
218 dcmd.extend(['--volume', volume])
220 dcmd.extend(['--entrypoint', sh_script])
223 # NOTE(flaper87): Always copy the DOCKER_* environment variables as
224 # they contain the access data for the docker daemon.
225 for k in filter(lambda k: k.startswith('DOCKER'), os.environ.keys()):
226 env[k] = os.environ.get(k)
228 if os.environ.get('NET_HOST', 'false') == 'true':
229 print('NET_HOST enabled')
230 dcmd.extend(['--net', 'host', '--volume',
231 '/etc/hosts:/etc/hosts:ro'])
232 dcmd.append(config_image)
234 subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE,
235 stderr=subprocess.PIPE, env=env)
236 cmd_stdout, cmd_stderr = subproc.communicate()
239 if subproc.returncode != 0:
240 print('Failed running docker-puppet.py for %s' % config_volume)
241 rm_container('docker-puppet-%s' % config_volume)
242 return subproc.returncode
244 # Holds all the information for each process to consume.
245 # Instead of starting them all linearly we run them using a process
246 # pool. This creates a list of arguments for the above function
250 for config_volume in configs:
252 service = configs[config_volume]
253 puppet_tags = service[1] or ''
254 manifest = service[2] or ''
255 config_image = service[3] or ''
256 volumes = service[4] if len(service) > 4 else []
259 puppet_tags = "file,file_line,concat,augeas,%s" % puppet_tags
261 puppet_tags = "file,file_line,concat,augeas"
263 process_map.append([config_volume, puppet_tags, manifest, config_image, volumes])
265 for p in process_map:
268 # Fire off processes to perform each configuration. Defaults
269 # to the number of CPUs on the system.
270 p = multiprocessing.Pool(process_count)
271 returncodes = list(p.map(mp_puppet_config, process_map))
272 config_volumes = [pm[0] for pm in process_map]
274 for returncode, config_volume in zip(returncodes, config_volumes):
276 print('ERROR configuring %s' % config_volume)