1 ##############################################################################
2 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
10 # Common building utilities for undercloud and overcloud
20 import apex.builders.overcloud_builder as oc_builder
21 from apex import build_utils
22 from apex.builders import exceptions as exc
23 from apex.common import constants as con
24 from apex.common import utils
25 from apex.virtual import utils as virt_utils
28 def project_to_path(project):
30 Translates project to absolute file path to use in patching
31 :param project: name of project
34 if project.startswith('openstack/'):
35 project = os.path.basename(project)
36 if 'puppet' in project:
37 return "/etc/puppet/modules/{}".format(project.replace('puppet-', ''))
38 elif 'tripleo-heat-templates' in project:
39 return "/usr/share/openstack-tripleo-heat-templates"
41 # assume python. python patches will apply to a project name subdir.
42 # For example, python-tripleoclient patch will apply to the
43 # tripleoclient directory, which is the directory extracted during
44 # python install into the PYTHONPATH. Therefore we need to just be
45 # in the PYTHONPATH directory to apply a patch
46 return "/usr/lib/python2.7/site-packages/"
49 def project_to_docker_image(project):
51 Translates OpenStack project to OOO services that are containerized
52 :param project: name of OpenStack project
53 :return: List of OOO docker service names
55 # Fetch all docker containers in docker hub with tripleo and filter
58 hub_output = utils.open_webpage(
59 urllib.parse.urljoin(con.DOCKERHUB_OOO, '?page_size=1024'), timeout=10)
61 results = json.loads(hub_output.decode())['results']
62 except Exception as e:
63 logging.error("Unable to parse docker hub output for"
64 "tripleoupstream repository")
65 logging.debug("HTTP response from dockerhub:\n{}".format(hub_output))
66 raise exc.ApexCommonBuilderException(
67 "Failed to parse docker image info from Docker Hub: {}".format(e))
68 logging.debug("Docker Hub tripleoupstream entities found: {}".format(
70 docker_images = list()
71 for result in results:
72 if result['name'].startswith("centos-binary-{}".format(project)):
73 # add as docker image shortname (just service name)
74 docker_images.append(result['name'].replace('centos-binary-', ''))
79 def is_patch_promoted(change, branch, docker_image=None):
81 Checks to see if a patch that is in merged exists in either the docker
82 container or the promoted tripleo images
83 :param change: gerrit change json output
84 :param branch: branch to use when polling artifacts (does not include
86 :param docker_image: container this applies to if (defaults to None)
87 :return: True if the patch exists in a promoted artifact upstream
89 assert isinstance(change, dict)
90 assert 'status' in change
92 # if not merged we already know this is not closed/abandoned, so we know
93 # this is not promoted
94 if change['status'] != 'MERGED':
96 assert 'submitted' in change
97 # drop microseconds cause who cares
98 stime = re.sub('\..*$', '', change['submitted'])
99 submitted_date = datetime.datetime.strptime(stime, "%Y-%m-%d %H:%M:%S")
100 # Patch applies to overcloud/undercloud
101 if docker_image is None:
102 oc_url = urllib.parse.urljoin(
103 con.UPSTREAM_RDO.replace('master', branch), 'overcloud-full.tar')
104 oc_mtime = utils.get_url_modified_date(oc_url)
105 if oc_mtime > submitted_date:
106 logging.debug("oc image was last modified at {}, which is"
107 "newer than merge date: {}".format(oc_mtime,
111 # must be a docker patch, check docker tag modified time
112 docker_url = con.DOCKERHUB_OOO.replace('tripleomaster',
113 "tripleo{}".format(branch))
114 url_path = "{}/tags/{}".format(docker_image, con.DOCKER_TAG)
115 docker_url = urllib.parse.urljoin(docker_url, url_path)
116 logging.debug("docker url is: {}".format(docker_url))
117 docker_output = utils.open_webpage(docker_url, 10)
118 logging.debug('Docker web output: {}'.format(docker_output))
119 hub_mtime = json.loads(docker_output.decode())['last_updated']
120 hub_mtime = re.sub('\..*$', '', hub_mtime)
121 # docker modified time is in this format '2018-06-11T15:23:55.135744Z'
122 # and we drop microseconds
123 hub_dtime = datetime.datetime.strptime(hub_mtime, "%Y-%m-%dT%H:%M:%S")
124 if hub_dtime > submitted_date:
125 logging.debug("docker image: {} was last modified at {}, which is"
126 "newer than merge date: {}".format(docker_image,
133 def add_upstream_patches(patches, image, tmp_dir,
134 default_branch=os.path.join('stable',
135 con.DEFAULT_OS_VERSION),
136 uc_ip=None, docker_tag=None):
138 Adds patches from upstream OpenStack gerrit to Undercloud for deployment
139 :param patches: list of patches
140 :param image: undercloud image
141 :param tmp_dir: to store temporary patch files
142 :param default_branch: default branch to fetch commit (if not specified
144 :param uc_ip: undercloud IP (required only for docker patches)
145 :param docker_tag: Docker Tag (required only for docker patches)
146 :return: Set of docker services patched (if applicable)
148 virt_ops = [{con.VIRT_INSTALL: 'patch'}]
149 logging.debug("Evaluating upstream patches:\n{}".format(patches))
150 docker_services = set()
151 for patch in patches:
152 assert isinstance(patch, dict)
153 assert all(i in patch.keys() for i in ['project', 'change-id'])
154 if 'branch' in patch.keys():
155 branch = patch['branch']
157 branch = default_branch
158 patch_diff = build_utils.get_patch(patch['change-id'],
159 patch['project'], branch)
160 project_path = project_to_path(patch['project'])
161 # If docker tag and python we know this patch belongs on docker
162 # container for a docker service. Therefore we build the dockerfile
163 # and move the patch into the containers directory. We also assume
164 # this builder call is for overcloud, because we do not support
165 # undercloud containers
166 if docker_tag and 'python' in project_path:
167 # Projects map to multiple THT services, need to check which
169 ooo_docker_services = project_to_docker_image(patch['project'])
170 docker_img = ooo_docker_services[0]
172 ooo_docker_services = []
174 change = build_utils.get_change(con.OPENSTACK_GERRIT,
175 patch['project'], branch,
177 patch_promoted = is_patch_promoted(change,
178 branch.replace('/stable', ''),
181 if patch_diff and not patch_promoted:
182 patch_file = "{}.patch".format(patch['change-id'])
183 # If we found services, then we treat the patch like it applies to
185 if ooo_docker_services:
186 os_version = default_branch.replace('stable/', '')
187 for service in ooo_docker_services:
188 docker_services = docker_services.union({service})
190 "WORKDIR {}".format(project_path),
191 "ADD {} {}".format(patch_file, project_path),
192 "RUN patch -p1 < {}".format(patch_file)
194 src_img_uri = "{}:8787/tripleo{}/centos-binary-{}:" \
195 "{}".format(uc_ip, os_version, service,
197 oc_builder.build_dockerfile(service, tmp_dir, docker_cmds,
199 patch_file_path = os.path.join(tmp_dir, 'containers',
202 patch_file_path = os.path.join(tmp_dir, patch_file)
204 {con.VIRT_UPLOAD: "{}:{}".format(patch_file_path,
206 {con.VIRT_RUN_CMD: "cd {} && patch -p1 < {}".format(
207 project_path, patch_file)}])
208 logging.info("Adding patch {} to {}".format(patch_file,
210 with open(patch_file_path, 'w') as fh:
213 logging.info("Ignoring patch:\n{}".format(patch))
214 if len(virt_ops) > 1:
215 virt_utils.virt_customize(virt_ops, image)
216 return docker_services
219 def add_repo(repo_url, repo_name, image, tmp_dir):
220 assert repo_name is not None
221 assert repo_url is not None
222 repo_file = "{}.repo".format(repo_name)
223 repo_file_path = os.path.join(tmp_dir, repo_file)
225 "[{}]".format(repo_name),
226 "name={}".format(repo_name),
227 "baseurl={}".format(repo_url),
230 logging.debug("Creating repo file {}".format(repo_name))
231 with open(repo_file_path, 'w') as fh:
232 fh.writelines("{}\n".format(line) for line in content)
233 logging.debug("Adding repo {} to {}".format(repo_file, image))
234 virt_utils.virt_customize([
235 {con.VIRT_UPLOAD: "{}:/etc/yum.repos.d/".format(repo_file_path)}],
240 def create_git_archive(repo_url, repo_name, tmp_dir,
241 branch='master', prefix=''):
242 repo = git.Repo.clone_from(repo_url, os.path.join(tmp_dir, repo_name))
244 if branch != str(repo.active_branch):
245 repo_git.checkout("origin/{}".format(branch))
246 archive_path = os.path.join(tmp_dir, "{}.tar".format(repo_name))
247 with open(archive_path, 'wb') as fh:
248 repo.archive(fh, prefix=prefix)
249 logging.debug("Wrote archive file: {}".format(archive_path))