1 ##############################################################################
2 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
10 # Common building utilities for undercloud and overcloud
18 import apex.builders.overcloud_builder as oc_builder
19 from apex import build_utils
20 from apex.builders import exceptions as exc
21 from apex.common import constants as con
22 from apex.common import utils
23 from apex.virtual import utils as virt_utils
26 def project_to_path(project):
28 Translates project to absolute file path
29 :param project: name of project
32 if project.startswith('openstack/'):
33 project = os.path.basename(project)
34 if 'puppet' in project:
35 return "/etc/puppet/modules/{}".format(project.replace('puppet-', ''))
36 elif 'tripleo-heat-templates' in project:
37 return "/usr/share/openstack-tripleo-heat-templates"
40 return "/usr/lib/python2.7/site-packages/{}".format(project)
43 def project_to_docker_image(project):
45 Translates OpenStack project to OOO services that are containerized
46 :param project: name of OpenStack project
47 :return: List of OOO docker service names
49 # Fetch all docker containers in docker hub with tripleo and filter
51 hub_output = utils.open_webpage(con.DOCKERHUB_OOO, timeout=10)
53 results = json.loads(hub_output.decode())['results']
54 except Exception as e:
55 logging.error("Unable to parse docker hub output for"
56 "tripleoupstream repository")
57 logging.debug("HTTP response from dockerhub:\n{}".format(hub_output))
58 raise exc.ApexCommonBuilderException(
59 "Failed to parse docker image info from Docker Hub: {}".format(e))
60 logging.debug("Docker Hub tripleoupstream entities found: {}".format(
62 docker_images = list()
63 for result in results:
64 if result['name'].startswith("centos-binary-{}".format(project)):
65 # add as docker image shortname (just service name)
66 docker_images.append(result['name'].replace('centos-binary-', ''))
71 def add_upstream_patches(patches, image, tmp_dir,
72 default_branch=os.path.join('stable',
73 con.DEFAULT_OS_VERSION),
74 uc_ip=None, docker_tag=None):
76 Adds patches from upstream OpenStack gerrit to Undercloud for deployment
77 :param patches: list of patches
78 :param image: undercloud image
79 :param tmp_dir: to store temporary patch files
80 :param default_branch: default branch to fetch commit (if not specified
82 :param uc_ip: undercloud IP (required only for docker patches)
83 :param docker_tag: Docker Tag (required only for docker patches)
84 :return: Set of docker services patched (if applicable)
86 virt_ops = [{con.VIRT_INSTALL: 'patch'}]
87 logging.debug("Evaluating upstream patches:\n{}".format(patches))
88 docker_services = set()
90 assert isinstance(patch, dict)
91 assert all(i in patch.keys() for i in ['project', 'change-id'])
92 if 'branch' in patch.keys():
93 branch = patch['branch']
95 branch = default_branch
96 patch_diff = build_utils.get_patch(patch['change-id'],
97 patch['project'], branch)
99 patch_file = "{}.patch".format(patch['change-id'])
100 project_path = project_to_path(patch['project'])
101 # If docker tag and python we know this patch belongs on docker
102 # container for a docker service. Therefore we build the dockerfile
103 # and move the patch into the containers directory. We also assume
104 # this builder call is for overcloud, because we do not support
105 # undercloud containers
106 if docker_tag and 'python' in project_path:
107 # Projects map to multiple THT services, need to check which
109 ooo_docker_services = project_to_docker_image(patch['project'])
111 ooo_docker_services = []
112 # If we found services, then we treat the patch like it applies to
114 if ooo_docker_services:
115 os_version = default_branch.replace('stable/', '')
116 for service in ooo_docker_services:
117 docker_services = docker_services.union({service})
119 "WORKDIR {}".format(project_path),
120 "ADD {} {}".format(patch_file, project_path),
121 "RUN patch -p1 < {}".format(patch_file)
123 src_img_uri = "{}:8787/{}/centos-binary-{}:" \
124 "{}".format(uc_ip, os_version, service,
126 oc_builder.build_dockerfile(service, tmp_dir, docker_cmds,
128 patch_file_path = os.path.join(tmp_dir, 'containers',
131 patch_file_path = os.path.join(tmp_dir, patch_file)
133 {con.VIRT_UPLOAD: "{}:{}".format(patch_file_path,
135 {con.VIRT_RUN_CMD: "cd {} && patch -p1 < {}".format(
136 project_path, patch_file)}])
137 logging.info("Adding patch {} to {}".format(patch_file,
139 with open(patch_file_path, 'w') as fh:
142 logging.info("Ignoring patch:\n{}".format(patch))
143 if len(virt_ops) > 1:
144 virt_utils.virt_customize(virt_ops, image)
145 return docker_services
148 def add_repo(repo_url, repo_name, image, tmp_dir):
149 assert repo_name is not None
150 assert repo_url is not None
151 repo_file = "{}.repo".format(repo_name)
152 repo_file_path = os.path.join(tmp_dir, repo_file)
154 "[{}]".format(repo_name),
155 "name={}".format(repo_name),
156 "baseurl={}".format(repo_url),
159 logging.debug("Creating repo file {}".format(repo_name))
160 with open(repo_file_path, 'w') as fh:
161 fh.writelines("{}\n".format(line) for line in content)
162 logging.debug("Adding repo {} to {}".format(repo_file, image))
163 virt_utils.virt_customize([
164 {con.VIRT_UPLOAD: "{}:/etc/yum.repos.d/".format(repo_file_path)}],
169 def create_git_archive(repo_url, repo_name, tmp_dir,
170 branch='master', prefix=''):
171 repo = git.Repo.clone_from(repo_url, os.path.join(tmp_dir, repo_name))
173 if branch != str(repo.active_branch):
174 repo_git.checkout("origin/{}".format(branch))
175 archive_path = os.path.join(tmp_dir, "{}.tar".format(repo_name))
176 with open(archive_path, 'wb') as fh:
177 repo.archive(fh, prefix=prefix)
178 logging.debug("Wrote archive file: {}".format(archive_path))