0cd683c3f090167285b8f0d2c9bfb675ddae6b92
[apex.git] / apex / builders / common_builder.py
1 ##############################################################################
2 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
3 #
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
9
10 # Common building utilities for undercloud and overcloud
11
12 import git
13 import json
14 import logging
15 import os
16 import re
17
18 import apex.builders.overcloud_builder as oc_builder
19 from apex import build_utils
20 from apex.builders import exceptions as exc
21 from apex.common import constants as con
22 from apex.common import utils
23 from apex.virtual import utils as virt_utils
24
25
26 def project_to_path(project):
27     """
28     Translates project to absolute file path to use in patching
29     :param project: name of project
30     :return: File path
31     """
32     if project.startswith('openstack/'):
33         project = os.path.basename(project)
34     if 'puppet' in project:
35         return "/etc/puppet/modules/{}".format(project.replace('puppet-', ''))
36     elif 'tripleo-heat-templates' in project:
37         return "/usr/share/openstack-tripleo-heat-templates"
38     else:
39         # assume python.  python patches will apply to a project name subdir.
40         # For example, python-tripleoclient patch will apply to the
41         # tripleoclient directory, which is the directory extracted during
42         # python install into the PYTHONPATH.  Therefore we need to just be
43         # in the PYTHONPATH directory to apply a patch
44         return "/usr/lib/python2.7/site-packages/"
45
46
47 def project_to_docker_image(project):
48     """
49     Translates OpenStack project to OOO services that are containerized
50     :param project: name of OpenStack project
51     :return: List of OOO docker service names
52     """
53     # Fetch all docker containers in docker hub with tripleo and filter
54     # based on project
55     hub_output = utils.open_webpage(con.DOCKERHUB_OOO, timeout=10)
56     try:
57         results = json.loads(hub_output.decode())['results']
58     except Exception as e:
59         logging.error("Unable to parse docker hub output for"
60                       "tripleoupstream repository")
61         logging.debug("HTTP response from dockerhub:\n{}".format(hub_output))
62         raise exc.ApexCommonBuilderException(
63             "Failed to parse docker image info from Docker Hub: {}".format(e))
64     logging.debug("Docker Hub tripleoupstream entities found: {}".format(
65         results))
66     docker_images = list()
67     for result in results:
68         if result['name'].startswith("centos-binary-{}".format(project)):
69             # add as docker image shortname (just service name)
70             docker_images.append(result['name'].replace('centos-binary-', ''))
71
72     return docker_images
73
74
75 def add_upstream_patches(patches, image, tmp_dir,
76                          default_branch=os.path.join('stable',
77                                                      con.DEFAULT_OS_VERSION),
78                          uc_ip=None, docker_tag=None):
79     """
80     Adds patches from upstream OpenStack gerrit to Undercloud for deployment
81     :param patches: list of patches
82     :param image: undercloud image
83     :param tmp_dir: to store temporary patch files
84     :param default_branch: default branch to fetch commit (if not specified
85     in patch)
86     :param uc_ip: undercloud IP (required only for docker patches)
87     :param docker_tag: Docker Tag (required only for docker patches)
88     :return: Set of docker services patched (if applicable)
89     """
90     virt_ops = [{con.VIRT_INSTALL: 'patch'}]
91     logging.debug("Evaluating upstream patches:\n{}".format(patches))
92     docker_services = set()
93     for patch in patches:
94         assert isinstance(patch, dict)
95         assert all(i in patch.keys() for i in ['project', 'change-id'])
96         if 'branch' in patch.keys():
97             branch = patch['branch']
98         else:
99             branch = default_branch
100         patch_diff = build_utils.get_patch(patch['change-id'],
101                                            patch['project'], branch)
102         if patch_diff:
103             patch_file = "{}.patch".format(patch['change-id'])
104             project_path = project_to_path(patch['project'])
105             # If docker tag and python we know this patch belongs on docker
106             # container for a docker service. Therefore we build the dockerfile
107             # and move the patch into the containers directory.  We also assume
108             # this builder call is for overcloud, because we do not support
109             # undercloud containers
110             if docker_tag and 'python' in project_path:
111                 # Projects map to multiple THT services, need to check which
112                 # are supported
113                 ooo_docker_services = project_to_docker_image(patch['project'])
114             else:
115                 ooo_docker_services = []
116             # If we found services, then we treat the patch like it applies to
117             # docker only
118             if ooo_docker_services:
119                 os_version = default_branch.replace('stable/', '')
120                 for service in ooo_docker_services:
121                     docker_services = docker_services.union({service})
122                     docker_cmds = [
123                         "WORKDIR {}".format(project_path),
124                         "ADD {} {}".format(patch_file, project_path),
125                         "RUN patch -p1 < {}".format(patch_file)
126                     ]
127                     src_img_uri = "{}:8787/tripleo{}/centos-binary-{}:" \
128                                   "{}".format(uc_ip, os_version, service,
129                                               docker_tag)
130                     oc_builder.build_dockerfile(service, tmp_dir, docker_cmds,
131                                                 src_img_uri)
132                 patch_file_path = os.path.join(tmp_dir, 'containers',
133                                                patch_file)
134             else:
135                 patch_file_path = os.path.join(tmp_dir, patch_file)
136                 virt_ops.extend([
137                     {con.VIRT_UPLOAD: "{}:{}".format(patch_file_path,
138                                                      project_path)},
139                     {con.VIRT_RUN_CMD: "cd {} && patch -p1 < {}".format(
140                         project_path, patch_file)}])
141                 logging.info("Adding patch {} to {}".format(patch_file,
142                                                             image))
143             with open(patch_file_path, 'w') as fh:
144                 fh.write(patch_diff)
145         else:
146             logging.info("Ignoring patch:\n{}".format(patch))
147     if len(virt_ops) > 1:
148         virt_utils.virt_customize(virt_ops, image)
149     return docker_services
150
151
152 def add_repo(repo_url, repo_name, image, tmp_dir):
153     assert repo_name is not None
154     assert repo_url is not None
155     repo_file = "{}.repo".format(repo_name)
156     repo_file_path = os.path.join(tmp_dir, repo_file)
157     content = [
158         "[{}]".format(repo_name),
159         "name={}".format(repo_name),
160         "baseurl={}".format(repo_url),
161         "gpgcheck=0"
162     ]
163     logging.debug("Creating repo file {}".format(repo_name))
164     with open(repo_file_path, 'w') as fh:
165         fh.writelines("{}\n".format(line) for line in content)
166     logging.debug("Adding repo {} to {}".format(repo_file, image))
167     virt_utils.virt_customize([
168         {con.VIRT_UPLOAD: "{}:/etc/yum.repos.d/".format(repo_file_path)}],
169         image
170     )
171
172
173 def create_git_archive(repo_url, repo_name, tmp_dir,
174                        branch='master', prefix=''):
175     repo = git.Repo.clone_from(repo_url, os.path.join(tmp_dir, repo_name))
176     repo_git = repo.git
177     if branch != str(repo.active_branch):
178         repo_git.checkout("origin/{}".format(branch))
179     archive_path = os.path.join(tmp_dir, "{}.tar".format(repo_name))
180     with open(archive_path, 'wb') as fh:
181         repo.archive(fh, prefix=prefix)
182     logging.debug("Wrote archive file: {}".format(archive_path))
183     return archive_path