apex/builders/common_builder.py

   1 ##############################################################################
   2 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 # Common building utilities for undercloud and overcloud
  11
  12 import datetime
  13 import git
  14 import json
  15 import logging
  16 import os
  17 import re
  18 import urllib.parse
  19
  20 import apex.builders.overcloud_builder as oc_builder
  21 from apex import build_utils
  22 from apex.builders import exceptions as exc
  23 from apex.common import constants as con
  24 from apex.common import utils
  25 from apex.virtual import utils as virt_utils
  26
  27
  28 def project_to_path(project, patch=None):
  29     """
  30     Translates project to absolute file path to use in patching
  31     :param project: name of project
  32     :param patch: the patch to applied to the project
  33     :return: File path
  34     """
  35     if project.startswith('openstack/'):
  36         project = os.path.basename(project)
  37     if 'puppet' in project:
  38         return "/etc/puppet/modules/{}".format(project.replace('puppet-', ''))
  39     elif 'tripleo-heat-templates' in project:
  40         return "/usr/share/openstack-tripleo-heat-templates"
  41     elif ('tripleo-common' in project and
  42           build_utils.is_path_in_patch(patch, 'container-images/')):
  43         # tripleo-common has python and another component to it
  44         # here we detect if there is a change to the yaml component and if so
  45         # treat it like it is not python. This has the caveat of if there
  46         # is a patch to both python and yaml this will not work
  47         # FIXME(trozet): add ability to split tripleo-common patches that
  48         # modify both python and yaml
  49         return "/usr/share/openstack-tripleo-common-containers/"
  50     else:
  51         # assume python.  python patches will apply to a project name subdir.
  52         # For example, python-tripleoclient patch will apply to the
  53         # tripleoclient directory, which is the directory extracted during
  54         # python install into the PYTHONPATH.  Therefore we need to just be
  55         # in the PYTHONPATH directory to apply a patch
  56         return "/usr/lib/python2.7/site-packages/"
  57
  58
  59 def project_to_docker_image(project):
  60     """
  61     Translates OpenStack project to OOO services that are containerized
  62     :param project: name of OpenStack project
  63     :return: List of OOO docker service names
  64     """
  65     # Fetch all docker containers in docker hub with tripleo and filter
  66     # based on project
  67
  68     hub_output = utils.open_webpage(
  69         urllib.parse.urljoin(con.DOCKERHUB_OOO, '?page_size=1024'), timeout=10)
  70     try:
  71         results = json.loads(hub_output.decode())['results']
  72     except Exception as e:
  73         logging.error("Unable to parse docker hub output for"
  74                       "tripleoupstream repository")
  75         logging.debug("HTTP response from dockerhub:\n{}".format(hub_output))
  76         raise exc.ApexCommonBuilderException(
  77             "Failed to parse docker image info from Docker Hub: {}".format(e))
  78     logging.debug("Docker Hub tripleoupstream entities found: {}".format(
  79         results))
  80     docker_images = list()
  81     for result in results:
  82         if result['name'].startswith("centos-binary-{}".format(project)):
  83             # add as docker image shortname (just service name)
  84             docker_images.append(result['name'].replace('centos-binary-', ''))
  85
  86     return docker_images
  87
  88
  89 def is_patch_promoted(change, branch, docker_image=None):
  90     """
  91     Checks to see if a patch that is in merged exists in either the docker
  92     container or the promoted tripleo images
  93     :param change: gerrit change json output
  94     :param branch: branch to use when polling artifacts (does not include
  95     stable prefix)
  96     :param docker_image: container this applies to if (defaults to None)
  97     :return: True if the patch exists in a promoted artifact upstream
  98     """
  99     assert isinstance(change, dict)
 100     assert 'status' in change
 101
 102     # if not merged we already know this is not closed/abandoned, so we know
 103     # this is not promoted
 104     if change['status'] != 'MERGED':
 105         return False
 106     assert 'submitted' in change
 107     # drop microseconds cause who cares
 108     stime = re.sub('\..*$', '', change['submitted'])
 109     submitted_date = datetime.datetime.strptime(stime, "%Y-%m-%d %H:%M:%S")
 110     # Patch applies to overcloud/undercloud
 111     if docker_image is None:
 112         oc_url = urllib.parse.urljoin(
 113             con.UPSTREAM_RDO.replace('master', branch), 'overcloud-full.tar')
 114         oc_mtime = utils.get_url_modified_date(oc_url)
 115         if oc_mtime > submitted_date:
 116             logging.debug("oc image was last modified at {}, which is"
 117                           "newer than merge date: {}".format(oc_mtime,
 118                                                              submitted_date))
 119             return True
 120     else:
 121         # must be a docker patch, check docker tag modified time
 122         docker_url = con.DOCKERHUB_OOO.replace('tripleomaster',
 123                                                "tripleo{}".format(branch))
 124         url_path = "{}/tags/{}".format(docker_image, con.DOCKER_TAG)
 125         docker_url = urllib.parse.urljoin(docker_url, url_path)
 126         logging.debug("docker url is: {}".format(docker_url))
 127         docker_output = utils.open_webpage(docker_url, 10)
 128         logging.debug('Docker web output: {}'.format(docker_output))
 129         hub_mtime = json.loads(docker_output.decode())['last_updated']
 130         hub_mtime = re.sub('\..*$', '', hub_mtime)
 131         # docker modified time is in this format '2018-06-11T15:23:55.135744Z'
 132         # and we drop microseconds
 133         hub_dtime = datetime.datetime.strptime(hub_mtime, "%Y-%m-%dT%H:%M:%S")
 134         if hub_dtime > submitted_date:
 135             logging.debug("docker image: {} was last modified at {}, which is"
 136                           "newer than merge date: {}".format(docker_image,
 137                                                              hub_dtime,
 138                                                              submitted_date))
 139             return True
 140     return False
 141
 142
 143 def add_upstream_patches(patches, image, tmp_dir,
 144                          default_branch=os.path.join('stable',
 145                                                      con.DEFAULT_OS_VERSION),
 146                          uc_ip=None, docker_tag=None):
 147     """
 148     Adds patches from upstream OpenStack gerrit to Undercloud for deployment
 149     :param patches: list of patches
 150     :param image: undercloud image
 151     :param tmp_dir: to store temporary patch files
 152     :param default_branch: default branch to fetch commit (if not specified
 153     in patch)
 154     :param uc_ip: undercloud IP (required only for docker patches)
 155     :param docker_tag: Docker Tag (required only for docker patches)
 156     :return: Set of docker services patched (if applicable)
 157     """
 158     virt_ops = [{con.VIRT_INSTALL: 'patch'}]
 159     logging.debug("Evaluating upstream patches:\n{}".format(patches))
 160     docker_services = set()
 161     for patch in patches:
 162         assert isinstance(patch, dict)
 163         assert all(i in patch.keys() for i in ['project', 'change-id'])
 164         if 'branch' in patch.keys():
 165             branch = patch['branch']
 166         else:
 167             branch = default_branch
 168         patch_diff = build_utils.get_patch(patch['change-id'],
 169                                            patch['project'], branch)
 170         project_path = project_to_path(patch['project'], patch_diff)
 171         # If docker tag and python we know this patch belongs on docker
 172         # container for a docker service. Therefore we build the dockerfile
 173         # and move the patch into the containers directory.  We also assume
 174         # this builder call is for overcloud, because we do not support
 175         # undercloud containers
 176         if docker_tag and 'python' in project_path:
 177             # Projects map to multiple THT services, need to check which
 178             # are supported
 179             ooo_docker_services = project_to_docker_image(patch['project'])
 180             docker_img = ooo_docker_services[0]
 181         else:
 182             ooo_docker_services = []
 183             docker_img = None
 184         change = build_utils.get_change(con.OPENSTACK_GERRIT,
 185                                         patch['project'], branch,
 186                                         patch['change-id'])
 187         patch_promoted = is_patch_promoted(change,
 188                                            branch.replace('stable/', ''),
 189                                            docker_img)
 190
 191         if patch_diff and not patch_promoted:
 192             patch_file = "{}.patch".format(patch['change-id'])
 193             # If we found services, then we treat the patch like it applies to
 194             # docker only
 195             if ooo_docker_services:
 196                 os_version = default_branch.replace('stable/', '')
 197                 for service in ooo_docker_services:
 198                     docker_services = docker_services.union({service})
 199                     docker_cmds = [
 200                         "WORKDIR {}".format(project_path),
 201                         "ADD {} {}".format(patch_file, project_path),
 202                         "RUN patch -p1 < {}".format(patch_file)
 203                     ]
 204                     src_img_uri = "{}:8787/tripleo{}/centos-binary-{}:" \
 205                                   "{}".format(uc_ip, os_version, service,
 206                                               docker_tag)
 207                     oc_builder.build_dockerfile(service, tmp_dir, docker_cmds,
 208                                                 src_img_uri)
 209                 patch_file_path = os.path.join(tmp_dir, 'containers',
 210                                                patch_file)
 211             else:
 212                 patch_file_path = os.path.join(tmp_dir, patch_file)
 213                 virt_ops.extend([
 214                     {con.VIRT_UPLOAD: "{}:{}".format(patch_file_path,
 215                                                      project_path)},
 216                     {con.VIRT_RUN_CMD: "cd {} && patch -p1 < {}".format(
 217                         project_path, patch_file)}])
 218                 logging.info("Adding patch {} to {}".format(patch_file,
 219                                                             image))
 220             with open(patch_file_path, 'w') as fh:
 221                 fh.write(patch_diff)
 222         else:
 223             logging.info("Ignoring patch:\n{}".format(patch))
 224     if len(virt_ops) > 1:
 225         virt_utils.virt_customize(virt_ops, image)
 226     return docker_services
 227
 228
 229 def add_repo(repo_url, repo_name, image, tmp_dir):
 230     assert repo_name is not None
 231     assert repo_url is not None
 232     repo_file = "{}.repo".format(repo_name)
 233     repo_file_path = os.path.join(tmp_dir, repo_file)
 234     content = [
 235         "[{}]".format(repo_name),
 236         "name={}".format(repo_name),
 237         "baseurl={}".format(repo_url),
 238         "gpgcheck=0"
 239     ]
 240     logging.debug("Creating repo file {}".format(repo_name))
 241     with open(repo_file_path, 'w') as fh:
 242         fh.writelines("{}\n".format(line) for line in content)
 243     logging.debug("Adding repo {} to {}".format(repo_file, image))
 244     virt_utils.virt_customize([
 245         {con.VIRT_UPLOAD: "{}:/etc/yum.repos.d/".format(repo_file_path)}],
 246         image
 247     )
 248
 249
 250 def create_git_archive(repo_url, repo_name, tmp_dir,
 251                        branch='master', prefix=''):
 252     repo = git.Repo.clone_from(repo_url, os.path.join(tmp_dir, repo_name))
 253     repo_git = repo.git
 254     if branch != str(repo.active_branch):
 255         repo_git.checkout("origin/{}".format(branch))
 256     archive_path = os.path.join(tmp_dir, "{}.tar".format(repo_name))
 257     with open(archive_path, 'wb') as fh:
 258         repo.archive(fh, prefix=prefix)
 259     logging.debug("Wrote archive file: {}".format(archive_path))
 260     return archive_path