apex/builders/common_builder.py

   1 ##############################################################################
   2 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 # Common building utilities for undercloud and overcloud
  11
  12 import datetime
  13 import git
  14 import json
  15 import logging
  16 import os
  17 import re
  18 import urllib.parse
  19
  20 import apex.builders.overcloud_builder as oc_builder
  21 from apex import build_utils
  22 from apex.builders import exceptions as exc
  23 from apex.common import constants as con
  24 from apex.common import utils
  25 from apex.virtual import utils as virt_utils
  26
  27
  28 def project_to_path(project, patch=None):
  29     """
  30     Translates project to absolute file path to use in patching
  31     :param project: name of project
  32     :param patch: the patch to applied to the project
  33     :return: File path
  34     """
  35     if project.startswith('openstack/'):
  36         project = os.path.basename(project)
  37     if 'puppet' in project:
  38         return "/etc/puppet/modules/{}".format(project.replace('puppet-', ''))
  39     elif 'tripleo-heat-templates' in project:
  40         return "/usr/share/openstack-tripleo-heat-templates"
  41     elif ('tripleo-common' in project and
  42           build_utils.is_path_in_patch(patch, 'container-images/')):
  43         # tripleo-common has python and another component to it
  44         # here we detect if there is a change to the yaml component and if so
  45         # treat it like it is not python. This has the caveat of if there
  46         # is a patch to both python and yaml this will not work
  47         # FIXME(trozet): add ability to split tripleo-common patches that
  48         # modify both python and yaml
  49         return "/usr/share/openstack-tripleo-common-containers/"
  50     else:
  51         # assume python.  python patches will apply to a project name subdir.
  52         # For example, python-tripleoclient patch will apply to the
  53         # tripleoclient directory, which is the directory extracted during
  54         # python install into the PYTHONPATH.  Therefore we need to just be
  55         # in the PYTHONPATH directory to apply a patch
  56         return "/usr/lib/python2.7/site-packages/"
  57
  58
  59 def project_to_docker_image(project):
  60     """
  61     Translates OpenStack project to OOO services that are containerized
  62     :param project: name of OpenStack project
  63     :return: List of OOO docker service names
  64     """
  65     # Fetch all docker containers in docker hub with tripleo and filter
  66     # based on project
  67
  68     hub_output = utils.open_webpage(
  69         urllib.parse.urljoin(con.DOCKERHUB_OOO, '?page_size=1024'), timeout=10)
  70     try:
  71         results = json.loads(hub_output.decode())['results']
  72     except Exception as e:
  73         logging.error("Unable to parse docker hub output for"
  74                       "tripleoupstream repository")
  75         logging.debug("HTTP response from dockerhub:\n{}".format(hub_output))
  76         raise exc.ApexCommonBuilderException(
  77             "Failed to parse docker image info from Docker Hub: {}".format(e))
  78     logging.debug("Docker Hub tripleoupstream entities found: {}".format(
  79         results))
  80     docker_images = list()
  81     for result in results:
  82         if result['name'].startswith("centos-binary-{}".format(project)):
  83             # add as docker image shortname (just service name)
  84             docker_images.append(result['name'].replace('centos-binary-', ''))
  85
  86     return docker_images
  87
  88
  89 def is_patch_promoted(change, branch, docker_image=None):
  90     """
  91     Checks to see if a patch that is in merged exists in either the docker
  92     container or the promoted tripleo images
  93     :param change: gerrit change json output
  94     :param branch: branch to use when polling artifacts (does not include
  95     stable prefix)
  96     :param docker_image: container this applies to if (defaults to None)
  97     :return: True if the patch exists in a promoted artifact upstream
  98     """
  99     assert isinstance(change, dict)
 100     assert 'status' in change
 101
 102     # if not merged we already know this is not closed/abandoned, so we know
 103     # this is not promoted
 104     if change['status'] != 'MERGED':
 105         return False
 106     assert 'submitted' in change
 107     # drop microseconds cause who cares
 108     stime = re.sub('\..*$', '', change['submitted'])
 109     submitted_date = datetime.datetime.strptime(stime, "%Y-%m-%d %H:%M:%S")
 110     # Patch applies to overcloud/undercloud
 111     if docker_image is None:
 112         oc_url = urllib.parse.urljoin(
 113             con.UPSTREAM_RDO.replace(con.DEFAULT_OS_VERSION,
 114                                      branch), 'overcloud-full.tar')
 115         oc_mtime = utils.get_url_modified_date(oc_url)
 116         if oc_mtime > submitted_date:
 117             logging.debug("oc image was last modified at {}, which is"
 118                           "newer than merge date: {}".format(oc_mtime,
 119                                                              submitted_date))
 120             return True
 121     else:
 122         # must be a docker patch, check docker tag modified time
 123         docker_url = con.DOCKERHUB_OOO.replace('tripleomaster',
 124                                                "tripleo{}".format(branch))
 125         url_path = "{}/tags/{}".format(docker_image, con.DOCKER_TAG)
 126         docker_url = urllib.parse.urljoin(docker_url, url_path)
 127         logging.debug("docker url is: {}".format(docker_url))
 128         docker_output = utils.open_webpage(docker_url, 10)
 129         logging.debug('Docker web output: {}'.format(docker_output))
 130         hub_mtime = json.loads(docker_output.decode())['last_updated']
 131         hub_mtime = re.sub('\..*$', '', hub_mtime)
 132         # docker modified time is in this format '2018-06-11T15:23:55.135744Z'
 133         # and we drop microseconds
 134         hub_dtime = datetime.datetime.strptime(hub_mtime, "%Y-%m-%dT%H:%M:%S")
 135         if hub_dtime > submitted_date:
 136             logging.debug("docker image: {} was last modified at {}, which is"
 137                           "newer than merge date: {}".format(docker_image,
 138                                                              hub_dtime,
 139                                                              submitted_date))
 140             return True
 141     return False
 142
 143
 144 def add_upstream_patches(patches, image, tmp_dir,
 145                          default_branch=os.path.join('stable',
 146                                                      con.DEFAULT_OS_VERSION),
 147                          uc_ip=None, docker_tag=None):
 148     """
 149     Adds patches from upstream OpenStack gerrit to Undercloud for deployment
 150     :param patches: list of patches
 151     :param image: undercloud image
 152     :param tmp_dir: to store temporary patch files
 153     :param default_branch: default branch to fetch commit (if not specified
 154     in patch)
 155     :param uc_ip: undercloud IP (required only for docker patches)
 156     :param docker_tag: Docker Tag (required only for docker patches)
 157     :return: Set of docker services patched (if applicable)
 158     """
 159     virt_ops = [{con.VIRT_INSTALL: 'patch'}]
 160     logging.debug("Evaluating upstream patches:\n{}".format(patches))
 161     docker_services = set()
 162     for patch in patches:
 163         assert isinstance(patch, dict)
 164         assert all(i in patch.keys() for i in ['project', 'change-id'])
 165         if 'branch' in patch.keys():
 166             branch = patch['branch']
 167         else:
 168             branch = default_branch
 169         patch_diff = build_utils.get_patch(patch['change-id'],
 170                                            patch['project'], branch)
 171         project_path = project_to_path(patch['project'], patch_diff)
 172         # If docker tag and python we know this patch belongs on docker
 173         # container for a docker service. Therefore we build the dockerfile
 174         # and move the patch into the containers directory.  We also assume
 175         # this builder call is for overcloud, because we do not support
 176         # undercloud containers
 177         if docker_tag and 'python' in project_path:
 178             # Projects map to multiple THT services, need to check which
 179             # are supported
 180             ooo_docker_services = project_to_docker_image(patch['project'])
 181             docker_img = ooo_docker_services[0]
 182         else:
 183             ooo_docker_services = []
 184             docker_img = None
 185         change = build_utils.get_change(con.OPENSTACK_GERRIT,
 186                                         patch['project'], branch,
 187                                         patch['change-id'])
 188         patch_promoted = is_patch_promoted(change,
 189                                            branch.replace('stable/', ''),
 190                                            docker_img)
 191
 192         if patch_diff and not patch_promoted:
 193             patch_file = "{}.patch".format(patch['change-id'])
 194             # If we found services, then we treat the patch like it applies to
 195             # docker only
 196             if ooo_docker_services:
 197                 os_version = default_branch.replace('stable/', '')
 198                 for service in ooo_docker_services:
 199                     docker_services = docker_services.union({service})
 200                     docker_cmds = [
 201                         "WORKDIR {}".format(project_path),
 202                         "ADD {} {}".format(patch_file, project_path),
 203                         "RUN patch -p1 < {}".format(patch_file)
 204                     ]
 205                     src_img_uri = "{}:8787/tripleo{}/centos-binary-{}:" \
 206                                   "{}".format(uc_ip, os_version, service,
 207                                               docker_tag)
 208                     oc_builder.build_dockerfile(service, tmp_dir, docker_cmds,
 209                                                 src_img_uri)
 210                 patch_file_path = os.path.join(tmp_dir, 'containers',
 211                                                patch_file)
 212             else:
 213                 patch_file_path = os.path.join(tmp_dir, patch_file)
 214                 virt_ops.extend([
 215                     {con.VIRT_UPLOAD: "{}:{}".format(patch_file_path,
 216                                                      project_path)},
 217                     {con.VIRT_RUN_CMD: "cd {} && patch -p1 < {}".format(
 218                         project_path, patch_file)}])
 219                 logging.info("Adding patch {} to {}".format(patch_file,
 220                                                             image))
 221             with open(patch_file_path, 'w') as fh:
 222                 fh.write(patch_diff)
 223         else:
 224             logging.info("Ignoring patch:\n{}".format(patch))
 225     if len(virt_ops) > 1:
 226         virt_utils.virt_customize(virt_ops, image)
 227     return docker_services
 228
 229
 230 def add_repo(repo_url, repo_name, image, tmp_dir):
 231     assert repo_name is not None
 232     assert repo_url is not None
 233     repo_file = "{}.repo".format(repo_name)
 234     repo_file_path = os.path.join(tmp_dir, repo_file)
 235     content = [
 236         "[{}]".format(repo_name),
 237         "name={}".format(repo_name),
 238         "baseurl={}".format(repo_url),
 239         "gpgcheck=0"
 240     ]
 241     logging.debug("Creating repo file {}".format(repo_name))
 242     with open(repo_file_path, 'w') as fh:
 243         fh.writelines("{}\n".format(line) for line in content)
 244     logging.debug("Adding repo {} to {}".format(repo_file, image))
 245     virt_utils.virt_customize([
 246         {con.VIRT_UPLOAD: "{}:/etc/yum.repos.d/".format(repo_file_path)}],
 247         image
 248     )
 249
 250
 251 def create_git_archive(repo_url, repo_name, tmp_dir,
 252                        branch='master', prefix=''):
 253     repo = git.Repo.clone_from(repo_url, os.path.join(tmp_dir, repo_name))
 254     repo_git = repo.git
 255     if branch != str(repo.active_branch):
 256         repo_git.checkout("origin/{}".format(branch))
 257     archive_path = os.path.join(tmp_dir, "{}.tar".format(repo_name))
 258     with open(archive_path, 'wb') as fh:
 259         repo.archive(fh, prefix=prefix)
 260     logging.debug("Wrote archive file: {}".format(archive_path))
 261     return archive_path