apex/builders/common_builder.py

   1 ##############################################################################
   2 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 # Common building utilities for undercloud and overcloud
  11
  12 import datetime
  13 import git
  14 import json
  15 import logging
  16 import os
  17 import re
  18 import urllib.parse
  19
  20 import apex.builders.overcloud_builder as oc_builder
  21 from apex import build_utils
  22 from apex.builders import exceptions as exc
  23 from apex.common import constants as con
  24 from apex.common import utils
  25 from apex.virtual import utils as virt_utils
  26
  27
  28 def project_to_path(project):
  29     """
  30     Translates project to absolute file path to use in patching
  31     :param project: name of project
  32     :return: File path
  33     """
  34     if project.startswith('openstack/'):
  35         project = os.path.basename(project)
  36     if 'puppet' in project:
  37         return "/etc/puppet/modules/{}".format(project.replace('puppet-', ''))
  38     elif 'tripleo-heat-templates' in project:
  39         return "/usr/share/openstack-tripleo-heat-templates"
  40     else:
  41         # assume python.  python patches will apply to a project name subdir.
  42         # For example, python-tripleoclient patch will apply to the
  43         # tripleoclient directory, which is the directory extracted during
  44         # python install into the PYTHONPATH.  Therefore we need to just be
  45         # in the PYTHONPATH directory to apply a patch
  46         return "/usr/lib/python2.7/site-packages/"
  47
  48
  49 def project_to_docker_image(project):
  50     """
  51     Translates OpenStack project to OOO services that are containerized
  52     :param project: name of OpenStack project
  53     :return: List of OOO docker service names
  54     """
  55     # Fetch all docker containers in docker hub with tripleo and filter
  56     # based on project
  57
  58     hub_output = utils.open_webpage(
  59         urllib.parse.urljoin(con.DOCKERHUB_OOO, '?page_size=1024'), timeout=10)
  60     try:
  61         results = json.loads(hub_output.decode())['results']
  62     except Exception as e:
  63         logging.error("Unable to parse docker hub output for"
  64                       "tripleoupstream repository")
  65         logging.debug("HTTP response from dockerhub:\n{}".format(hub_output))
  66         raise exc.ApexCommonBuilderException(
  67             "Failed to parse docker image info from Docker Hub: {}".format(e))
  68     logging.debug("Docker Hub tripleoupstream entities found: {}".format(
  69         results))
  70     docker_images = list()
  71     for result in results:
  72         if result['name'].startswith("centos-binary-{}".format(project)):
  73             # add as docker image shortname (just service name)
  74             docker_images.append(result['name'].replace('centos-binary-', ''))
  75
  76     return docker_images
  77
  78
  79 def is_patch_promoted(change, branch, docker_image=None):
  80     """
  81     Checks to see if a patch that is in merged exists in either the docker
  82     container or the promoted tripleo images
  83     :param change: gerrit change json output
  84     :param branch: branch to use when polling artifacts (does not include
  85     stable prefix)
  86     :param docker_image: container this applies to if (defaults to None)
  87     :return: True if the patch exists in a promoted artifact upstream
  88     """
  89     assert isinstance(change, dict)
  90     assert 'status' in change
  91
  92     # if not merged we already know this is not closed/abandoned, so we know
  93     # this is not promoted
  94     if change['status'] != 'MERGED':
  95         return False
  96     assert 'submitted' in change
  97     # drop microseconds cause who cares
  98     stime = re.sub('\..*$', '', change['submitted'])
  99     submitted_date = datetime.datetime.strptime(stime, "%Y-%m-%d %H:%M:%S")
 100     # Patch applies to overcloud/undercloud
 101     if docker_image is None:
 102         oc_url = urllib.parse.urljoin(
 103             con.UPSTREAM_RDO.replace('master', branch), 'overcloud-full.tar')
 104         oc_mtime = utils.get_url_modified_date(oc_url)
 105         if oc_mtime > submitted_date:
 106             logging.debug("oc image was last modified at {}, which is"
 107                           "newer than merge date: {}".format(oc_mtime,
 108                                                              submitted_date))
 109             return True
 110     else:
 111         # must be a docker patch, check docker tag modified time
 112         docker_url = con.DOCKERHUB_OOO.replace('tripleomaster',
 113                                                "tripleo{}".format(branch))
 114         url_path = "{}/tags/{}".format(docker_image, con.DOCKER_TAG)
 115         docker_url = urllib.parse.urljoin(docker_url, url_path)
 116         logging.debug("docker url is: {}".format(docker_url))
 117         docker_output = utils.open_webpage(docker_url, 10)
 118         logging.debug('Docker web output: {}'.format(docker_output))
 119         hub_mtime = json.loads(docker_output.decode())['last_updated']
 120         hub_mtime = re.sub('\..*$', '', hub_mtime)
 121         # docker modified time is in this format '2018-06-11T15:23:55.135744Z'
 122         # and we drop microseconds
 123         hub_dtime = datetime.datetime.strptime(hub_mtime, "%Y-%m-%dT%H:%M:%S")
 124         if hub_dtime > submitted_date:
 125             logging.debug("docker image: {} was last modified at {}, which is"
 126                           "newer than merge date: {}".format(docker_image,
 127                                                              hub_dtime,
 128                                                              submitted_date))
 129             return True
 130     return False
 131
 132
 133 def add_upstream_patches(patches, image, tmp_dir,
 134                          default_branch=os.path.join('stable',
 135                                                      con.DEFAULT_OS_VERSION),
 136                          uc_ip=None, docker_tag=None):
 137     """
 138     Adds patches from upstream OpenStack gerrit to Undercloud for deployment
 139     :param patches: list of patches
 140     :param image: undercloud image
 141     :param tmp_dir: to store temporary patch files
 142     :param default_branch: default branch to fetch commit (if not specified
 143     in patch)
 144     :param uc_ip: undercloud IP (required only for docker patches)
 145     :param docker_tag: Docker Tag (required only for docker patches)
 146     :return: Set of docker services patched (if applicable)
 147     """
 148     virt_ops = [{con.VIRT_INSTALL: 'patch'}]
 149     logging.debug("Evaluating upstream patches:\n{}".format(patches))
 150     docker_services = set()
 151     for patch in patches:
 152         assert isinstance(patch, dict)
 153         assert all(i in patch.keys() for i in ['project', 'change-id'])
 154         if 'branch' in patch.keys():
 155             branch = patch['branch']
 156         else:
 157             branch = default_branch
 158         patch_diff = build_utils.get_patch(patch['change-id'],
 159                                            patch['project'], branch)
 160         project_path = project_to_path(patch['project'])
 161         # If docker tag and python we know this patch belongs on docker
 162         # container for a docker service. Therefore we build the dockerfile
 163         # and move the patch into the containers directory.  We also assume
 164         # this builder call is for overcloud, because we do not support
 165         # undercloud containers
 166         if docker_tag and 'python' in project_path:
 167             # Projects map to multiple THT services, need to check which
 168             # are supported
 169             ooo_docker_services = project_to_docker_image(patch['project'])
 170             docker_img = ooo_docker_services[0]
 171         else:
 172             ooo_docker_services = []
 173             docker_img = None
 174         change = build_utils.get_change(con.OPENSTACK_GERRIT,
 175                                         patch['project'], branch,
 176                                         patch['change-id'])
 177         patch_promoted = is_patch_promoted(change,
 178                                            branch.replace('/stable', ''),
 179                                            docker_img)
 180
 181         if patch_diff and not patch_promoted:
 182             patch_file = "{}.patch".format(patch['change-id'])
 183             # If we found services, then we treat the patch like it applies to
 184             # docker only
 185             if ooo_docker_services:
 186                 os_version = default_branch.replace('stable/', '')
 187                 for service in ooo_docker_services:
 188                     docker_services = docker_services.union({service})
 189                     docker_cmds = [
 190                         "WORKDIR {}".format(project_path),
 191                         "ADD {} {}".format(patch_file, project_path),
 192                         "RUN patch -p1 < {}".format(patch_file)
 193                     ]
 194                     src_img_uri = "{}:8787/tripleo{}/centos-binary-{}:" \
 195                                   "{}".format(uc_ip, os_version, service,
 196                                               docker_tag)
 197                     oc_builder.build_dockerfile(service, tmp_dir, docker_cmds,
 198                                                 src_img_uri)
 199                 patch_file_path = os.path.join(tmp_dir, 'containers',
 200                                                patch_file)
 201             else:
 202                 patch_file_path = os.path.join(tmp_dir, patch_file)
 203                 virt_ops.extend([
 204                     {con.VIRT_UPLOAD: "{}:{}".format(patch_file_path,
 205                                                      project_path)},
 206                     {con.VIRT_RUN_CMD: "cd {} && patch -p1 < {}".format(
 207                         project_path, patch_file)}])
 208                 logging.info("Adding patch {} to {}".format(patch_file,
 209                                                             image))
 210             with open(patch_file_path, 'w') as fh:
 211                 fh.write(patch_diff)
 212         else:
 213             logging.info("Ignoring patch:\n{}".format(patch))
 214     if len(virt_ops) > 1:
 215         virt_utils.virt_customize(virt_ops, image)
 216     return docker_services
 217
 218
 219 def add_repo(repo_url, repo_name, image, tmp_dir):
 220     assert repo_name is not None
 221     assert repo_url is not None
 222     repo_file = "{}.repo".format(repo_name)
 223     repo_file_path = os.path.join(tmp_dir, repo_file)
 224     content = [
 225         "[{}]".format(repo_name),
 226         "name={}".format(repo_name),
 227         "baseurl={}".format(repo_url),
 228         "gpgcheck=0"
 229     ]
 230     logging.debug("Creating repo file {}".format(repo_name))
 231     with open(repo_file_path, 'w') as fh:
 232         fh.writelines("{}\n".format(line) for line in content)
 233     logging.debug("Adding repo {} to {}".format(repo_file, image))
 234     virt_utils.virt_customize([
 235         {con.VIRT_UPLOAD: "{}:/etc/yum.repos.d/".format(repo_file_path)}],
 236         image
 237     )
 238
 239
 240 def create_git_archive(repo_url, repo_name, tmp_dir,
 241                        branch='master', prefix=''):
 242     repo = git.Repo.clone_from(repo_url, os.path.join(tmp_dir, repo_name))
 243     repo_git = repo.git
 244     if branch != str(repo.active_branch):
 245         repo_git.checkout("origin/{}".format(branch))
 246     archive_path = os.path.join(tmp_dir, "{}.tar".format(repo_name))
 247     with open(archive_path, 'wb') as fh:
 248         repo.archive(fh, prefix=prefix)
 249     logging.debug("Wrote archive file: {}".format(archive_path))
 250     return archive_path