apex/builders/common_builder.py

   1 ##############################################################################
   2 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 # Common building utilities for undercloud and overcloud
  11
  12 import datetime
  13 import git
  14 import json
  15 import logging
  16 import os
  17 import platform
  18 import pprint
  19 import re
  20 import urllib.parse
  21 import yaml
  22
  23 import apex.builders.overcloud_builder as oc_builder
  24 from apex import build_utils
  25 from apex.builders import exceptions as exc
  26 from apex.common import constants as con
  27 from apex.common import utils
  28 from apex.virtual import utils as virt_utils
  29
  30
  31 def project_to_path(project, patch=None):
  32     """
  33     Translates project to absolute file path to use in patching
  34     :param project: name of project
  35     :param patch: the patch to applied to the project
  36     :return: File path
  37     """
  38     if project.startswith('openstack/'):
  39         project = os.path.basename(project)
  40     if 'puppet' in project:
  41         return "/etc/puppet/modules/{}".format(project.replace('puppet-', ''))
  42     elif 'tripleo-heat-templates' in project:
  43         return "/usr/share/openstack-tripleo-heat-templates"
  44     elif ('tripleo-common' in project and
  45           build_utils.is_path_in_patch(patch, 'container-images/')):
  46         # tripleo-common has python and another component to it
  47         # here we detect if there is a change to the yaml component and if so
  48         # treat it like it is not python. This has the caveat of if there
  49         # is a patch to both python and yaml this will not work
  50         # FIXME(trozet): add ability to split tripleo-common patches that
  51         # modify both python and yaml
  52         return "/usr/share/openstack-tripleo-common-containers/"
  53     else:
  54         # assume python.  python patches will apply to a project name subdir.
  55         # For example, python-tripleoclient patch will apply to the
  56         # tripleoclient directory, which is the directory extracted during
  57         # python install into the PYTHONPATH.  Therefore we need to just be
  58         # in the PYTHONPATH directory to apply a patch
  59         return "/usr/lib/python2.7/site-packages/"
  60
  61
  62 def project_to_docker_image(project, docker_url):
  63     """
  64     Translates OpenStack project to OOO services that are containerized
  65     :param project: name of OpenStack project
  66     :return: List of OOO docker service names
  67     """
  68     # Fetch all docker containers in docker hub with tripleo and filter
  69     # based on project
  70
  71     hub_output = utils.open_webpage(
  72         urllib.parse.urljoin(docker_url,
  73                              '?page_size=1024'), timeout=10)
  74     try:
  75         results = json.loads(hub_output.decode())['results']
  76     except Exception as e:
  77         logging.error("Unable to parse docker hub output for"
  78                       "tripleoupstream repository")
  79         logging.debug("HTTP response from dockerhub:\n{}".format(hub_output))
  80         raise exc.ApexCommonBuilderException(
  81             "Failed to parse docker image info from Docker Hub: {}".format(e))
  82     logging.debug("Docker Hub tripleoupstream entities found: {}".format(
  83         results))
  84     docker_images = list()
  85     for result in results:
  86         if result['name'].startswith("centos-binary-{}".format(project)):
  87             # add as docker image shortname (just service name)
  88             docker_images.append(result['name'].replace('centos-binary-', ''))
  89
  90     return docker_images
  91
  92
  93 def is_patch_promoted(change, branch, docker_url, docker_image=None):
  94     """
  95     Checks to see if a patch that is in merged exists in either the docker
  96     container or the promoted tripleo images
  97     :param change: gerrit change json output
  98     :param branch: branch to use when polling artifacts (does not include
  99     stable prefix)
 100     :param docker_image: container this applies to if (defaults to None)
 101     :return: True if the patch exists in a promoted artifact upstream
 102     """
 103     assert isinstance(change, dict)
 104     assert 'status' in change
 105
 106     # if not merged we already know this is not closed/abandoned, so we know
 107     # this is not promoted
 108     if change['status'] != 'MERGED':
 109         return False
 110     assert 'submitted' in change
 111     # drop microseconds cause who cares
 112     stime = re.sub('\..*$', '', change['submitted'])
 113     submitted_date = datetime.datetime.strptime(stime, "%Y-%m-%d %H:%M:%S")
 114     # Patch applies to overcloud/undercloud
 115     if docker_image is None:
 116         oc_url = urllib.parse.urljoin(
 117             con.UPSTREAM_RDO.replace('master', branch), 'overcloud-full.tar')
 118         oc_mtime = utils.get_url_modified_date(oc_url)
 119         if oc_mtime > submitted_date:
 120             logging.debug("oc image was last modified at {}, which is"
 121                           "newer than merge date: {}".format(oc_mtime,
 122                                                              submitted_date))
 123             return True
 124     else:
 125         # must be a docker patch, check docker tag modified time
 126         docker_url = docker_url.replace('tripleomaster',
 127                                         "tripleo{}".format(branch))
 128         url_path = "{}/tags/{}".format(docker_image, con.DOCKER_TAG)
 129         docker_url = urllib.parse.urljoin(docker_url, url_path)
 130         logging.debug("docker url is: {}".format(docker_url))
 131         docker_output = utils.open_webpage(docker_url, 10)
 132         logging.debug('Docker web output: {}'.format(docker_output))
 133         hub_mtime = json.loads(docker_output.decode())['last_updated']
 134         hub_mtime = re.sub('\..*$', '', hub_mtime)
 135         # docker modified time is in this format '2018-06-11T15:23:55.135744Z'
 136         # and we drop microseconds
 137         hub_dtime = datetime.datetime.strptime(hub_mtime, "%Y-%m-%dT%H:%M:%S")
 138         if hub_dtime > submitted_date:
 139             logging.debug("docker image: {} was last modified at {}, which is"
 140                           "newer than merge date: {}".format(docker_image,
 141                                                              hub_dtime,
 142                                                              submitted_date))
 143             return True
 144     return False
 145
 146
 147 def add_upstream_patches(patches, image, tmp_dir,
 148                          default_branch=os.path.join('stable',
 149                                                      con.DEFAULT_OS_VERSION),
 150                          uc_ip=None, docker_tag=None):
 151     """
 152     Adds patches from upstream OpenStack gerrit to Undercloud for deployment
 153     :param patches: list of patches
 154     :param image: undercloud image
 155     :param tmp_dir: to store temporary patch files
 156     :param default_branch: default branch to fetch commit (if not specified
 157     in patch)
 158     :param uc_ip: undercloud IP (required only for docker patches)
 159     :param docker_tag: Docker Tag (required only for docker patches)
 160     :return: Set of docker services patched (if applicable)
 161     """
 162     virt_ops = [{con.VIRT_INSTALL: 'patch'}]
 163     logging.debug("Evaluating upstream patches:\n{}".format(patches))
 164     docker_services = set()
 165     for patch in patches:
 166         assert isinstance(patch, dict)
 167         assert all(i in patch.keys() for i in ['project', 'change-id'])
 168         if 'branch' in patch.keys():
 169             branch = patch['branch']
 170         else:
 171             branch = default_branch
 172         patch_diff = build_utils.get_patch(patch['change-id'],
 173                                            patch['project'], branch)
 174         project_path = project_to_path(patch['project'], patch_diff)
 175         # If docker tag and python we know this patch belongs on docker
 176         # container for a docker service. Therefore we build the dockerfile
 177         # and move the patch into the containers directory.  We also assume
 178         # this builder call is for overcloud, because we do not support
 179         # undercloud containers
 180         if platform.machine() == 'aarch64':
 181             docker_url = con.DOCKERHUB_AARCH64
 182         else:
 183             docker_url = con.DOCKERHUB_OOO
 184         if docker_tag and 'python' in project_path:
 185             # Projects map to multiple THT services, need to check which
 186             # are supported
 187             ooo_docker_services = project_to_docker_image(patch['project'],
 188                                                           docker_url)
 189             docker_img = ooo_docker_services[0]
 190         else:
 191             ooo_docker_services = []
 192             docker_img = None
 193         change = build_utils.get_change(con.OPENSTACK_GERRIT,
 194                                         patch['project'], branch,
 195                                         patch['change-id'])
 196         patch_promoted = is_patch_promoted(change,
 197                                            branch.replace('stable/', ''),
 198                                            docker_url,
 199                                            docker_img)
 200
 201         if patch_diff and not patch_promoted:
 202             patch_file = "{}.patch".format(patch['change-id'])
 203             # If we found services, then we treat the patch like it applies to
 204             # docker only
 205             if ooo_docker_services:
 206                 os_version = default_branch.replace('stable/', '')
 207                 for service in ooo_docker_services:
 208                     docker_services = docker_services.union({service})
 209                     docker_cmds = [
 210                         "WORKDIR {}".format(project_path),
 211                         "ADD {} {}".format(patch_file, project_path),
 212                         "RUN patch -p1 < {}".format(patch_file)
 213                     ]
 214                     src_img_uri = "{}:8787/tripleo{}/centos-binary-{}:" \
 215                                   "{}".format(uc_ip, os_version, service,
 216                                               docker_tag)
 217                     oc_builder.build_dockerfile(service, tmp_dir, docker_cmds,
 218                                                 src_img_uri)
 219                 patch_file_path = os.path.join(tmp_dir, 'containers',
 220                                                patch_file)
 221             else:
 222                 patch_file_path = os.path.join(tmp_dir, patch_file)
 223                 virt_ops.extend([
 224                     {con.VIRT_UPLOAD: "{}:{}".format(patch_file_path,
 225                                                      project_path)},
 226                     {con.VIRT_RUN_CMD: "cd {} && patch -p1 < {}".format(
 227                         project_path, patch_file)}])
 228                 logging.info("Adding patch {} to {}".format(patch_file,
 229                                                             image))
 230             with open(patch_file_path, 'w') as fh:
 231                 fh.write(patch_diff)
 232         else:
 233             logging.info("Ignoring patch:\n{}".format(patch))
 234     if len(virt_ops) > 1:
 235         virt_utils.virt_customize(virt_ops, image)
 236     return docker_services
 237
 238
 239 def add_repo(repo_url, repo_name, image, tmp_dir):
 240     assert repo_name is not None
 241     assert repo_url is not None
 242     repo_file = "{}.repo".format(repo_name)
 243     repo_file_path = os.path.join(tmp_dir, repo_file)
 244     content = [
 245         "[{}]".format(repo_name),
 246         "name={}".format(repo_name),
 247         "baseurl={}".format(repo_url),
 248         "gpgcheck=0"
 249     ]
 250     logging.debug("Creating repo file {}".format(repo_name))
 251     with open(repo_file_path, 'w') as fh:
 252         fh.writelines("{}\n".format(line) for line in content)
 253     logging.debug("Adding repo {} to {}".format(repo_file, image))
 254     virt_utils.virt_customize([
 255         {con.VIRT_UPLOAD: "{}:/etc/yum.repos.d/".format(repo_file_path)}],
 256         image
 257     )
 258
 259
 260 def create_git_archive(repo_url, repo_name, tmp_dir,
 261                        branch='master', prefix=''):
 262     repo = git.Repo.clone_from(repo_url, os.path.join(tmp_dir, repo_name))
 263     repo_git = repo.git
 264     if branch != str(repo.active_branch):
 265         repo_git.checkout("origin/{}".format(branch))
 266     archive_path = os.path.join(tmp_dir, "{}.tar".format(repo_name))
 267     with open(archive_path, 'wb') as fh:
 268         repo.archive(fh, prefix=prefix)
 269     logging.debug("Wrote archive file: {}".format(archive_path))
 270     return archive_path
 271
 272
 273 def get_neutron_driver(ds_opts):
 274     sdn = ds_opts.get('sdn_controller', None)
 275
 276     if sdn == 'opendaylight':
 277         return 'odl'
 278     elif sdn == 'ovn':
 279         return sdn
 280     elif ds_opts.get('vpp', False):
 281         return 'vpp'
 282     else:
 283         return None
 284
 285
 286 def prepare_container_images(prep_file, branch='master', neutron_driver=None):
 287     if not os.path.isfile(prep_file):
 288         raise exc.ApexCommonBuilderException("Prep file does not exist: "
 289                                              "{}".format(prep_file))
 290     with open(prep_file) as fh:
 291         data = yaml.safe_load(fh)
 292     try:
 293         p_set = data['parameter_defaults']['ContainerImagePrepare'][0]['set']
 294         if neutron_driver:
 295             p_set['neutron_driver'] = neutron_driver
 296         p_set['namespace'] = "docker.io/tripleo{}".format(branch)
 297         if platform.machine() == 'aarch64':
 298             p_set['namespace'] = "docker.io/armbandapex"
 299             p_set['ceph_tag'] = 'v3.1.0-stable-3.1-luminous-centos-7-aarch64'
 300
 301     except KeyError:
 302         logging.error("Invalid prep file format: {}".format(prep_file))
 303         raise exc.ApexCommonBuilderException("Invalid format for prep file")
 304
 305     logging.debug("Writing new container prep file:\n{}".format(
 306         pprint.pformat(data)))
 307     with open(prep_file, 'w') as fh:
 308         yaml.safe_dump(data, fh, default_flow_style=False)