apex/builders/common_builder.py

   1 ##############################################################################
   2 # Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 # Common building utilities for undercloud and overcloud
  11
  12 import git
  13 import json
  14 import logging
  15 import os
  16 import re
  17
  18 import apex.builders.overcloud_builder as oc_builder
  19 from apex import build_utils
  20 from apex.builders import exceptions as exc
  21 from apex.common import constants as con
  22 from apex.common import utils
  23 from apex.virtual import utils as virt_utils
  24
  25
  26 def project_to_path(project):
  27     """
  28     Translates project to absolute file path
  29     :param project: name of project
  30     :return: File path
  31     """
  32     if project.startswith('openstack/'):
  33         project = os.path.basename(project)
  34     if 'puppet' in project:
  35         return "/etc/puppet/modules/{}".format(project.replace('puppet-', ''))
  36     elif 'tripleo-heat-templates' in project:
  37         return "/usr/share/openstack-tripleo-heat-templates"
  38     else:
  39         # assume python
  40         return "/usr/lib/python2.7/site-packages/{}".format(project)
  41
  42
  43 def project_to_docker_image(project):
  44     """
  45     Translates OpenStack project to OOO services that are containerized
  46     :param project: name of OpenStack project
  47     :return: List of OOO docker service names
  48     """
  49     # Fetch all docker containers in docker hub with tripleo and filter
  50     # based on project
  51     hub_output = utils.open_webpage(con.DOCKERHUB_OOO, timeout=10)
  52     try:
  53         results = json.loads(hub_output.decode())['results']
  54     except Exception as e:
  55         logging.error("Unable to parse docker hub output for"
  56                       "tripleoupstream repository")
  57         logging.debug("HTTP response from dockerhub:\n{}".format(hub_output))
  58         raise exc.ApexCommonBuilderException(
  59             "Failed to parse docker image info from Docker Hub: {}".format(e))
  60     logging.debug("Docker Hub tripleoupstream entities found: {}".format(
  61         results))
  62     docker_images = list()
  63     for result in results:
  64         if result['name'].startswith("centos-binary-{}".format(project)):
  65             # add as docker image shortname (just service name)
  66             docker_images.append(result['name'].replace('centos-binary-', ''))
  67
  68     return docker_images
  69
  70
  71 def add_upstream_patches(patches, image, tmp_dir,
  72                          default_branch=os.path.join('stable',
  73                                                      con.DEFAULT_OS_VERSION),
  74                          uc_ip=None, docker_tag=None):
  75     """
  76     Adds patches from upstream OpenStack gerrit to Undercloud for deployment
  77     :param patches: list of patches
  78     :param image: undercloud image
  79     :param tmp_dir: to store temporary patch files
  80     :param default_branch: default branch to fetch commit (if not specified
  81     in patch)
  82     :param uc_ip: undercloud IP (required only for docker patches)
  83     :param docker_tag: Docker Tag (required only for docker patches)
  84     :return: Set of docker services patched (if applicable)
  85     """
  86     virt_ops = [{con.VIRT_INSTALL: 'patch'}]
  87     logging.debug("Evaluating upstream patches:\n{}".format(patches))
  88     docker_services = set()
  89     for patch in patches:
  90         assert isinstance(patch, dict)
  91         assert all(i in patch.keys() for i in ['project', 'change-id'])
  92         if 'branch' in patch.keys():
  93             branch = patch['branch']
  94         else:
  95             branch = default_branch
  96         patch_diff = build_utils.get_patch(patch['change-id'],
  97                                            patch['project'], branch)
  98         if patch_diff:
  99             patch_file = "{}.patch".format(patch['change-id'])
 100             project_path = project_to_path(patch['project'])
 101             # If docker tag and python we know this patch belongs on docker
 102             # container for a docker service. Therefore we build the dockerfile
 103             # and move the patch into the containers directory.  We also assume
 104             # this builder call is for overcloud, because we do not support
 105             # undercloud containers
 106             if docker_tag and 'python' in project_path:
 107                 # Projects map to multiple THT services, need to check which
 108                 # are supported
 109                 ooo_docker_services = project_to_docker_image(patch['project'])
 110             else:
 111                 ooo_docker_services = []
 112             # If we found services, then we treat the patch like it applies to
 113             # docker only
 114             if ooo_docker_services:
 115                 os_version = default_branch.replace('stable/', '')
 116                 for service in ooo_docker_services:
 117                     docker_services = docker_services.union({service})
 118                     docker_cmds = [
 119                         "WORKDIR {}".format(project_path),
 120                         "ADD {} {}".format(patch_file, project_path),
 121                         "RUN patch -p1 < {}".format(patch_file)
 122                     ]
 123                     src_img_uri = "{}:8787/{}/centos-binary-{}:" \
 124                                   "{}".format(uc_ip, os_version, service,
 125                                               docker_tag)
 126                     oc_builder.build_dockerfile(service, tmp_dir, docker_cmds,
 127                                                 src_img_uri)
 128                 patch_file_path = os.path.join(tmp_dir, 'containers',
 129                                                patch_file)
 130             else:
 131                 patch_file_path = os.path.join(tmp_dir, patch_file)
 132                 virt_ops.extend([
 133                     {con.VIRT_UPLOAD: "{}:{}".format(patch_file_path,
 134                                                      project_path)},
 135                     {con.VIRT_RUN_CMD: "cd {} && patch -p1 < {}".format(
 136                         project_path, patch_file)}])
 137                 logging.info("Adding patch {} to {}".format(patch_file,
 138                                                             image))
 139             with open(patch_file_path, 'w') as fh:
 140                 fh.write(patch_diff)
 141         else:
 142             logging.info("Ignoring patch:\n{}".format(patch))
 143     if len(virt_ops) > 1:
 144         virt_utils.virt_customize(virt_ops, image)
 145     return docker_services
 146
 147
 148 def add_repo(repo_url, repo_name, image, tmp_dir):
 149     assert repo_name is not None
 150     assert repo_url is not None
 151     repo_file = "{}.repo".format(repo_name)
 152     repo_file_path = os.path.join(tmp_dir, repo_file)
 153     content = [
 154         "[{}]".format(repo_name),
 155         "name={}".format(repo_name),
 156         "baseurl={}".format(repo_url),
 157         "gpgcheck=0"
 158     ]
 159     logging.debug("Creating repo file {}".format(repo_name))
 160     with open(repo_file_path, 'w') as fh:
 161         fh.writelines("{}\n".format(line) for line in content)
 162     logging.debug("Adding repo {} to {}".format(repo_file, image))
 163     virt_utils.virt_customize([
 164         {con.VIRT_UPLOAD: "{}:/etc/yum.repos.d/".format(repo_file_path)}],
 165         image
 166     )
 167
 168
 169 def create_git_archive(repo_url, repo_name, tmp_dir,
 170                        branch='master', prefix=''):
 171     repo = git.Repo.clone_from(repo_url, os.path.join(tmp_dir, repo_name))
 172     repo_git = repo.git
 173     if branch != str(repo.active_branch):
 174         repo_git.checkout("origin/{}".format(branch))
 175     archive_path = os.path.join(tmp_dir, "{}.tar".format(repo_name))
 176     with open(archive_path, 'wb') as fh:
 177         repo.archive(fh, prefix=prefix)
 178     logging.debug("Wrote archive file: {}".format(archive_path))
 179     return archive_path