From f39b860650068f625c91ef94979a1f6257f32039 Mon Sep 17 00:00:00 2001 From: Trevor Bramwell Date: Fri, 27 Apr 2018 14:00:16 -0700 Subject: [PATCH 1/1] Verify and Create Tags from Release File Updates the releases verify and merge jobs to check release files for the following: - commits exist - commits are on stable-branch If both of these criteria are met and the patch is merged, the repos will be tagged at the specified commits and pushed. If a repo has already been tagged but not on stable-branch the job will not exit, but will warn that the commit doesn't exist on the correct branch. This is because tags should never be removed. This adds an additional script 'repos.py' that provides an quick interface for pulling out information from release files, along with the script 'release-status.sh' for checking the status of tags. NOTE: The branch creation script has been disabled until it can be reworked. Change-Id: I498bc74f20aa50d2bd321771f20a77905b246399 Signed-off-by: Trevor Bramwell --- ...ase-verify.sh => releng-release-create-venv.sh} | 23 +++---- jjb/releng/releng-release-jobs.yaml | 32 +++++----- jjb/releng/releng-release-tagging.sh | 70 ++++++++++++++++++++++ releases/fraser/{armband.yml => armband.yaml} | 0 releases/scripts/release-status.sh | 25 ++++++++ releases/scripts/repos.py | 60 +++++++++++++++++++ 6 files changed, 178 insertions(+), 32 deletions(-) rename jjb/releng/{releng-release-verify.sh => releng-release-create-venv.sh} (51%) create mode 100644 jjb/releng/releng-release-tagging.sh rename releases/fraser/{armband.yml => armband.yaml} (100%) create mode 100644 releases/scripts/release-status.sh create mode 100644 releases/scripts/repos.py diff --git a/jjb/releng/releng-release-verify.sh b/jjb/releng/releng-release-create-venv.sh similarity index 51% rename from jjb/releng/releng-release-verify.sh rename to jjb/releng/releng-release-create-venv.sh index c1262e2c9..0d5635b59 100644 --- a/jjb/releng/releng-release-verify.sh +++ b/jjb/releng/releng-release-create-venv.sh @@ -7,21 +7,12 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -set -xe +set -e -o pipefail +echo "---> Create virtualenv" -# Activate virtualenv, supressing shellcheck warning -# shellcheck source=/dev/null -. $WORKSPACE/venv/bin/activate +sudo pip install virtualenv +virtualenv $WORKSPACE/venv +# shellcheck source=$WORKSPACE/venv/bin/activate disable=SC1091 +source $WORKSPACE/venv/bin/activate +pip install --upgrade pip pip install -r releases/scripts/requirements.txt - -STREAM=${STREAM:-'nostream'} -RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM") - -# TODO: The create_branch.py should be refactored so it can be used here -# to verify the commit exists that is being added, along with -# jjb/ -for release_file in $RELEASE_FILES; do - python releases/scripts/verify_schema.py \ - -s releases/schema.yaml \ - -y $release_file -done diff --git a/jjb/releng/releng-release-jobs.yaml b/jjb/releng/releng-release-jobs.yaml index e31320b81..0c059ee9b 100644 --- a/jjb/releng/releng-release-jobs.yaml +++ b/jjb/releng/releng-release-jobs.yaml @@ -10,6 +10,8 @@ - project: name: releng-release-jobs + build-node: 'opnfv-build' + stream: - fraser @@ -23,6 +25,7 @@ name: 'releng-release-{stream}-verify' parameters: + - '{build-node}-defaults' - stream-parameter: stream: '{stream}' - project-parameter: @@ -59,9 +62,9 @@ pattern: 'releases/scripts/verify_schema.py' builders: - - create-virtualenv - - shell: - !include-raw-escape: releng-release-verify.sh + - shell: !include-raw-escape: + - releng-release-create-venv.sh + - releng-release-tagging.sh publishers: - email-jenkins-admins-on-failure @@ -82,6 +85,12 @@ - project-parameter: project: '{project}' branch: 'master' + # Override GIT_BASE so we can send patches back to Gerrit and + # modify repos + - string: + name: GIT_BASE + default: ssh://$USER@gerrit.opnfv.org:29418/$PROJECT + description: 'Git URL to use on this Jenkins Slave' scm: - git-scm-gerrit @@ -93,9 +102,10 @@ files: 'releases/{stream}/*' builders: - - create-virtualenv - - shell: - !include-raw-escape: releng-release-create-branch.sh + - shell: !include-raw-escape: + - releng-release-create-venv.sh + - releng-release-tagging.sh + # - releng-release-create-branch.sh publishers: - email-jenkins-admins-on-failure @@ -107,13 +117,3 @@ name: STREAM default: '{stream}' description: "OPNFV Stable Stream" - -- builder: - name: create-virtualenv - builders: - - shell: | - #!/bin/bash - sudo pip install virtualenv - virtualenv $WORKSPACE/venv - . $WORKSPACE/venv/bin/activate - pip install --upgrade pip diff --git a/jjb/releng/releng-release-tagging.sh b/jjb/releng/releng-release-tagging.sh new file mode 100644 index 000000000..10c0cc8c9 --- /dev/null +++ b/jjb/releng/releng-release-tagging.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -e -o pipefail + +GIT_URL=${GIT_URL:-https://gerrit.opnfv.org/gerrit} +STREAM=${STREAM:-'nostream'} +RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM") + +echo "--> Verifying $RELEASE_FILES." +for release_file in $RELEASE_FILES; do + # Verify the release file schema + python releases/scripts/verify_schema.py \ + -s releases/schema.yaml \ + -y $release_file + + # Verify tag for each repo exist and are attached to commits on stable-branch + while read -r repo tag ref + do + echo "--> Cloning $repo" + if [ ! -d $repo ]; then + git clone $GIT_URL/$repo.git $repo + fi + pushd $repo &> /dev/null + + echo "--> Checking for tag: $tag" + if ! (git tag -l | grep $tag &> /dev/null); then + echo "$tag does not exist" + TAG_EXISTS=false + else + git cat-file tag $tag + TAG_EXISTS=true + fi + + echo "--> Checking if $ref is on stable/$STREAM" + if ! (git branch -a --contains $ref | grep "stable/$STREAM"); then + echo "--> ERROR: $ref for $repo is not on stable/$STREAM!" + # If the tag exists but is on the wrong ref, there's nothing + # we can do. But if the tag neither exists nor is on the + # correct branch we need to fail the verification. + if [ $TAG_EXISTS = false ]; then + exit 1 + fi + else + if [[ $TAG_EXISTS = false && "$JOB_NAME" =~ "merge" ]]; then + # If the tag doesn't exist and we're in a merge job, + # everything has been verified up to this point and we + # are ready to create the tag. + git config --global user.name "jenkins-ci" + git config --global user.email "jenkins-opnfv-ci@opnfv.org" + echo "--> Creating $tag tag for $repo at $ref" + git tag -am "$tag" $tag $ref + echo "--> Pushing tag" + echo "[noop] git push origin $tag" + else + # For non-merge jobs just output the ref info. + git show -s --format="%h %s %d" $ref + fi + fi + + popd &> /dev/null + echo "--> Done verifing $repo" + done < <(python releases/scripts/repos.py -f $release_file) +done diff --git a/releases/fraser/armband.yml b/releases/fraser/armband.yaml similarity index 100% rename from releases/fraser/armband.yml rename to releases/fraser/armband.yaml diff --git a/releases/scripts/release-status.sh b/releases/scripts/release-status.sh new file mode 100644 index 000000000..da66bc936 --- /dev/null +++ b/releases/scripts/release-status.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o pipefail + +TAG="${TAG:-opnfv-6.0.0}" +RELEASE="${RELEASE:-fraser}" + +for project in releases/$RELEASE/*; do + python releases/scripts/repos.py -n -f $project >> repos.txt +done + +while read -r repo +do + tag="$(git ls-remote "https://gerrit.opnfv.org/gerrit/$repo.git" "refs/tags/$TAG")" + echo "$repo $tag" +done < repos.txt + +# rm repos.txt diff --git a/releases/scripts/repos.py b/releases/scripts/repos.py new file mode 100644 index 000000000..0ded0207b --- /dev/null +++ b/releases/scripts/repos.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python2 +# SPDX-License-Identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +""" +List Release Repos +""" + +import argparse +import yaml + + +def main(): + """Given a release yamlfile list the repos it contains""" + + parser = argparse.ArgumentParser() + parser.add_argument('--file', '-f', + type=argparse.FileType('r'), + required=True) + parser.add_argument('--names', '-n', + action='store_true', + default=False, + help="Only print the names of repos, " + "not their SHAs") + parser.add_argument('--release', '-r', + type=str, + help="Only print" + "SHAs for the specified release") + args = parser.parse_args() + + project = yaml.safe_load(args.file) + + list_repos(project, args) + + +def list_repos(project, args): + """List repositories in the project file""" + + lookup = project.get('releases', []) + if 'releases' not in project: + exit(0) + + for item in lookup: + repo, ref = next(iter(item['location'].items())) + if args.names: + print(repo) + elif args.release and item['version'] == args.release: + print("%s %s" % (repo, ref)) + elif not args.release: + # Print all releases + print("%s %s %s" % (repo, item['version'], ref)) + + +if __name__ == "__main__": + main() -- 2.16.6