From: Trevor Bramwell Date: Tue, 26 Jun 2018 12:45:16 +0000 (+0000) Subject: Merge "Fix release tagging noop" X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=commitdiff_plain;h=ea6bc61f16d7fcbf1b6d33269d690fa349722d61;hp=989134f3cc15c0bc060686475bf3a7fd14453414;p=releng.git Merge "Fix release tagging noop" --- diff --git a/jjb/3rd_party_ci/download-netvirt-artifact.sh b/jjb/3rd_party_ci/download-netvirt-artifact.sh index 7ecf8d78d..5a50e8a93 100755 --- a/jjb/3rd_party_ci/download-netvirt-artifact.sh +++ b/jjb/3rd_party_ci/download-netvirt-artifact.sh @@ -10,7 +10,7 @@ if [ "$ODL_BRANCH" != 'master' ]; then DIST=$(echo ${ODL_BRANCH} | sed -rn 's#([a-zA-Z]+)/([a-zA-Z]+)#\2#p') ODL_BRANCH=$(echo ${ODL_BRANCH} | sed -rn 's#([a-zA-Z]+)/([a-zA-Z]+)#\1%2F\2#p') else - DIST='nitrogen' + DIST='flourine' fi CHANGE_DETAILS_URL="https://git.opendaylight.org/gerrit/changes/netvirt~${ODL_BRANCH}~${GERRIT_CHANGE_ID}/detail" # due to limitation with the Jenkins Gerrit Trigger, we need to use Gerrit REST API to get the change details diff --git a/jjb/3rd_party_ci/odl-netvirt.yaml b/jjb/3rd_party_ci/odl-netvirt.yaml index 863eb94dd..c077fce2c 100644 --- a/jjb/3rd_party_ci/odl-netvirt.yaml +++ b/jjb/3rd_party_ci/odl-netvirt.yaml @@ -13,8 +13,8 @@ branch: '{stream}' gs-pathname: '' disabled: false - - carbon: - branch: 'stable/carbon' + - oxygen: + branch: 'stable/oxygen' gs-pathname: '' disabled: false ##################################### @@ -22,11 +22,11 @@ ##################################### phase: - 'create-apex-vms': - slave-label: 'odl-netvirt-virtual-intel' + slave-label: 'apex-virtual-master' - 'install-netvirt': - slave-label: 'odl-netvirt-virtual-intel' + slave-label: 'apex-virtual-master' - 'postprocess': - slave-label: 'odl-netvirt-virtual-intel' + slave-label: 'apex-virtual-master' ##################################### # jobs ##################################### @@ -53,7 +53,17 @@ max-total: 5 max-per-node: 1 option: 'project' - + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'apex-verify.*' + - 'apex-.*-promote.*' + - 'apex-virtual.*' + - 'odl-netvirt-verify-virtual-create-apex-vms-.*' + - 'odl-netvirt-verify-virtual-install-netvirt-.*' + - 'functest-netvirt-virtual-suite-.*' + - 'odl-netvirt-verify-virtual-postprocess-.*' + block-level: 'NODE' scm: - git: url: https://gerrit.opnfv.org/gerrit/apex @@ -69,7 +79,7 @@ - string: name: NETVIRT_ARTIFACT default: distribution-karaf.tar.gz - - 'odl-netvirt-virtual-intel-defaults' + - 'apex-virtual-master-defaults' triggers: - gerrit: @@ -136,7 +146,7 @@ projects: - name: 'functest-netvirt-virtual-suite-master' predefined-parameters: | - DEPLOY_SCENARIO=os-odl_l3-nofeature-ha + DEPLOY_SCENARIO=os-odl-nofeature-ha FUNCTEST_MODE=testcase FUNCTEST_SUITE_NAME=odl_netvirt RC_FILE_PATH=$HOME/cloner-info/overcloudrc @@ -206,7 +216,7 @@ - '{installer}-defaults' - string: name: DEPLOY_SCENARIO - default: 'os-odl_l2-bgpvpn-noha' + default: 'os-odl-nofeature-noha' description: 'Scenario to deploy and test' - string: name: GS_URL diff --git a/jjb/apex/apex-snapshot-create.sh b/jjb/apex/apex-snapshot-create.sh index 7e38d56b4..b73931470 100644 --- a/jjb/apex/apex-snapshot-create.sh +++ b/jjb/apex/apex-snapshot-create.sh @@ -91,10 +91,16 @@ echo "Snapshot saved as apex-${SNAP_TYPE}-snap-${DATE}.tar.gz" # update opnfv properties file if [ "$SNAP_TYPE" == 'csit' ]; then - curl -O -L http://$GS_URL/snapshot.properties - sed -i '/^OPNFV_SNAP_URL=/{h;s#=.*#='${GS_URL}'/apex-csit-snap-'${DATE}'.tar.gz#};${x;/^$/{s##OPNFV_SNAP_URL='${GS_URL}'/apex-csit-snap-'${DATE}'.tar.gz#;H};x}' snapshot.properties snap_sha=$(sha512sum apex-csit-snap-${DATE}.tar.gz | cut -d' ' -f1) - sed -i '/^OPNFV_SNAP_SHA512SUM=/{h;s/=.*/='${snap_sha}'/};${x;/^$/{s//OPNFV_SNAP_SHA512SUM='${snap_sha}'/;H};x}' snapshot.properties + if curl --fail -O -L http://$GS_URL/snapshot.properties; then + sed -i '/^OPNFV_SNAP_URL=/{h;s#=.*#='${GS_URL}'/apex-csit-snap-'${DATE}'.tar.gz#};${x;/^$/{s##OPNFV_SNAP_URL='${GS_URL}'/apex-csit-snap-'${DATE}'.tar.gz#;H};x}' snapshot.properties + sed -i '/^OPNFV_SNAP_SHA512SUM=/{h;s/=.*/='${snap_sha}'/};${x;/^$/{s//OPNFV_SNAP_SHA512SUM='${snap_sha}'/;H};x}' snapshot.properties + else + cat << EOF > snapshot.properties +OPNFV_SNAP_URL=${GS_URL}/apex-csit-snap-${DATE}.tar.gz +OPNFV_SNAP_SHA512SUM=${snap_sha} +EOF + fi echo "OPNFV_SNAP_URL=$GS_URL/apex-csit-snap-${DATE}.tar.gz" echo "OPNFV_SNAP_SHA512SUM=$(sha512sum apex-csit-snap-${DATE}.tar.gz | cut -d' ' -f1)" echo "Updated properties file: " diff --git a/jjb/apex/apex-snapshot-deploy.sh b/jjb/apex/apex-snapshot-deploy.sh index a93421c8e..1810a0036 100644 --- a/jjb/apex/apex-snapshot-deploy.sh +++ b/jjb/apex/apex-snapshot-deploy.sh @@ -141,14 +141,15 @@ done mkdir -p $HOME/cloner-info cp -f overcloudrc $HOME/cloner-info/ -admin_controller_ip=$(cat overcloudrc | grep -Eo -m 1 "192.0.2.[0-9]+") +admin_controller_ip=$(cat overcloudrc | grep -Eo -m 1 "192.0.2.[0-9]+" | head -1) netvirt_url="http://${admin_controller_ip}:8081/restconf/operational/network-topology:network-topology/topology/netvirt:1" source overcloudrc counter=1 while [ "$counter" -le 10 ]; do - if curl --fail --silent ${admin_controller_ip}:80 > /dev/null; then - echo "Overcloud Horizon is up...Checking if OpenDaylight NetVirt is up..." + echo "Checking if OpenStack is up" + if nc -z ${admin_controller_ip} 9696 > /dev/null; then + echo "Overcloud Neutron is up...Checking if OpenDaylight NetVirt is up..." if curl --fail --silent -u admin:admin ${netvirt_url} > /dev/null; then echo "OpenDaylight is up. Overcloud deployment complete" exit 0 @@ -156,7 +157,7 @@ while [ "$counter" -le 10 ]; do echo "OpenDaylight not yet up, try ${counter}" fi else - echo "Horizon/Apache not yet up, try ${counter}" + echo "Neutron not yet up, try ${counter}" fi counter=$((counter+1)) sleep 60 diff --git a/jjb/apex/apex-upload-artifact.sh b/jjb/apex/apex-upload-artifact.sh index 8743368fb..5c777a824 100755 --- a/jjb/apex/apex-upload-artifact.sh +++ b/jjb/apex/apex-upload-artifact.sh @@ -24,7 +24,9 @@ importkey () { git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null #this is where we import the siging key if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then - source $WORKSPACE/releng/utils/gpg_import_key.sh + if ! $WORKSPACE/releng/utils/gpg_import_key.sh; then + echo "WARNING: Failed to run gpg key import" + fi fi } @@ -75,13 +77,16 @@ uploadrpm () { uploadsnap () { # Uploads snapshot artifact and updated properties file echo "Uploading snapshot artifacts" + # snapshot dir is the same node in the create job workspace + # only 1 promotion job can run at a time on a slave + snapshot_dir="${WORKSPACE}/../apex-create-snapshot" if [ -z "$SNAP_TYPE" ]; then echo "ERROR: SNAP_TYPE not provided...exiting" exit 1 fi - gsutil cp $WORKSPACE/apex-${SNAP_TYPE}-snap-`date +%Y-%m-%d`.tar.gz gs://$GS_URL/ > gsutil.iso.log + gsutil cp ${snapshot_dir}/apex-${SNAP_TYPE}-snap-`date +%Y-%m-%d`.tar.gz gs://$GS_URL/ > gsutil.iso.log if [ "$SNAP_TYPE" == 'csit' ]; then - gsutil cp $WORKSPACE/snapshot.properties gs://$GS_URL/snapshot.properties > gsutil.latest.log + gsutil cp ${snapshot_dir}/snapshot.properties gs://$GS_URL/snapshot.properties > gsutil.latest.log fi echo "Upload complete for Snapshot" } diff --git a/jjb/apex/apex-verify-jobs.yaml b/jjb/apex/apex-verify-jobs.yaml index 0837a76e4..81e59bdf5 100644 --- a/jjb/apex/apex-verify-jobs.yaml +++ b/jjb/apex/apex-verify-jobs.yaml @@ -312,6 +312,7 @@ - 'apex-verify.*' - 'apex-virtual.*' - 'apex-.*-promote.*' + - 'odl-netvirt.*' - throttle: max-per-node: 1 max-total: 10 diff --git a/jjb/apex/apex.yaml b/jjb/apex/apex.yaml index 9ad72e404..e24a2c358 100644 --- a/jjb/apex/apex.yaml +++ b/jjb/apex/apex.yaml @@ -8,7 +8,7 @@ - 'apex-virtual-{stream}' - 'apex-deploy-{platform}-{stream}' - 'apex-daily-{stream}' - - 'apex-csit-promote-daily-{stream}-{os_version}' + - 'apex-csit-promote-daily-{stream}-os-{os_version}' - 'apex-fdio-promote-daily-{stream}' - 'apex-{scenario}-baremetal-{scenario_stream}' - 'apex-testsuite-{scenario}-baremetal-{scenario_stream}' @@ -347,6 +347,7 @@ - 'yardstick.*' - 'dovetail.*' - 'storperf.*' + - 'odl-netvirt.*' - throttle: max-per-node: 1 max-total: 10 @@ -401,6 +402,7 @@ - 'apex-run.*' - 'apex-virtual-.*' - 'apex-verify-gate-.*' + - 'odl-netvirt.*' - throttle: max-per-node: 1 max-total: 10 @@ -752,13 +754,11 @@ - job-template: name: 'apex-create-snapshot' - # Job template for clean - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - disabled: false + parameters: + - '{project}-defaults' + builders: - shell: !include-raw-escape: ./apex-snapshot-create.sh @@ -767,13 +767,11 @@ - job-template: name: 'apex-upload-snapshot' - # Job template for clean - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - disabled: false + parameters: + - '{project}-defaults' + builders: - inject: properties-content: ARTIFACT_TYPE=snapshot @@ -781,7 +779,7 @@ # CSIT promote - job-template: - name: 'apex-csit-promote-daily-{stream}-{os_version}' + name: 'apex-csit-promote-daily-{stream}-os-{os_version}' # Job template for promoting CSIT Snapshots # @@ -847,7 +845,7 @@ condition: SUCCESSFUL projects: - name: 'apex-create-snapshot' - current-parameters: false + current-parameters: true predefined-parameters: | SNAP_TYPE=csit node-parameters: true @@ -859,7 +857,7 @@ condition: SUCCESSFUL projects: - name: 'apex-upload-snapshot' - current-parameters: false + current-parameters: true predefined-parameters: | SNAP_TYPE=csit node-parameters: true diff --git a/jjb/apex/apex.yaml.j2 b/jjb/apex/apex.yaml.j2 index 29ddef9c9..1f5232b5f 100644 --- a/jjb/apex/apex.yaml.j2 +++ b/jjb/apex/apex.yaml.j2 @@ -8,7 +8,7 @@ - 'apex-virtual-{stream}' - 'apex-deploy-{platform}-{stream}' - 'apex-daily-{stream}' - - 'apex-csit-promote-daily-{stream}-{os_version}' + - 'apex-csit-promote-daily-{stream}-os-{os_version}' - 'apex-fdio-promote-daily-{stream}' - 'apex-{scenario}-baremetal-{scenario_stream}' - 'apex-testsuite-{scenario}-baremetal-{scenario_stream}' @@ -219,6 +219,7 @@ - 'yardstick.*' - 'dovetail.*' - 'storperf.*' + - 'odl-netvirt.*' - throttle: max-per-node: 1 max-total: 10 @@ -273,6 +274,7 @@ - 'apex-run.*' - 'apex-virtual-.*' - 'apex-verify-gate-.*' + - 'odl-netvirt.*' - throttle: max-per-node: 1 max-total: 10 @@ -624,13 +626,11 @@ - job-template: name: 'apex-create-snapshot' - # Job template for clean - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - disabled: false + parameters: + - '{project}-defaults' + builders: - shell: !include-raw-escape: ./apex-snapshot-create.sh @@ -639,13 +639,11 @@ - job-template: name: 'apex-upload-snapshot' - # Job template for clean - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - disabled: false + parameters: + - '{project}-defaults' + builders: - inject: properties-content: ARTIFACT_TYPE=snapshot @@ -653,7 +651,7 @@ # CSIT promote - job-template: - name: 'apex-csit-promote-daily-{stream}-{os_version}' + name: 'apex-csit-promote-daily-{stream}-os-{os_version}' # Job template for promoting CSIT Snapshots # @@ -719,7 +717,7 @@ condition: SUCCESSFUL projects: - name: 'apex-create-snapshot' - current-parameters: false + current-parameters: true predefined-parameters: | SNAP_TYPE=csit node-parameters: true @@ -731,7 +729,7 @@ condition: SUCCESSFUL projects: - name: 'apex-upload-snapshot' - current-parameters: false + current-parameters: true predefined-parameters: | SNAP_TYPE=csit node-parameters: true diff --git a/jjb/releng/releng-ci-jobs.yaml b/jjb/releng/releng-ci-jobs.yaml index f9a2bc232..790bd7eb4 100644 --- a/jjb/releng/releng-ci-jobs.yaml +++ b/jjb/releng/releng-ci-jobs.yaml @@ -11,5 +11,9 @@ jjb-version: '2.0.3' lftools-version: '<1.0.0' + rtd-build-url: 'https://readthedocs.org/api/v2/webhook/opnfv-releng/38594/' + rtd-token: '291c6a0109493b4457e566d06141212452c65784' + jobs: - '{project-name}-ci-jobs' + - '{project-name}-rtd-jobs' diff --git a/jjb/sfc/sfc-project-jobs.yaml b/jjb/sfc/sfc-project-jobs.yaml index fea792467..70fc332b8 100644 --- a/jjb/sfc/sfc-project-jobs.yaml +++ b/jjb/sfc/sfc-project-jobs.yaml @@ -60,14 +60,43 @@ pattern: 'docs/**|.gitignore' builders: - - sfc-unit-tests + - sfc-unit-tests-and-docs + + publishers: + - sfc-unit-tests-and-docs-publisher ################################ # job builders ################################ - builder: - name: sfc-unit-tests + name: sfc-unit-tests-and-docs builders: - shell: | - cd $WORKSPACE && yamllint $(git ls-tree -r HEAD --name-only | egrep 'yml$|yaml$' | egrep -v 'scenarios') + cd $WORKSPACE && tox + +################################ +# job publishers +################################ +- publisher: + name: 'sfc-unit-tests-and-docs-publisher' + publishers: + - junit: + results: nosetests.xml + - cobertura: + report-file: "coverage.xml" + only-stable: "true" + fail-no-reports: "true" + health-auto-update: "true" + stability-auto-update: "true" + zoom-coverage-chart: "true" + targets: + - files: + healthy: 0 + unhealthy: 0 + failing: 0 + - method: + healthy: 0 + unhealthy: 0 + failing: 0 + - email-jenkins-admins-on-failure diff --git a/releases/scripts/release-status.sh b/releases/scripts/release-status.sh old mode 100644 new mode 100755 index da66bc936..6790100a7 --- a/releases/scripts/release-status.sh +++ b/releases/scripts/release-status.sh @@ -12,6 +12,8 @@ set -o pipefail TAG="${TAG:-opnfv-6.0.0}" RELEASE="${RELEASE:-fraser}" +[ -a repos.txt ] && rm repos.txt + for project in releases/$RELEASE/*; do python releases/scripts/repos.py -n -f $project >> repos.txt done @@ -21,5 +23,3 @@ do tag="$(git ls-remote "https://gerrit.opnfv.org/gerrit/$repo.git" "refs/tags/$TAG")" echo "$repo $tag" done < repos.txt - -# rm repos.txt diff --git a/releases/scripts/repos.py b/releases/scripts/repos.py index 0ded0207b..58454c318 100644 --- a/releases/scripts/repos.py +++ b/releases/scripts/repos.py @@ -15,6 +15,38 @@ import argparse import yaml +class Repo(object): + """Object representing a repo listed in the release file. + + Includes eq, hash, and ne methods so set comparisons work + """ + + def __init__(self, repo=None, ref=None, version=None): + self.repo = repo + self.ref = ref + self.version = version + + def __repr__(self): + if self.version: + return "%s %s %s" % (self.repo, self.ref, self.version) + elif self.ref: + return "%s %s" % (self.repo, self.ref) + return "%s" % self.repo + + def __eq__(self, obj): + if isinstance(obj, Repo): + return ((self.repo == obj.repo) and + (self.ref == obj.ref) and + (self.version == obj.version)) + return False + + def __ne__(self, obj): + return (not self.__eq__(obj)) + + def __hash__(self): + return hash(self.__repr__()) + + def main(): """Given a release yamlfile list the repos it contains""" @@ -45,15 +77,17 @@ def list_repos(project, args): if 'releases' not in project: exit(0) + repos = set() for item in lookup: repo, ref = next(iter(item['location'].items())) if args.names: - print(repo) + repos.add(Repo(repo)) elif args.release and item['version'] == args.release: - print("%s %s" % (repo, ref)) + repos.add(Repo(repo, ref)) elif not args.release: - # Print all releases - print("%s %s %s" % (repo, item['version'], ref)) + repos.add(Repo(repo, ref, item['version'])) + for repo in repos: + print repo if __name__ == "__main__": diff --git a/utils/build-server-ansible/main.yml b/utils/build-server-ansible/main.yml index 0fcce715d..c9f244bee 100644 --- a/utils/build-server-ansible/main.yml +++ b/utils/build-server-ansible/main.yml @@ -31,6 +31,10 @@ pip: name: tox state: present + - name: install yamllint + pip: + name: yamllint + state: present - include: vars/docker-compose-CentOS.yml when: ansible_distribution == "CentOS" - include: vars/docker-compose-Ubuntu.yml diff --git a/utils/build-server-ansible/vars/CentOS.yml b/utils/build-server-ansible/vars/CentOS.yml index 528388f27..7a2c1d1d5 100644 --- a/utils/build-server-ansible/vars/CentOS.yml +++ b/utils/build-server-ansible/vars/CentOS.yml @@ -46,6 +46,7 @@ - sysstat - xmlstarlet - facter + - jq - name: install the 'Development tools' package group yum: diff --git a/utils/build-server-ansible/vars/Ubuntu.yml b/utils/build-server-ansible/vars/Ubuntu.yml index e91c48569..f50255868 100644 --- a/utils/build-server-ansible/vars/Ubuntu.yml +++ b/utils/build-server-ansible/vars/Ubuntu.yml @@ -59,6 +59,7 @@ - sysstat - xmlstarlet - facter + - jq - name: Add Docker apt key. apt_key: diff --git a/utils/fetch_os_creds.sh b/utils/fetch_os_creds.sh index 4bc585885..b40b75b75 100755 --- a/utils/fetch_os_creds.sh +++ b/utils/fetch_os_creds.sh @@ -164,7 +164,13 @@ elif [ "$installer_type" == "apex" ]; then if [ -f /root/.ssh/id_rsa ]; then chmod 600 /root/.ssh/id_rsa fi - sudo scp $ssh_options root@$installer_ip:/home/stack/overcloudrc.v3 $dest_path + + if [ "${BRANCH}" == "stable/fraser" ]; then + rc_file=overcloudrc.v3 + else + rc_file=overcloudrc + fi + sudo scp $ssh_options root@$installer_ip:/home/stack/${rc_file} $dest_path elif [ "$installer_type" == "compass" ]; then if [ "${BRANCH}" == "stable/danube" ]; then diff --git a/utils/gpg_import_key.sh b/utils/gpg_import_key.sh index f45e40919..94ffbaab5 100755 --- a/utils/gpg_import_key.sh +++ b/utils/gpg_import_key.sh @@ -51,10 +51,14 @@ else if [ -z "$NODE_NAME" ]; then echo "Cannot find node name" exit 0 - else echo "Importing key for '$NODE_NAME'"; - gsutil cp gs://opnfv-signing-keys/"$NODE_NAME"-subkey . - gpg2 --import "$NODE_NAME"-subkey - rm -f "$NODE_NAME"-subkey - fi + elif gsutil ls gs://opnfv-signing-keys | grep $NODE_NAME; then + echo "Importing key for '$NODE_NAME'" + gsutil cp gs://opnfv-signing-keys/"$NODE_NAME"-subkey . + gpg2 --import "$NODE_NAME"-subkey + rm -f "$NODE_NAME"-subkey + else + echo "No keys found locally or remotely for host, skipping import" + exit 0 + fi fi