From 4d7414428a945c602bc01067288b72e3b2ec34e4 Mon Sep 17 00:00:00 2001 From: agardner Date: Wed, 21 Feb 2018 15:24:10 -0500 Subject: [PATCH] Move generation of artifacts.opnfv.org to jenkins Move generation of artifacts.opnfv.org/index.html and sub pages into a jenkins job. Script was run on cron on master previously. run on lf-build2 Run job once per hour (it takes a long time to run) Block if job is already running Change-Id: Icc03cac61a802e036e4af8a9b9e3a244e147abea Signed-off-by: agardner --- jjb/releng/opnfv-utils.yml | 38 ++++++++++ utils/artifacts.opnfv.org.sh | 162 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100755 utils/artifacts.opnfv.org.sh diff --git a/jjb/releng/opnfv-utils.yml b/jjb/releng/opnfv-utils.yml index b12f663aa..93b3615ef 100644 --- a/jjb/releng/opnfv-utils.yml +++ b/jjb/releng/opnfv-utils.yml @@ -8,6 +8,7 @@ - 'archive-repositories' - 'check-status-of-slaves' - 'ansible-build-server' + - 'generate-artifacts-index-pages' ######################## # job templates @@ -164,3 +165,40 @@ sudo ansible-playbook -C -D -i \ $WORKSPACE/utils/build-server-ansible/inventory.ini \ $WORKSPACE/utils/build-server-ansible/main.yml + + +- job-template: + name: 'generate-artifacts-index-pages' + + project-type: freestyle + + disabled: false + + concurrent: false + + parameters: + - node: + name: SLAVE_NAME + description: Build Servers + default-slaves: + - lf-build2 + - project-parameter: + project: releng + branch: master + + scm: + - git-scm + + triggers: + - timed: '@hourly' + + builders: + - generate-artifact-html + + +- builder: + name: generate-artifact-html + builders: + - shell: | + cd $WORKSPACE/utils/ + ./artifacts.opnfv.org.sh diff --git a/utils/artifacts.opnfv.org.sh b/utils/artifacts.opnfv.org.sh new file mode 100755 index 000000000..1984b4992 --- /dev/null +++ b/utils/artifacts.opnfv.org.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +#export PATH=${PATH}:/root/gsutil + +#Step Generate index.html +if [ -f index.html ] ; then + rm -f index.html +fi + +OUTPUT="index.html" + +for index in $(gsutil ls -l gs://artifacts.opnfv.org | grep -v logs | grep -v review | awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,,) +do +echo $index + echo "
  • "$index"
    • " >> $OUTPUT +done + +#functest logs########################## + +for project in functest vswitchperf +do + + for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, ) + do + index="$(echo ${index%/*} | sed s,/,_,g)" + echo "
  • "$index"
    • " >> $OUTPUT + done + +done +#End step 1 +##################################### + + +#genrate html files for all project except vswitchperf +for index in $(gsutil ls -l gs://artifacts.opnfv.org | grep -v logs |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,,) +do +OUTPUT=${index%/*}.html +rm -f $OUTPUT + + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "" >> $OUTPUT + else + echo "
  • "$filepath"
    • " >> $OUTPUT + fi +done + +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + +done + + + +#generate file for vswitch perf (I dont know what happend here but there is a wierd character in this bucket) + +index=vswitchperf +OUTPUT=${index%/*}.html +rm -f $OUTPUT + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "" >> $OUTPUT + else + echo "
  • "$filepath"
    • " >> $OUTPUT + fi + +done + + +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + +# Gerate html for logs + +for project in functest vswitchperf +do + for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, ) + do + + OUTPUT="$(echo ${index%/*}.html | sed s,/,_,g)" + echo $OUTPUT + rm -f $OUTPUT + + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "" >> $OUTPUT + else + echo "
  • "$filepath"
    • " >> $OUTPUT + fi + + + done + + + gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + + gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + + + done +done + + + +OUTPUT="index.html" +echo "

    Generated on $(date)

    " >> $OUTPUT + +cat <> $OUTPUT + +EOF + +#copy and uplad index file genrated in first step, last +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ -- 2.16.6