From: Trevor Bramwell Date: Thu, 16 Nov 2017 23:51:38 +0000 (-0800) Subject: Email Weekly Anteater Reports to PTLs X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=commitdiff_plain;h=0f548e950248e1c35c7dcd8e922834b3228a79e4;p=releng.git Email Weekly Anteater Reports to PTLs Completely replaces the current weekly security scan job. Instead of publishing weekly security scan reports they will be emailed to individual project PTLs. Uses a modified copy of 'anteater-security-audit.sh' to ensure the security scan job is not affected in this change. A later change will be made to merge the file back in and update the jobs. This is why 'anteater-parameters' are added to both jobs-templates. Change-Id: Ia8ebffbfce7a2d4feb83ef68ff0ab0c7bb4d2104 Signed-off-by: Trevor Bramwell --- diff --git a/jjb/ci_gate_security/anteater-clone-all-repos.sh b/jjb/ci_gate_security/anteater-clone-all-repos.sh deleted file mode 100755 index 8a9e73d85..000000000 --- a/jjb/ci_gate_security/anteater-clone-all-repos.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# SPDX-license-identifier: Apache-2.0 -set -o errexit -set -o pipefail -set -o nounset -export PATH=$PATH:/usr/local/bin/ - - -#WORKSPACE="$(pwd)" - -cd $WORKSPACE -if [ ! -d "$WORKSPACE/allrepos" ]; then - mkdir $WORKSPACE/allrepos -fi - -cd $WORKSPACE/allrepos - -declare -a PROJECT_LIST -EXCLUDE_PROJECTS="All-Projects|All-Users|securedlab" - -PROJECT_LIST=($(ssh gerrit.opnfv.org -p 29418 gerrit ls-projects | egrep -v $EXCLUDE_PROJECTS)) -echo "PROJECT_LIST=(${PROJECT_LIST[*]})" > $WORKSPACE/opnfv-projects.sh - -for PROJECT in ${PROJECT_LIST[@]}; do - echo "> Cloning $PROJECT" - if [ ! -d "$PROJECT" ]; then - git clone "https://gerrit.opnfv.org/gerrit/$PROJECT.git" - else - pushd "$PROJECT" > /dev/null - git pull -f - popd > /dev/null - fi -done diff --git a/jjb/ci_gate_security/anteater-security-audit-weekly.sh b/jjb/ci_gate_security/anteater-security-audit-weekly.sh index 11909636a..25850af28 100644 --- a/jjb/ci_gate_security/anteater-security-audit-weekly.sh +++ b/jjb/ci_gate_security/anteater-security-audit-weekly.sh @@ -1,37 +1,51 @@ #!/bin/bash # SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +ANTEATER_SCAN_PATCHSET="${ANTEATER_SCAN_PATCHSET:-true}" + +cd $WORKSPACE +REPORTDIR='.reports' +mkdir -p $REPORTDIR +# Ensure any user can read the reports directory +chmod 777 $REPORTDIR + +ANTEATER_FILES="--patchset /home/opnfv/anteater/$PROJECT/patchset" + +if [[ "$ANTEATER_SCAN_PATCHSET" == "true" ]]; then + echo "Generating patchset file to list changed files" + git diff HEAD^1 --name-only | sed "s#^#/home/opnfv/anteater/$PROJECT/#" > $WORKSPACE/patchset + echo "Changed files are" + echo "--------------------------------------------------------" + cat $WORKSPACE/patchset + echo "--------------------------------------------------------" +else + echo "Checking full project $PROJECT" + ANTEATER_FILES="--path /home/opnfv/anteater/$PROJECT" +fi + +vols="-v $WORKSPACE:/home/opnfv/anteater/$PROJECT -v $WORKSPACE/$REPORTDIR:/home/opnfv/anteater/$REPORTDIR" +envs="-e PROJECT=$PROJECT" -echo "--------------------------------------------------------" -vols="-v $WORKSPACE/allrepos/:/home/opnfv/anteater/allrepos/" echo "Pulling releng-anteater docker image" echo "--------------------------------------------------------" docker pull opnfv/releng-anteater echo "--------------------------------------------------------" -cmd="docker run -id $vols opnfv/releng-anteater /bin/bash" -echo "Running docker command $cmd" -container_id=$($cmd) -echo "Container ID is $container_id" -source $WORKSPACE/opnfv-projects.sh -for project in "${PROJECT_LIST[@]}" - -do - cmd="/home/opnfv/venv/bin/anteater --project testproj --path /home/opnfv/anteater/allrepos/$project" - echo "Executing command inside container" - echo "$cmd" - echo "--------------------------------------------------------" - docker exec $container_id $cmd > $WORKSPACE/"$project".securityaudit.log 2>&1 -done +cmd="docker run -i $envs $vols --rm opnfv/releng-anteater \ +/home/opnfv/venv/bin/anteater --project $PROJECT $ANTEATER_FILES" +echo "Running docker container" +echo "$cmd" +$cmd > $WORKSPACE/securityaudit.log 2>&1 exit_code=$? echo "--------------------------------------------------------" -echo "Stopping docker container with ID $container_id" -docker stop $container_id - - -#gsutil cp $WORKSPACE/securityaudit.log \ -# gs://$GS_URL/$PROJECT-securityaudit-weekly.log 2>&1 -# -#gsutil -m setmeta \ -# -h "Content-Type:text/html" \ -# -h "Cache-Control:private, max-age=0, no-transform" \ -# gs://$GS_URL/$PROJECT-securityaudit-weekly.log > /dev/null 2>&1 +echo "Docker container exited with code: $exit_code" +echo "--------------------------------------------------------" +cat securityaudit.log +exit 0 diff --git a/jjb/ci_gate_security/opnfv-ci-gate-security.yml b/jjb/ci_gate_security/opnfv-ci-gate-security.yml index 0a412c240..5a2534ae8 100644 --- a/jjb/ci_gate_security/opnfv-ci-gate-security.yml +++ b/jjb/ci_gate_security/opnfv-ci-gate-security.yml @@ -9,9 +9,76 @@ project: anteaterfw + repo: + - apex + - apex-os-net-config + - apex-puppet-tripleo + - apex-tripleo-heat-templates + - armband + - auto + - availability + - bamboo + - barometer + - bottlenecks + - calipso + - clover + - compass-containers + - compass4nfv + - conductor + - container4nfv + - copper + - cperf + - daisy + - doctor + - domino + - dovetail + - dpacc + - enfv + - fastpathmetrics + - fds + - fuel + - functest + - ipv6 + - joid + - kvmfornfv + - models + - moon + - multisite + - netready + - nfvbench + - octopus + - onosfw + - openretriever + - opera + - opnfvdocs + - orchestra + - ovn4nfv + - ovno + - ovsnfv + - parser + - pharos + - pharos-tools + - promise + - qtip + - releng + - releng-anteater + - releng-testresults + - releng-utils + - releng-xci + - samplevnf + - sdnvpn + - securityscanning + - sfc + - snaps + - stor4nfv + - storperf + - ves + - vswitchperf + - yardstick + jobs: - 'opnfv-security-audit-verify-{stream}' - - 'opnfv-security-audit-weekly-{stream}' + - 'opnfv-security-audit-{repo}-weekly-{stream}' stream: - master: @@ -23,24 +90,34 @@ # job templates ######################## - job-template: - name: 'opnfv-security-audit-weekly-{stream}' + name: 'opnfv-security-audit-{repo}-weekly-{stream}' disabled: '{obj:disabled}' parameters: - - label: - name: SLAVE_LABEL - default: 'ericsson-build3' - description: 'Slave label on Jenkins' + - ericsson-build3-defaults + - string: + name: ANTEATER_SCAN_PATCHSET + default: "false" + description: "Have anteater scan patchsets (true) or full project (false)" - project-parameter: - project: releng + project: '{repo}' branch: '{branch}' + scm: + - git-scm-gerrit + triggers: - timed: '@weekly' builders: - anteater-security-audit-weekly + - clean-workspace + + publishers: + # defined in jjb/global/releng-macros.yml + - 'email-{repo}-ptl': + subject: 'OPNFV Security Scan Result: {repo}' - job-template: name: 'opnfv-security-audit-verify-{stream}' @@ -117,12 +194,8 @@ - shell: !include-raw: ./anteater-report-to-gerrit.sh -# yamllint disable rule:indentation - builder: name: anteater-security-audit-weekly builders: - shell: - !include-raw: - - ./anteater-clone-all-repos.sh - - ./anteater-security-audit-weekly.sh -# yamllint enable rule:indentation + !include-raw: ./anteater-security-audit-weekly.sh diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml index 08766943c..28216388e 100644 --- a/jjb/global/releng-macros.yml +++ b/jjb/global/releng-macros.yml @@ -463,3 +463,504 @@ failure: true send-to: - recipients + +# Email PTL publishers +- email_ptl_defaults: &email_ptl_defaults + name: 'email_ptl_defaults' + content-type: text + attach-build-log: true + compress-log: true + always: true + subject: '{subject}' + +- publisher: + name: 'email-apex-ptl' + publishers: &email_apex_ptl_defaults + - email-ext: + <<: *email_ptl_defaults + recipients: > + trozet@redhat.com +- publisher: + name: 'email-apex-os-net-config-ptl' + publishers: + <<: *email_apex_ptl_defaults +- publisher: + name: 'email-apex-puppet-tripleo-ptl' + publishers: + <<: *email_apex_ptl_defaults +- publisher: + name: 'email-apex-tripleo-heat-templates-ptl' + publishers: + <<: *email_apex_ptl_defaults + +- publisher: + name: 'email-armband-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + bob.monkman@arm.com + +- publisher: + name: 'email-auto-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + tina.tsou@arm.com + +- publisher: + name: 'email-availability-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + fuqiao@chinamobile.com + +- publisher: + name: 'email-bamboo-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + donaldh@cisco.com + +- publisher: + name: 'email-barometer-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + aasmith@redhat.com + +- publisher: + name: 'email-bottlenecks-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + gabriel.yuyang@huawei.com + +- publisher: + name: 'email-calipso-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + korlev@cisco.com + +- publisher: + name: 'email-clover-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + stephen.wong1@huawei.com + +- publisher: + name: 'email-compass4nfv-ptl' + publishers: &email_compass4nfv_ptl_defaults + - email-ext: + <<: *email_ptl_defaults + recipients: > + chigang@huawei.com +- publisher: + name: 'email-compass-containers-ptl' + publishers: + <<: *email_compass4nfv_ptl_defaults + +- publisher: + name: 'email-conductor-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + limingjiang@huawei.com + +- publisher: + name: 'email-container4nfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + jiaxuan@chinamobile.com + +- publisher: + name: 'email-copper-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + aimeeu.opensource@gmail.com + +- publisher: + name: 'email-cperf-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + matt.welch@intel.com + +- publisher: + name: 'email-daisy-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + hu.zhijiang@zte.com.cn + +- publisher: + name: 'email-doctor-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + r-mibu@cq.jp.nec.com + +- publisher: + name: 'email-domino-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + ulas.kozat@huawei.com + +- publisher: + name: 'email-dovetail-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + hongbo.tianhongbo@huawei.com + +- publisher: + name: 'email-dpacc-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + denglingli@chinamobile.com + +- publisher: + name: 'email-enfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + JBuchanan@advaoptical.com + +- publisher: + name: 'email-escalator-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + kong.wei2@zte.com.cn + +- publisher: + name: 'email-fastpathmetrics-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + maryam.tahhan@intel.com + +- publisher: + name: 'email-fds-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + fbrockne@cisco.com + +- publisher: + name: 'email-fuel-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + gelkinbard@mirantis.com + +- publisher: + name: 'email-functest-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + cedric.ollivier@orange.com + +- publisher: + name: 'email-ipv6-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + bh526r@att.com + +- publisher: + name: 'email-joid-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + artur.tyloch@canonical.com + +- publisher: + name: 'email-kvmfornfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + raghuveer.reddy@intel.com + +- publisher: + name: 'email-models-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + bs3131@att.com + +- publisher: + name: 'email-moon-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + ruan.he@orange.com + +- publisher: + name: 'email-multisite-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + joehuang@huawei.com + +- publisher: + name: 'email-netready-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + georg.kunz@ericsson.com + +- publisher: + name: 'email-nfvbench-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + ahothan@cisco.com + +- publisher: + name: 'email-octopus-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + ulrich.kleber@huawei.com + +- publisher: + name: 'email-onosfw-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + su.wei@huawei.com + +- publisher: + name: 'email-openretriever-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + jiaxuan@chinamobile.com + +- publisher: + name: 'email-opera-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + Yingjun.li@huawei.com + +- publisher: + name: 'email-opnfvdocs-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + sofia.wallin@ericsson.com + +- publisher: + name: 'email-orchestra-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + giuseppe.carella@fokus.fraunhofer.de + +- publisher: + name: 'email-ovn4nfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + trinath.somanchi@gmail.com + +- publisher: + name: 'email-ovno-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + wsmackie@juniper.net + +- publisher: + name: 'email-ovsnfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + MarkD.Graymark.d.gray@intel.com + +- publisher: + name: 'email-parser-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + shang.xiaodong@zte.com.cn + +- publisher: + name: 'email-pharos-ptl' + publishers: &email_pharos_ptl_defaults + - email-ext: + <<: *email_ptl_defaults + recipients: > + jack.morgan@intel.com +- publisher: + name: 'email-pharos-tools-ptl' + publishers: + <<: *email_pharos_ptl_defaults + +- publisher: + name: 'email-promise-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + kunzmann@docomolab-euro.com + +- publisher: + name: 'email-qtip-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + wu.zhihui1@zte.com.cn + +- publisher: + name: 'email-releng-ptl' + publishers: &email_releng_ptl_defaults + - email-ext: + <<: *email_ptl_defaults + recipients: > + fatih.degirmenci@ericsson.com +- publisher: + name: 'email-releng-anteater-ptl' + publishers: + <<: *email_releng_ptl_defaults +- publisher: + name: 'email-releng-testresults-ptl' + publishers: + <<: *email_releng_ptl_defaults +- publisher: + name: 'email-releng-utils-ptl' + publishers: + <<: *email_releng_ptl_defaults +- publisher: + name: 'email-releng-xci-ptl' + publishers: + <<: *email_releng_ptl_defaults + +- publisher: + name: 'email-samplevnf-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + deepak.s@intel.com + +- publisher: + name: 'email-sdnvpn-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + tim.irnich@ericsson.com + +- publisher: + name: 'email-securityscanning-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + lhinds@redhat.com + +- publisher: + name: 'email-sfc-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + ManuelBuilmbuil@suse.com + +- publisher: + name: 'email-snaps-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + s.pisarski@cablelabs.com + +- publisher: + name: 'email-stor4nfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + shane.wang@intel.com + +- publisher: + name: 'email-storperf-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + mark.beierl@emc.com + +- publisher: + name: 'email-ves-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + bryan.sullivan@att.com + +- publisher: + name: 'email-vswitchperf-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + sridhar.rao@spirent.com + +- publisher: + name: 'email-yardstick-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + ross.b.brattain@intel.com diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml index 04de1e091..8ce576ed6 100644 --- a/jjb/global/slave-params.yml +++ b/jjb/global/slave-params.yml @@ -456,6 +456,18 @@ default: $WORKSPACE/build_output description: "Directory where the build artifact will be located upon the completion of the build." +- parameter: + name: 'ericsson-build3-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'ericsson-build3' + description: 'Slave label on Jenkins' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - parameter: name: 'huawei-build-defaults' parameters: