X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=jjb%2Ffunctest%2Fxtesting.yaml;h=cecd047076411202c40dc924d354f8d7459f345f;hb=39104196379b50039ee7801514536837768112ee;hp=573e02ca5320227e6bab512ffaf999f039b4d62f;hpb=00cf0af21949ddcff4811702848f0cce5d492cff;p=releng.git diff --git a/jjb/functest/xtesting.yaml b/jjb/functest/xtesting.yaml index 573e02ca5..cecd04707 100644 --- a/jjb/functest/xtesting.yaml +++ b/jjb/functest/xtesting.yaml @@ -13,45 +13,51 @@ tag: - latest: from: - buildargs: + build_args: branch: master - slave: lf-virtual1 - dependency: 3.13 - - wallaby: + node: xtesting + dependency: 3.17 + - 2023.1: from: - buildargs: - branch: stable/wallaby - slave: lf-virtual1 - dependency: 3.13 - - leguer: + build_args: + branch: stable/2023.1 + node: xtesting + dependency: 3.17 + - zed: from: - buildargs: - branch: stable/leguer - slave: lf-virtual1 - dependency: 3.12 - - kali: + build_args: + branch: stable/zed + node: xtesting + dependency: 3.16 + - yoga: from: - buildargs: - branch: stable/kali - slave: lf-virtual1 - dependency: 3.11 - - jerma: + build_args: + branch: stable/yoga + node: xtesting + dependency: 3.16 + - xena: from: - buildargs: - branch: stable/jerma - slave: lf-virtual1 - dependency: 3.10 + build_args: + branch: stable/xena + node: xtesting + dependency: 3.14 + - wallaby: + from: + build_args: + branch: stable/wallaby + node: xtesting + dependency: 3.13 - xtesting-jobs: &xtesting-jobs name: 'xtesting-jobs' current-parameters: true - parameter: - name: xtesting-slave + name: xtesting-node parameters: - label: - name: slave - default: '{slave}' + name: node + default: '{node}' - parameter: name: xtesting-build_tag @@ -67,6 +73,9 @@ volumes: '{volumes}' env: '{env}' network: '{network}' + uid: '{uid}' + gid: '{gid}' + published_ports: '{published_ports}' - builder: name: xtesting-pull-containers @@ -80,16 +89,29 @@ else image={repo}:{port}/{container}:{tag} fi - docker pull $image + sudo docker pull $image - builder: name: xtesting-run-containers builders: - shell: | - set -ex - volumes=; for i in $(echo {volumes} | tr -d '[] ' |sed "s/,/ /g" ); do volumes="-v $i $volumes"; done - env=; for i in $(eval echo {env} | tr -d '[] ' |sed "s/,/ /g" ); do env="-e $i $env"; done - [ ! -z "$WORKSPACE" ] && rm -rf $WORKSPACE/results || true + set +x + volumes=; + if [ "{volumes}" != "None" ]; then + for i in $(echo {volumes} | tr -d '[]' |sed "s/, / /g" ); \ + do volumes="-v $i $volumes"; done + fi + env=; + if [ "{env}" != "None" ]; then + for i in $(eval echo {env} | tr -d '[]' |sed "s/, / /g" ); \ + do env="-e $i $env"; done + fi + published_ports=; + if [ "{published_ports}" != "None" ]; then + for i in $(echo {published_ports} | tr -d '[]' |sed "s/, / /g" ); \ + do published_ports="-p $i $published_ports"; done + fi + [ ! -z "$WORKSPACE" ] && sudo rm -rf $WORKSPACE/results || true if [ "{repo}" = "_" ]; then image={container}:{tag} elif [ "{port}" = "None" ]; then @@ -97,18 +119,21 @@ else image={repo}:{port}/{container}:{tag} fi - docker run --rm \ + sudo mkdir -p $WORKSPACE/results + sudo chown {uid}:{gid} $WORKSPACE/results + sudo docker run --rm \ --privileged={privileged} \ --network={network} \ $volumes \ $env \ + $published_ports \ -e S3_ENDPOINT_URL=https://storage.googleapis.com \ -e S3_DST_URL=s3://artifacts.opnfv.org/xtesting/$BUILD_TAG/$JOB_NAME-$BUILD_ID \ -e HTTP_DST_URL=http://artifacts.opnfv.org/xtesting/$BUILD_TAG/$JOB_NAME-$BUILD_ID \ - -v /home/opnfv/xtesting/.boto:/root/.boto \ + -v /home/opnfv/xtesting/.boto:/etc/boto.cfg \ -e TEST_DB_URL=http://testresults.opnfv.org/test/api/v1/results \ -e TEST_DB_EXT_URL=http://testresults.opnfv.org/test/api/v1/results \ - -e NODE_NAME=$slave \ + -e NODE_NAME=$node \ -e BUILD_TAG=$BUILD_TAG \ -v $WORKSPACE/../$JOB_NAME/results:/var/lib/xtesting/results \ $image run_tests -t {test} -p -r @@ -125,13 +150,13 @@ else image={repo}:{port}/{container}:{tag} fi - docker rmi $image || true + sudo docker rmi $image || true - job-template: name: 'xtesting-{repo}-{container}-{tag}-pull' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' builders: - xtesting-pull-containers: <<: *xtesting-containers @@ -143,18 +168,11 @@ jobs: - 'xtesting-{repo}-{container}-{tag}-pull' -- project: - name: 'xtesting-opnfv-xtesting-mts-pull' - <<: *xtesting-params - container: 'xtesting-mts' - jobs: - - 'xtesting-{repo}-{container}-{tag}-pull' - - job-template: name: 'xtesting-{repo}-{container}-{tag}-rmi' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' builders: - xtesting-remove-images: <<: *xtesting-containers @@ -166,18 +184,11 @@ jobs: - 'xtesting-{repo}-{container}-{tag}-rmi' -- project: - name: 'xtesting-opnfv-xtesting-mts-rmi' - <<: *xtesting-params - container: 'xtesting-mts' - jobs: - - 'xtesting-{repo}-{container}-{tag}-rmi' - - job-template: name: 'xtesting-opnfv-xtesting-{tag}-{test}-run' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' - xtesting-build_tag: build_tag: '' builders: @@ -189,6 +200,7 @@ <<: *xtesting-params volumes: env: + published_ports: container: 'xtesting' test: - first @@ -198,50 +210,40 @@ - fifth - sixth - eighth + - nineth privileged: 'false' network: bridge + uid: 1000 + gid: 1000 exclude: - - tag: leguer - test: eighth - - tag: kali - test: eighth - - tag: jerma - test: eighth + - tag: wallaby + test: nineth + - tag: xena + test: nineth jobs: - 'xtesting-opnfv-xtesting-{tag}-{test}-run' -- job-template: - name: 'xtesting-opnfv-xtesting-mts-{tag}-{test}-run' - parameters: - - xtesting-slave: - slave: '{slave}' - - xtesting-build_tag: - build_tag: '' - builders: - - xtesting-run-containers: - <<: *xtesting-run-containers - -- project: - name: 'xtesting-opnfv-xtesting-mts' - <<: *xtesting-params - volumes: - env: - container: 'xtesting-mts' - test: - - seventh - privileged: 'false' - network: bridge - jobs: - - 'xtesting-opnfv-xtesting-mts-{tag}-{test}-run' - - builder: name: xtesting-zip builders: - shell: | set +x - volumes=; for i in $(echo {volumes} | tr -d '[] ' |sed "s/,/ /g" ); do volumes="-v $i $volumes"; done - env=; for i in $(eval echo {env} | tr -d '[] ' |sed "s/,/ /g" ); do env="-e $i $env"; done - [ ! -z "$WORKSPACE" ] && rm -rf $WORKSPACE/results || true + volumes=; + if [ "{volumes}" != "None" ]; then + for i in $(echo {volumes} | tr -d '[]' |sed "s/, / /g" ); \ + do volumes="-v $i $volumes"; done + fi + env=; + if [ "{env}" != "None" ]; then + for i in $(eval echo {env} | tr -d '[]' |sed "s/, / /g" ); \ + do env="-e $i $env"; done + fi + published_ports=; + if [ "{published_ports}" != "None" ]; then + for i in $(echo {published_ports} | tr -d '[]' |sed "s/, / /g" ); \ + do published_ports="-p $i $published_ports"; done + fi + [ ! -z "$WORKSPACE" ] && sudo rm -rf $WORKSPACE/results || true if [ "{repo}" = "_" ]; then image={container}:{tag} elif [ "{port}" = "None" ]; then @@ -249,13 +251,16 @@ else image={repo}:{port}/{container}:{tag} fi - docker run --rm \ + sudo mkdir -p $WORKSPACE/results + sudo chown {uid}:{gid} $WORKSPACE/results + sudo docker run --rm \ $volumes \ $env \ + $published_ports \ -e S3_ENDPOINT_URL=https://storage.googleapis.com \ -e S3_DST_URL=s3://artifacts.opnfv.org/xtesting \ -e HTTP_DST_URL=http://artifacts.opnfv.org/xtesting \ - -v /home/opnfv/xtesting/.boto:/root/.boto \ + -v /home/opnfv/xtesting/.boto:/etc/boto.cfg \ -e TEST_DB_URL=http://testresults.opnfv.org/test/api/v1/results \ -e TEST_DB_EXT_URL=http://testresults.opnfv.org/test/api/v1/results \ -e BUILD_TAG=$BUILD_TAG \ @@ -265,8 +270,8 @@ - job-template: name: 'xtesting-{tag}-zip' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' - xtesting-build_tag: build_tag: '' builders: @@ -274,13 +279,19 @@ <<: *xtesting-containers volumes: '{volumes}' env: '{env}' + uid: '{uid}' + gid: '{gid}' + published_ports: '{published_ports}' - project: name: 'xtesting-{tag}-zip' <<: *xtesting-params volumes: env: + published_ports: container: 'xtesting' + uid: 1000 + gid: 1000 jobs: - 'xtesting-{tag}-zip' @@ -290,8 +301,8 @@ triggers: - timed: '@daily' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' - xtesting-build_tag: build_tag: '' # PyYAML and yamllint differ here @@ -308,15 +319,11 @@ projects: - name: 'xtesting-opnfv-xtesting-{tag}-rmi' <<: *xtesting-jobs - - name: 'xtesting-opnfv-xtesting-mts-{tag}-rmi' - <<: *xtesting-jobs - multijob: name: pull containers projects: - name: 'xtesting-opnfv-xtesting-{tag}-pull' <<: *xtesting-jobs - - name: 'xtesting-opnfv-xtesting-mts-{tag}-pull' - <<: *xtesting-jobs - multijob: name: opnfv/xtesting:{tag} projects: @@ -334,16 +341,19 @@ <<: *xtesting-jobs - name: 'xtesting-opnfv-xtesting-{tag}-eighth-run' <<: *xtesting-jobs - - multijob: - name: opnfv/xtesting-mts:{tag} - projects: - - name: 'xtesting-opnfv-xtesting-mts-{tag}-seventh-run' + - name: 'xtesting-opnfv-xtesting-{tag}-nineth-run' <<: *xtesting-jobs - multijob: name: dump all campaign data projects: - name: 'xtesting-{tag}-zip' <<: *xtesting-jobs + publishers: + - email-ext: + failure: false + first-failure: true + fixed: true + recipients: cedric.ollivier@orange.com - project: name: 'xtesting-daily' @@ -368,7 +378,7 @@ <<: *xtesting-containers ref_arg: '{ref_arg}' path: '{path}' - buildargs: '{buildargs}' + build_args: '{build_args}' from: '{from}' - builder: @@ -384,8 +394,9 @@ image={repo}:{port}/{container}:{tag} fi build_args="" - if [ "{buildargs}" != "None" ]; then - build_args="{buildargs}" + if [ "{build_args}" != "None" ]; then + for i in $(echo {build_args} | tr -d '[]' |sed "s/, / /g" ); \ + do build_args="--build-arg $i $build_args"; done fi if [ "{ref_arg}" != "None" ]; then build_args="$build_args --build-arg {ref_arg}={ref}" @@ -394,7 +405,7 @@ if [ "{from}" != "None" ]; then sed -i {from} Dockerfile fi - docker build $build_args \ + sudo docker build $build_args \ --pull=false --no-cache --force-rm=true \ -t $image . @@ -426,7 +437,7 @@ else image=_:/alpine:{dependency} fi - docker pull $image || true + sudo docker pull $image || true - builder: name: xtesting-remove-dep-images @@ -440,13 +451,13 @@ else image=_:/alpine:{dependency} fi - docker rmi $image || true + sudo docker rmi $image || true - job-template: name: 'xtesting-{repo}-{tag}-dep-pull' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' builders: - xtesting-pull-dep-images: <<: *xtesting-dep @@ -460,8 +471,8 @@ - job-template: name: 'xtesting-{repo}-{tag}-dep-rmi' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' builders: - xtesting-remove-dep-images: <<: *xtesting-dep @@ -477,14 +488,21 @@ builders: - shell: | set +x - apt-get update && apt-get install \ - software-properties-common gpg -y - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update && apt-get install python3.8 \ - python3.8-dev python3.8-distutils \ - python3.7 python3.7-dev python3.6 python3.6-dev \ - python python-dev python3-pip enchant -y - pip3 install tox tox-pip-version + sudo apt-get -o DPkg::Lock::Timeout=300 update && \ + sudo DEBIAN_FRONTEND=noninteractive apt-get \ + -o DPkg::Lock::Timeout=300 dist-upgrade -y + sudo DEBIAN_FRONTEND=noninteractive \ + apt-get -o DPkg::Lock::Timeout=300 install software-properties-common gpg -y + sudo add-apt-repository -y ppa:deadsnakes/ppa + sudo apt-get -o DPkg::Lock::Timeout=300 update && \ + sudo DEBIAN_FRONTEND=noninteractive apt-get \ + -o DPkg::Lock::Timeout=300 install \ + libxml2-dev libxslt-dev libffi-dev libjpeg-dev \ + python3.10 python3.10-dev python3.10-distutils \ + python3.9 python3.9-dev python3.9-distutils \ + python3.8 python3.8-dev python3.8-distutils \ + python3-pip enchant-2 -y + sudo pip3 install tox tox-pip-version tox @@ -497,8 +515,8 @@ - xtesting-patchset-created: branch: '{branch}' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' builders: - xtesting-tox: @@ -511,8 +529,8 @@ - job-template: name: 'xtesting-{repo}-{container}-{tag}-gate' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' scm: - xtesting-scm: ref: $GERRIT_REFSPEC @@ -520,7 +538,7 @@ - xtesting-build-containers: <<: *xtesting-build-containers ref: $GERRIT_REFSPEC - buildargs: '{buildargs}' + build_args: '{build_args}' - project: name: xtesting-opnfv-xtesting-{tag}-gate @@ -531,15 +549,6 @@ jobs: - 'xtesting-{repo}-{container}-{tag}-gate' -- project: - name: xtesting-opnfv-xtesting-mts-{tag}-gate - <<: *xtesting-params - container: xtesting-mts - ref_arg: BRANCH - path: docker/mts - jobs: - - 'xtesting-{repo}-{container}-{tag}-gate' - - trigger: name: xtesting-patchset-created triggers: @@ -564,8 +573,8 @@ - xtesting-patchset-created: branch: '{branch}' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' - xtesting-build_tag: build_tag: '' # PyYAML and yamllint differ here @@ -582,8 +591,6 @@ projects: - name: 'xtesting-opnfv-xtesting-{tag}-rmi' <<: *xtesting-jobs - - name: 'xtesting-opnfv-xtesting-mts-{tag}-rmi' - <<: *xtesting-jobs - multijob: name: remove dependency projects: @@ -599,11 +606,6 @@ projects: - name: 'xtesting-opnfv-xtesting-{tag}-gate' <<: *xtesting-jobs - - multijob: - name: opnfv/xtesting-mts - projects: - - name: 'xtesting-opnfv-xtesting-mts-{tag}-gate' - <<: *xtesting-jobs - multijob: name: opnfv/xtesting:{tag} projects: @@ -621,11 +623,14 @@ <<: *xtesting-jobs - name: 'xtesting-opnfv-xtesting-{tag}-eighth-run' <<: *xtesting-jobs - - multijob: - name: opnfv/xtesting-mts:{tag} - projects: - - name: 'xtesting-opnfv-xtesting-mts-{tag}-seventh-run' + - name: 'xtesting-opnfv-xtesting-{tag}-nineth-run' <<: *xtesting-jobs + publishers: + - email-ext: + failure: false + first-failure: true + fixed: true + recipients: cedric.ollivier@orange.com - project: name: 'xtesting-review' @@ -669,7 +674,7 @@ else image={repo}:{port}/{container}:{tag} fi - docker push $image + sudo docker push $image - trigger: name: xtesting-commit @@ -680,8 +685,8 @@ - job-template: name: 'xtesting-{repo}-{container}-{tag}-build' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' scm: - xtesting-scm: ref: '{branch}' @@ -689,7 +694,7 @@ - xtesting-build-containers: <<: *xtesting-build-containers ref: '{branch}' - buildargs: '{buildargs}' + build_args: '{build_args}' - xtesting-push-containers: <<: *xtesting-build-containers ref: '{branch}' @@ -703,15 +708,6 @@ jobs: - 'xtesting-{repo}-{container}-{tag}-build' -- project: - name: xtesting-opnfv-xtesting-mts-{tag}-build - <<: *xtesting-params - container: xtesting-mts - ref_arg: BRANCH - path: docker/mts - jobs: - - 'xtesting-{repo}-{container}-{tag}-build' - - job-template: name: 'xtesting-{tag}-docker' project-type: multijob @@ -721,8 +717,8 @@ - xtesting-scm: ref: '{branch}' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' # PyYAML and yamllint differ here # see https://github.com/yaml/pyyaml/issues/234 # yamllint disable rule:indentation @@ -747,19 +743,22 @@ projects: - name: 'xtesting-opnfv-xtesting-{tag}-build' <<: *xtesting-jobs - - multijob: - name: opnfv/xtesting-mts - projects: - - name: 'xtesting-opnfv-xtesting-mts-{tag}-build' - <<: *xtesting-jobs + publishers: + - email-ext: + failure: false + first-failure: true + fixed: true + recipients: cedric.ollivier@orange.com - builder: name: xtesting-trivy builders: - shell: | - apt-get update && apt-get install curl -y + sudo apt-get -o DPkg::Lock::Timeout=300 update && \ + sudo DEBIAN_FRONTEND=noninteractive apt-get \ + -o DPkg::Lock::Timeout=300 install curl -y - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b . + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b . if [ "{repo}" = "_" ]; then image={container}:{tag} elif [ "{port}" = "None" ]; then @@ -767,18 +766,24 @@ else image={repo}:{port}/{container}:{tag} fi - ./trivy --exit-code 1 $image + ./trivy image --exit-code 1 $image - job-template: name: 'xtesting-{repo}-{container}-{tag}-trivy' triggers: - timed: '@daily' parameters: - - xtesting-slave: - slave: '{slave}' + - xtesting-node: + node: '{node}' builders: - xtesting-trivy: <<: *xtesting-containers + publishers: + - email-ext: + failure: false + first-failure: true + fixed: true + recipients: cedric.ollivier@orange.com - project: name: 'xtesting-opnfv-xtesting-trivy' @@ -787,12 +792,90 @@ jobs: - 'xtesting-{repo}-{container}-{tag}-trivy' +- builder: + name: xtesting-grype + builders: + - shell: | + sudo apt-get -o DPkg::Lock::Timeout=300 update && \ + sudo DEBIAN_FRONTEND=noninteractive apt-get \ + -o DPkg::Lock::Timeout=300 install curl -y + + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b . + if [ "{repo}" = "_" ]; then + image={container}:{tag} + elif [ "{port}" = "None" ]; then + image={repo}/{container}:{tag} + else + image={repo}:{port}/{container}:{tag} + fi + ./grype -q $image + +- job-template: + name: 'xtesting-{repo}-{container}-{tag}-grype' + triggers: + - timed: '@daily' + parameters: + - xtesting-node: + node: '{node}' + builders: + - xtesting-grype: + <<: *xtesting-containers + publishers: + - email-ext: + failure: false + first-failure: true + fixed: true + recipients: cedric.ollivier@orange.com + - project: - name: 'xtesting-opnfv-xtesting-mts-trivy' + name: 'xtesting-opnfv-xtesting-grype' <<: *xtesting-params - container: 'xtesting-mts' + container: 'xtesting' jobs: - - 'xtesting-{repo}-{container}-{tag}-trivy' + - 'xtesting-{repo}-{container}-{tag}-grype' + +- builder: + name: xtesting-sbom + builders: + - shell: | + sudo apt-get -o DPkg::Lock::Timeout=300 update && \ + sudo DEBIAN_FRONTEND=noninteractive apt-get \ + -o DPkg::Lock::Timeout=300 install curl -y + + sudo mkdir -p ~/.docker + curl -sSfL https://raw.githubusercontent.com/docker/sbom-cli-plugin/main/install.sh | sudo sh -s -- + if [ "{repo}" = "_" ]; then + image={container}:{tag} + elif [ "{port}" = "None" ]; then + image={repo}/{container}:{tag} + else + image={repo}:{port}/{container}:{tag} + fi + sudo docker sbom $image + +- job-template: + name: 'xtesting-{repo}-{container}-{tag}-sbom' + triggers: + - timed: '@daily' + parameters: + - xtesting-node: + node: '{node}' + builders: + - xtesting-sbom: + <<: *xtesting-containers + publishers: + - email-ext: + failure: false + first-failure: true + fixed: true + recipients: cedric.ollivier@orange.com + +- project: + name: 'xtesting-opnfv-xtesting-sbom' + <<: *xtesting-params + container: 'xtesting' + jobs: + - 'xtesting-{repo}-{container}-{tag}-sbom' - project: name: 'xtesting' @@ -823,3 +906,27 @@ - last-failure - last-duration regex: (?!xtesting-pi)^xtesting-[a-z-0-9.]+-trivy$ + +- view: + name: xtesting-grype + view-type: list + columns: + - status + - weather + - job + - last-success + - last-failure + - last-duration + regex: (?!xtesting-pi)^xtesting-[a-z-0-9.]+-grype$ + +- view: + name: xtesting-sbom + view-type: list + columns: + - status + - weather + - job + - last-success + - last-failure + - last-duration + regex: (?!xtesting-pi)^xtesting-[a-z-0-9.]+-sbom$