X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=jjb%2Ffunctest%2Fxtesting.yaml;h=a836711cd33454ddb94fe4f98c242764abfc8ada;hb=refs%2Fchanges%2F16%2F72816%2F1;hp=d4b6f9261b1185c6771ada3bfebcf08fc59fa6f0;hpb=ca4ebaa12ba7bb4ad1a364e1177b0d0498d706aa;p=releng.git diff --git a/jjb/functest/xtesting.yaml b/jjb/functest/xtesting.yaml index d4b6f9261..a836711cd 100644 --- a/jjb/functest/xtesting.yaml +++ b/jjb/functest/xtesting.yaml @@ -1,7 +1,10 @@ --- -- xtesting-jobs: &xtesting-jobs - name: 'xtesting-jobs' - current-parameters: true +- xtesting-containers: &xtesting-containers + name: 'xtesting-containers' + repo: '{repo}' + port: '{port}' + container: '{container}' + tag: '{tag}' - xtesting-params: &xtesting-params name: 'xtesting-params' @@ -9,23 +12,39 @@ port: tag: - latest: + from: + buildargs: branch: master slave: lf-virtual1 + dependency: 3.14 + - wallaby: + from: + buildargs: + branch: stable/wallaby + slave: lf-virtual1 + dependency: 3.13 - leguer: + from: + buildargs: branch: stable/leguer slave: lf-virtual1 + dependency: 3.12 - kali: + from: + buildargs: branch: stable/kali slave: lf-virtual1 + dependency: 3.11 - jerma: + from: + buildargs: branch: stable/jerma slave: lf-virtual1 - - iruya: - branch: stable/iruya - slave: lf-virtual1 - - hunter: - branch: stable/hunter - slave: lf-virtual1 + dependency: 3.10 + +- xtesting-jobs: &xtesting-jobs + name: 'xtesting-jobs' + current-parameters: true - parameter: name: xtesting-slave @@ -40,25 +59,13 @@ - random-string: name: build_tag -- parameter: - name: xtesting-DEBUG - parameters: - - string: - name: DEBUG - default: true - -- xtesting-containers: &xtesting-containers - name: 'xtesting-containers' - repo: '{repo}' - port: '{port}' - container: '{container}' - tag: '{tag}' - - xtesting-run-containers: &xtesting-run-containers name: 'xtesting-run-containers' <<: *xtesting-containers test: '{test}' privileged: '{privileged}' + volumes: '{volumes}' + env: '{env}' network: '{network}' - builder: @@ -80,6 +87,14 @@ builders: - shell: | set +x + volumes=; + if [ "{volumes}" != "None" ]; then + for i in $(echo {volumes} | tr -d '[] ' |sed "s/,/ /g" ); do volumes="-v $i $volumes"; done + fi + env=; + if [ "{env}" != "None" ]; then + for i in $(eval echo {env} | tr -d '[] ' |sed "s/,/ /g" ); do env="-e $i $env"; done + fi [ ! -z "$WORKSPACE" ] && sudo rm -rf $WORKSPACE/results || true if [ "{repo}" = "_" ]; then image={container}:{tag} @@ -91,6 +106,8 @@ sudo docker run --rm \ --privileged={privileged} \ --network={network} \ + $volumes \ + $env \ -e S3_ENDPOINT_URL=https://storage.googleapis.com \ -e S3_DST_URL=s3://artifacts.opnfv.org/xtesting/$BUILD_TAG/$JOB_NAME-$BUILD_ID \ -e HTTP_DST_URL=http://artifacts.opnfv.org/xtesting/$BUILD_TAG/$JOB_NAME-$BUILD_ID \ @@ -100,7 +117,6 @@ -e NODE_NAME=$slave \ -e BUILD_TAG=$BUILD_TAG \ -v $WORKSPACE/../$JOB_NAME/results:/var/lib/xtesting/results \ - -e DEBUG=$DEBUG \ $image run_tests -t {test} -p -r - builder: @@ -117,55 +133,6 @@ fi sudo docker rmi $image || true -- parameter: - name: xtesting-branch - parameters: - - string: - name: branch - default: '{branch}' - -- xtesting-build-containers: &xtesting-build-containers - name: 'xtesting-build-containers' - <<: *xtesting-containers - ref_arg: '{ref_arg}' - path: '{path}' - -- builder: - name: xtesting-build-containers - builders: - - shell: | - set +x - if [ "{repo}" = "_" ]; then - image={container}:{tag} - elif [ "{port}" = "None" ]; then - image={repo}/{container}:{tag} - else - image={repo}:{port}/{container}:{tag} - fi - if [ "{ref_arg}" = "None" ]; then - build_arg="" - else - build_arg="--build-arg {ref_arg}={ref}" - fi - case {tag} in - *latest|*leguer) - cd {path};; - *) - cd docker;; - esac - sudo docker build $build_arg \ - --pull=false --no-cache --force-rm=true \ - -t $image . - -- scm: - name: xtesting-scm - scm: - - git: - url: https://gerrit.opnfv.org/gerrit/functest-xtesting - refspec: '+refs/changes/*:refs/changes/*' - branches: - - '{ref}' - - job-template: name: 'xtesting-{repo}-{container}-{tag}-pull' parameters: @@ -186,11 +153,6 @@ name: 'xtesting-opnfv-xtesting-mts-pull' <<: *xtesting-params container: 'xtesting-mts' - exclude: - - tag: hunter - - tag: iruya - - tag: jerma - - tag: kali jobs: - 'xtesting-{repo}-{container}-{tag}-pull' @@ -214,23 +176,16 @@ name: 'xtesting-opnfv-xtesting-mts-rmi' <<: *xtesting-params container: 'xtesting-mts' - exclude: - - tag: hunter - - tag: iruya - - tag: jerma - - tag: kali jobs: - 'xtesting-{repo}-{container}-{tag}-rmi' - job-template: - name: 'xtesting-{repo}-{container}-{tag}-{test}-run' + name: 'xtesting-opnfv-xtesting-{tag}-{test}-run' parameters: - xtesting-slave: slave: '{slave}' - xtesting-build_tag: build_tag: '' - - xtesting-DEBUG: - DEBUG: true builders: - xtesting-run-containers: <<: *xtesting-run-containers @@ -238,6 +193,8 @@ - project: name: 'xtesting-opnfv-xtesting' <<: *xtesting-params + volumes: + env: container: 'xtesting' test: - first @@ -246,42 +203,57 @@ - fourth - fifth - sixth + - eighth privileged: 'false' network: bridge exclude: - - tag: hunter - test: sixth - - tag: iruya - test: sixth + - tag: leguer + test: eighth + - tag: kali + test: eighth + - tag: jerma + test: eighth jobs: - - 'xtesting-{repo}-{container}-{tag}-{test}-run' + - 'xtesting-opnfv-xtesting-{tag}-{test}-run' + +- job-template: + name: 'xtesting-opnfv-xtesting-mts-{tag}-{test}-run' + parameters: + - xtesting-slave: + slave: '{slave}' + - xtesting-build_tag: + build_tag: '' + builders: + - xtesting-run-containers: + <<: *xtesting-run-containers - project: name: 'xtesting-opnfv-xtesting-mts' <<: *xtesting-params + volumes: + env: container: 'xtesting-mts' test: - seventh privileged: 'false' network: bridge - exclude: - - tag: hunter - test: seventh - - tag: iruya - test: seventh - - tag: jerma - test: seventh - - tag: kali - test: seventh jobs: - - 'xtesting-{repo}-{container}-{tag}-{test}-run' - + - 'xtesting-opnfv-xtesting-mts-{tag}-{test}-run' - builder: name: xtesting-zip builders: - shell: | set +x + volumes=; + if [ "{volumes}" != "None" ]; then + for i in $(echo {volumes} | tr -d '[] ' |sed "s/,/ /g" ); do volumes="-v $i $volumes"; done + fi + env=; + if [ "{env}" != "None" ]; then + for i in $(eval echo {env} | tr -d '[] ' |sed "s/,/ /g" ); do env="-e $i $env"; done + fi + [ ! -z "$WORKSPACE" ] && sudo rm -rf $WORKSPACE/results || true if [ "{repo}" = "_" ]; then image={container}:{tag} elif [ "{port}" = "None" ]; then @@ -290,6 +262,8 @@ image={repo}:{port}/{container}:{tag} fi sudo docker run --rm \ + $volumes \ + $env \ -e S3_ENDPOINT_URL=https://storage.googleapis.com \ -e S3_DST_URL=s3://artifacts.opnfv.org/xtesting \ -e HTTP_DST_URL=http://artifacts.opnfv.org/xtesting \ @@ -298,7 +272,6 @@ -e TEST_DB_EXT_URL=http://testresults.opnfv.org/test/api/v1/results \ -e BUILD_TAG=$BUILD_TAG \ -v $WORKSPACE/../$JOB_NAME/results:/var/lib/xtesting/results \ - -e DEBUG=$DEBUG \ $image zip_campaign - job-template: @@ -308,15 +281,17 @@ slave: '{slave}' - xtesting-build_tag: build_tag: '' - - xtesting-DEBUG: - DEBUG: true builders: - xtesting-zip: <<: *xtesting-containers + volumes: '{volumes}' + env: '{env}' - project: name: 'xtesting-{tag}-zip' <<: *xtesting-params + volumes: + env: container: 'xtesting' jobs: - 'xtesting-{tag}-zip' @@ -331,14 +306,14 @@ slave: '{slave}' - xtesting-build_tag: build_tag: '' - - xtesting-DEBUG: - DEBUG: true + # PyYAML and yamllint differ here + # see https://github.com/yaml/pyyaml/issues/234 + # yamllint disable rule:indentation properties: - build-blocker: - use-build-blocker: true - blocking-level: 'NODE' blocking-jobs: - - '^xtesting-{tag}-(daily|gate)$' + - ^xtesting-(pi-)*{tag}-(daily|docker|review)$ + # yamllint enable rule:indentation builders: - multijob: name: remove former images @@ -369,6 +344,8 @@ <<: *xtesting-jobs - name: 'xtesting-opnfv-xtesting-{tag}-sixth-run' <<: *xtesting-jobs + - name: 'xtesting-opnfv-xtesting-{tag}-eighth-run' + <<: *xtesting-jobs - multijob: name: opnfv/xtesting-mts:{tag} projects: @@ -380,38 +357,183 @@ - name: 'xtesting-{tag}-zip' <<: *xtesting-jobs +- project: + name: 'xtesting-daily' + <<: *xtesting-params + jobs: + - 'xtesting-{tag}-daily' + +- view: + name: xtesting + view-type: list + columns: + - status + - weather + - job + - last-success + - last-failure + - last-duration + regex: ^xtesting-[a-z0-9.]+-daily$ + +- xtesting-build-containers: &xtesting-build-containers + name: 'xtesting-build-containers' + <<: *xtesting-containers + ref_arg: '{ref_arg}' + path: '{path}' + buildargs: '{buildargs}' + from: '{from}' + +- builder: + name: xtesting-build-containers + builders: + - shell: | + set +x + if [ "{repo}" = "_" ]; then + image={container}:{tag} + elif [ "{port}" = "None" ]; then + image={repo}/{container}:{tag} + else + image={repo}:{port}/{container}:{tag} + fi + build_args="" + if [ "{buildargs}" != "None" ]; then + build_args="{buildargs}" + fi + if [ "{ref_arg}" != "None" ]; then + build_args="$build_args --build-arg {ref_arg}={ref}" + fi + cd {path} + if [ "{from}" != "None" ]; then + sed -i {from} Dockerfile + fi + sudo docker build $build_args \ + --pull=false --no-cache --force-rm=true \ + -t $image . + +- scm: + name: xtesting-scm + scm: + - git: + url: 'https://gerrit.opnfv.org/gerrit/functest-xtesting' + refspec: '+refs/heads/*:refs/remotes/origin/* +refs/changes/*:refs/changes/*' + branches: + - '{ref}' + +- xtesting-dep: &xtesting-dep + name: 'xtesting-containers' + repo: '{repo}' + port: '{port}' + tag: '{tag}' + dependency: '{dependency}' + +- builder: + name: xtesting-pull-dep-images + builders: + - shell: | + set +x + if [ "_" = "_" ]; then + image=alpine:{dependency} + elif [ "" = "None" ]; then + image=_/alpine:{dependency} + else + image=_:/alpine:{dependency} + fi + sudo docker pull $image || true + +- builder: + name: xtesting-remove-dep-images + builders: + - shell: | + set +x + if [ "_" = "_" ]; then + image=alpine:{dependency} + elif [ "" = "None" ]; then + image=_/alpine:{dependency} + else + image=_:/alpine:{dependency} + fi + sudo docker rmi $image || true + - job-template: - name: 'xtesting-{repo}-{container}-{tag}-gate' + name: 'xtesting-{repo}-{tag}-dep-pull' parameters: - xtesting-slave: slave: '{slave}' + builders: + - xtesting-pull-dep-images: + <<: *xtesting-dep + +- project: + name: 'xtesting-{repo}-{tag}-dep-pull' + <<: *xtesting-params + jobs: + - 'xtesting-{repo}-{tag}-dep-pull' + +- job-template: + name: 'xtesting-{repo}-{tag}-dep-rmi' + parameters: + - xtesting-slave: + slave: '{slave}' + builders: + - xtesting-remove-dep-images: + <<: *xtesting-dep + +- project: + name: 'xtesting-{repo}-{tag}-dep-rmi' + <<: *xtesting-params + jobs: + - 'xtesting-{repo}-{tag}-dep-rmi' + +- builder: + name: xtesting-tox + builders: + - shell: | + set +x + sudo apt-get update && sudo apt-get install \ + software-properties-common gpg -y + sudo add-apt-repository -y ppa:deadsnakes/ppa + sudo apt-get update && sudo apt-get install \ + python3.9 python3.9-dev python3.9-distutils \ + python3.8 python3.8-dev python3.8-distutils \ + python3.7 python3.7-dev python3.6 python3.6-dev \ + python python-dev python3-pip enchant -y + sudo pip3 install tox tox-pip-version + + tox + +- job-template: + name: 'xtesting-{tag}-tox' scm: - xtesting-scm: ref: $GERRIT_REFSPEC + triggers: + - xtesting-patchset-created: + branch: '{branch}' + parameters: + - xtesting-slave: + slave: '{slave}' builders: - - xtesting-build-containers: - <<: *xtesting-build-containers - ref: $GERRIT_REFSPEC + - xtesting-tox: - project: - name: 'xtesting-_-alpine-3.13-rmi' - repo: _ - port: - container: alpine - tag: '3.13' - slave: master + name: xtesting-tox + <<: *xtesting-params jobs: - - 'xtesting-{repo}-{container}-{tag}-rmi' + - 'xtesting-{tag}-tox' -- project: - name: 'xtesting-_-alpine-3.13-pull' - repo: _ - port: - container: alpine - tag: '3.13' - slave: master - jobs: - - 'xtesting-{repo}-{container}-{tag}-pull' +- job-template: + name: 'xtesting-{repo}-{container}-{tag}-gate' + parameters: + - xtesting-slave: + slave: '{slave}' + scm: + - xtesting-scm: + ref: $GERRIT_REFSPEC + builders: + - xtesting-build-containers: + <<: *xtesting-build-containers + ref: $GERRIT_REFSPEC + buildargs: '{buildargs}' - project: name: xtesting-opnfv-xtesting-{tag}-gate @@ -428,20 +550,13 @@ container: xtesting-mts ref_arg: BRANCH path: docker/mts - exclude: - - tag: hunter - - tag: iruya - - tag: jerma - - tag: kali jobs: - 'xtesting-{repo}-{container}-{tag}-gate' - - trigger: name: xtesting-patchset-created triggers: - gerrit: - server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event - comment-added-contains-event: @@ -454,11 +569,6 @@ branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' - skip-vote: - successful: false - failed: false - unstable: false - notbuilt: false - job-template: name: 'xtesting-{tag}-review' @@ -471,14 +581,14 @@ slave: '{slave}' - xtesting-build_tag: build_tag: '' - - xtesting-DEBUG: - DEBUG: true + # PyYAML and yamllint differ here + # see https://github.com/yaml/pyyaml/issues/234 + # yamllint disable rule:indentation properties: - build-blocker: - use-build-blocker: true - blocking-level: 'NODE' blocking-jobs: - - '^xtesting-{tag}-(daily|gate)$' + - ^xtesting-(pi-)*{tag}-(daily|docker|review)$ + # yamllint enable rule:indentation builders: - multijob: name: remove former images @@ -488,14 +598,14 @@ - name: 'xtesting-opnfv-xtesting-mts-{tag}-rmi' <<: *xtesting-jobs - multijob: - name: remove dependencies + name: remove dependency projects: - - name: 'xtesting-_-alpine-3.13-rmi' + - name: 'xtesting-{repo}-{tag}-dep-rmi' <<: *xtesting-jobs - multijob: - name: pull dependencies + name: pull dependency projects: - - name: 'xtesting-_-alpine-3.13-pull' + - name: 'xtesting-{repo}-{tag}-dep-pull' <<: *xtesting-jobs - multijob: name: opnfv/xtesting @@ -522,26 +632,80 @@ <<: *xtesting-jobs - name: 'xtesting-opnfv-xtesting-{tag}-sixth-run' <<: *xtesting-jobs + - name: 'xtesting-opnfv-xtesting-{tag}-eighth-run' + <<: *xtesting-jobs - multijob: name: opnfv/xtesting-mts:{tag} projects: - name: 'xtesting-opnfv-xtesting-mts-{tag}-seventh-run' <<: *xtesting-jobs +- project: + name: 'xtesting-review' + <<: *xtesting-params + jobs: + - 'xtesting-{tag}-review' + +- view: + name: xtesting-review + view-type: list + columns: + - status + - weather + - job + - last-success + - last-failure + - last-duration + regex: ^xtesting-[a-z0-9.]+-review$ + +- view: + name: xtesting-tox + view-type: list + columns: + - status + - weather + - job + - last-success + - last-failure + - last-duration + regex: ^xtesting-[a-z0-9.]+-tox$ + +- builder: + name: xtesting-push-containers + builders: + - shell: | + set +x + if [ "{repo}" = "_" ]; then + image={container}:{tag} + elif [ "{port}" = "None" ]; then + image={repo}/{container}:{tag} + else + image={repo}:{port}/{container}:{tag} + fi + sudo docker push $image + +- trigger: + name: xtesting-commit + triggers: + - pollscm: + cron: "*/30 * * * *" + - job-template: name: 'xtesting-{repo}-{container}-{tag}-build' parameters: - xtesting-slave: slave: '{slave}' - - xtesting-branch: - branch: '{branch}' scm: - xtesting-scm: - ref: $branch + ref: '{branch}' builders: - xtesting-build-containers: <<: *xtesting-build-containers - ref: $branch + ref: '{branch}' + buildargs: '{buildargs}' + - xtesting-push-containers: + <<: *xtesting-build-containers + ref: '{branch}' - project: name: xtesting-opnfv-xtesting-{tag}-build @@ -558,35 +722,38 @@ container: xtesting-mts ref_arg: BRANCH path: docker/mts - exclude: - - tag: hunter - - tag: iruya - - tag: jerma - - tag: kali jobs: - 'xtesting-{repo}-{container}-{tag}-build' - - job-template: name: 'xtesting-{tag}-docker' project-type: multijob + triggers: + - xtesting-commit + scm: + - xtesting-scm: + ref: '{branch}' parameters: - xtesting-slave: slave: '{slave}' - - xtesting-branch: - branch: '{branch}' - - xtesting-DEBUG: - DEBUG: true + # PyYAML and yamllint differ here + # see https://github.com/yaml/pyyaml/issues/234 + # yamllint disable rule:indentation + properties: + - build-blocker: + blocking-jobs: + - ^xtesting-(pi-)*{tag}-(daily|docker|review)$ + # yamllint enable rule:indentation builders: - multijob: - name: remove dependencies + name: remove dependency projects: - - name: 'xtesting-_-alpine-3.13-rmi' + - name: 'xtesting-{repo}-{tag}-dep-rmi' <<: *xtesting-jobs - multijob: - name: pull dependencies + name: pull dependency projects: - - name: 'xtesting-_-alpine-3.13-pull' + - name: 'xtesting-{repo}-{tag}-dep-pull' <<: *xtesting-jobs - multijob: name: opnfv/xtesting @@ -599,28 +766,55 @@ - name: 'xtesting-opnfv-xtesting-mts-{tag}-build' <<: *xtesting-jobs +- builder: + name: xtesting-trivy + builders: + - shell: | + sudo apt-get update && sudo apt-get install curl -y + + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b . + if [ "{repo}" = "_" ]; then + image={container}:{tag} + elif [ "{port}" = "None" ]; then + image={repo}/{container}:{tag} + else + image={repo}:{port}/{container}:{tag} + fi + ./trivy --exit-code 1 $image + +- job-template: + name: 'xtesting-{repo}-{container}-{tag}-trivy' + triggers: + - timed: '@daily' + parameters: + - xtesting-slave: + slave: '{slave}' + builders: + - xtesting-trivy: + <<: *xtesting-containers + +- project: + name: 'xtesting-opnfv-xtesting-trivy' + <<: *xtesting-params + container: 'xtesting' + jobs: + - 'xtesting-{repo}-{container}-{tag}-trivy' + +- project: + name: 'xtesting-opnfv-xtesting-mts-trivy' + <<: *xtesting-params + container: 'xtesting-mts' + jobs: + - 'xtesting-{repo}-{container}-{tag}-trivy' + - project: name: 'xtesting' <<: *xtesting-params jobs: - - 'xtesting-{tag}-daily' - - 'xtesting-{tag}-review' - 'xtesting-{tag}-docker' - view: - name: xtesting-daily - view-type: list - columns: - - status - - weather - - job - - last-success - - last-failure - - last-duration - regex: ^xtesting-[a-z-]+-daily$ - -- view: - name: xtesting-review + name: xtesting-docker view-type: list columns: - status @@ -629,10 +823,10 @@ - last-success - last-failure - last-duration - regex: ^xtesting-[a-z-]+-review$ + regex: ^xtesting-[a-z0-9.]+-docker$ - view: - name: xtesting-docker + name: xtesting-trivy view-type: list columns: - status @@ -641,4 +835,4 @@ - last-success - last-failure - last-duration - regex: ^xtesting-[a-z-]+-docker$ + regex: (?!xtesting-pi)^xtesting-[a-z-0-9.]+-trivy$