From: Serena Feng Date: Thu, 22 Mar 2018 12:24:05 +0000 (+0000) Subject: Merge "QTIP: create a daily CI job for network test" X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=commitdiff_plain;h=14b5f30468a96c5e400d2139caac7c52a1542b26;hp=81657cbe25665f279d41b043316840f731e00dde;p=releng.git Merge "QTIP: create a daily CI job for network test" --- diff --git a/jjb/apex/apex-jjb-renderer.py b/jjb/apex/apex-jjb-renderer.py index 6fd83afe9..b67784376 100644 --- a/jjb/apex/apex-jjb-renderer.py +++ b/jjb/apex/apex-jjb-renderer.py @@ -12,30 +12,38 @@ import yaml from jinja2 import Environment from jinja2 import FileSystemLoader -gspathname = dict() -branch = dict() -build_slave = dict() -env = Environment(loader=FileSystemLoader('./'), autoescape=True) -with open('scenarios.yaml.hidden') as _: - scenarios = yaml.safe_load(_) +def render_jjb(): + """Render JJB output from scenarios.yaml.hidden file and jinja + template""" -template = env.get_template('apex.yml.j2') + gspathname = dict() + branch = dict() + build_slave = dict() + env = Environment(loader=FileSystemLoader('./'), autoescape=True) -print("Scenarios are: ") -pprint.pprint(scenarios) + with open('scenarios.yaml.hidden') as _: + scenarios = yaml.safe_load(_) -for stream in scenarios: - if stream == 'master': - gspathname['master'] = '' - branch[stream] = stream - else: - gspathname[stream] = '/' + stream - branch[stream] = 'stable/' + stream - build_slave[stream] = 'apex-baremetal-{}'.format(stream) + template = env.get_template('apex.yml.j2') -output = template.render(scenarios=scenarios, gspathname=gspathname, - branch=branch, build_slave=build_slave) + print("Scenarios are: ") + pprint.pprint(scenarios) -with open('./apex.yml', 'w') as fh: - fh.write(output) + for stream in scenarios: + if stream == 'master': + gspathname['master'] = '' + branch[stream] = stream + else: + gspathname[stream] = '/' + stream + branch[stream] = 'stable/' + stream + build_slave[stream] = 'apex-baremetal-{}'.format(stream) + + output = template.render(scenarios=scenarios, gspathname=gspathname, + branch=branch, build_slave=build_slave) + + with open('./apex.yml', 'w') as fh: + fh.write(output) + +if __name__ == "__main__": + render_jjb() diff --git a/jjb/clover/clover-project.yml b/jjb/clover/clover-project.yml new file mode 100644 index 000000000..b31f3deb2 --- /dev/null +++ b/jjb/clover/clover-project.yml @@ -0,0 +1,166 @@ +--- +################################################### +# All the jobs except verify have been removed! +# They will only be enabled on request by projects! +################################################### +- project: + name: clover + + project: '{name}' + + jobs: + - 'clover-verify-{stream}' + - 'clover-daily-upload-{stream}' + - 'clover-daily-deploy-{stream}' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + +- job-template: + name: 'clover-verify-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + # yamllint disable rule:line-length + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + # yamllint enable rule:line-length + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + + builders: + - shell: | + cd $WORKSPACE/ci + ./verify.sh + +- job-template: + name: 'clover-daily-upload-{stream}' + + disabled: '{obj:disabled}' + + concurrent: false + + scm: + - git-scm + + triggers: + - timed: '0 11 * * *' + + wrappers: + - fix-workspace-permissions + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + - 'clover-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - shell: | + cd $WORKSPACE/ci + ./upload.sh + +- job-template: + name: 'clover-daily-deploy-{stream}' + + project-type: freestyle + + disabled: '{obj:disabled}' + + concurrent: false + + properties: + - logrotate-default + - throttle: + enabled: true + max-per-node: 1 + option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'clover-daily-deploy-.*?' + - 'container4nfv-daily-deploy-.*?' + block-level: 'NODE' + + wrappers: + - timeout: + timeout: 180 + abort: true + - fix-workspace-permissions + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + # yamllint disable rule:line-length + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + # yamllint enable rule:line-length + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - huawei-virtual6 + default-slaves: + - huawei-virtual6 + + scm: + - git-scm + + triggers: + - timed: '0 12 * * *' + + builders: + - shell: | + cd $WORKSPACE/ci + ./deploy.sh + +################### +# parameter macros +################### +- parameter: + name: 'clover-defaults' + parameters: + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." diff --git a/jjb/container4nfv/container4nfv-project.yml b/jjb/container4nfv/container4nfv-project.yml index 03bbb655e..1ab50ae30 100644 --- a/jjb/container4nfv/container4nfv-project.yml +++ b/jjb/container4nfv/container4nfv-project.yml @@ -109,8 +109,21 @@ properties: - logrotate-default + - throttle: + enabled: true + max-per-node: 1 + option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'clover-daily-deploy-.*?' + - 'container4nfv-daily-deploy-.*?' + block-level: 'NODE' wrappers: + - timeout: + timeout: 240 + abort: true - fix-workspace-permissions parameters: diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index b8c432458..1c53dbee4 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -93,13 +93,24 @@ fi cat $OPENRC if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == "compass" ]]; then + compass_repo=${WORKSPACE}/compass4nfv/ + git clone https://github.com/opnfv/compass4nfv.git ${compass_repo} >/dev/null + pip install shyaml + scenario_file=${compass_repo}/deploy/conf/hardware_environment/$NODE_NAME/os-nosdn-nofeature-ha.yml + ipmiIp=$(cat ${scenario_file} | shyaml get-value hosts.0.ipmiIp) + ipmiPass=$(cat ${scenario_file} | shyaml get-value hosts.0.ipmiPass) + ipmiUser=root + jumpserver_ip=$(ifconfig | grep -A 5 docker0 | grep "inet addr" | cut -d ':' -f 2 | cut -d ' ' -f 1) + cat << EOF >${DOVETAIL_CONFIG}/pod.yaml nodes: -- {ip: 10.1.0.52, name: node1, password: root, role: controller, user: root} +- {ip: ${jumpserver_ip}, name: node0, password: root, role: Jumpserver, user: root} +- {ip: 10.1.0.50, name: node1, password: root, role: controller, user: root, + ipmi_ip: ${ipmiIp}, ipmi_user: ${ipmiUser}, ipmi_password: ${ipmiPass}} - {ip: 10.1.0.51, name: node2, password: root, role: controller, user: root} -- {ip: 10.1.0.50, name: node3, password: root, role: controller, user: root} -- {ip: 10.1.0.54, name: node4, password: root, role: compute, user: root} -- {ip: 10.1.0.53, name: node5, password: root, role: compute, user: root} +- {ip: 10.1.0.52, name: node3, password: root, role: controller, user: root} +- {ip: 10.1.0.53, name: node4, password: root, role: compute, user: root} +- {ip: 10.1.0.54, name: node5, password: root, role: compute, user: root} EOF fi @@ -154,6 +165,7 @@ if [[ ! -f ${DOVETAIL_CONFIG}/pod.yaml ]]; then fi if [ -f ${DOVETAIL_CONFIG}/pod.yaml ]; then + sudo chown jenkins:jenkins ${DOVETAIL_CONFIG}/pod.yaml echo "Adapt process info for $INSTALLER_TYPE ..." attack_process='rabbitmq' cat << EOF >> ${DOVETAIL_CONFIG}/pod.yaml @@ -307,8 +319,9 @@ sudo cp -r ${DOVETAIL_HOME}/results ./ # PRIMARY_GROUP=$(id -gn $CURRENT_USER) # sudo chown -R ${CURRENT_USER}:${PRIMARY_GROUP} ${WORKSPACE}/results -#remove useless workspace from yardstick to save disk space +#remove useless files to save disk space sudo rm -rf ./results/workspace +sudo rm -f ./results/yardstick.img echo "Dovetail: done!" diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml index a58e0bf3f..8ffbe9e63 100644 --- a/jjb/functest/functest-daily-jobs.yml +++ b/jjb/functest/functest-daily-jobs.yml @@ -199,7 +199,7 @@ - 'daily': job-timeout: 360 - 'arm-daily': - job-timeout: 360 + job-timeout: 480 jobs: - 'functest-{installer}-{pod}-{testsuite}-{stream}' diff --git a/jjb/opnfvdocs/project.cfg b/jjb/opnfvdocs/project.cfg index 0722b4036..598269737 100644 --- a/jjb/opnfvdocs/project.cfg +++ b/jjb/opnfvdocs/project.cfg @@ -2,6 +2,7 @@ apex armband availability bottlenecks +clover compass4nfv copper conductor diff --git a/jjb/xci/xci-merge-jobs.yml b/jjb/xci/xci-merge-jobs.yml index a986aa5c2..d6442a976 100644 --- a/jjb/xci/xci-merge-jobs.yml +++ b/jjb/xci/xci-merge-jobs.yml @@ -9,16 +9,23 @@ stream: - master: branch: '{stream}' + disabled: false # ------------------------------- # distros # ------------------------------- distro: - ubuntu: disabled: false + kill-phase-on: FAILURE + abort-all-job: true - centos: - disabled: false + disabled: true + kill-phase-on: NEVER + abort-all-job: false - opensuse: disabled: false + kill-phase-on: FAILURE + abort-all-job: true # ------------------------------- # postmerge promotion phases # ------------------------------- @@ -41,7 +48,7 @@ project-type: multijob - disabled: false + disabled: '{obj:disabled}' concurrent: false @@ -65,6 +72,21 @@ - comment-added-contains-event: comment-contains-value: 'remerge' projects: + - project-compare-type: 'ANT' + project-pattern: 'releng-xci' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'false' + forbidden-file-paths: + - compare-type: ANT + pattern: 'xci/scripts/vm/**' + - compare-type: ANT + pattern: 'docs/**' + - compare-type: ANT + pattern: 'prototypes/**' + - compare-type: ANT + pattern: 'upstream/**' - project-compare-type: 'REG_EXP' project-pattern: 'sfc|sdnvpn' branches: @@ -84,6 +106,15 @@ - label: name: SLAVE_LABEL default: 'xci-virtual' + - string: + name: CI_LOOP + default: 'merge' + - string: + name: FUNCTEST_MODE + default: 'tier' + - string: + name: FUNCTEST_SUITE_NAME + default: 'healthcheck' - string: name: DISTRO default: 'all' @@ -115,6 +146,9 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME node-parameters: false kill-phase-on: FAILURE abort-all-job: true @@ -128,6 +162,9 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME node-parameters: false kill-phase-on: FAILURE abort-all-job: true @@ -141,9 +178,12 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME node-parameters: false - kill-phase-on: FAILURE - abort-all-job: true + kill-phase-on: NEVER + abort-all-job: false - inject: properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" - multijob: @@ -158,6 +198,9 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME node-parameters: true kill-phase-on: NEVER abort-all-job: true @@ -167,7 +210,7 @@ project-type: multijob - disabled: false + disabled: '{obj:disabled}' concurrent: false @@ -205,6 +248,15 @@ - label: name: SLAVE_LABEL default: 'xci-virtual' + - string: + name: CI_LOOP + default: 'merge' + - string: + name: FUNCTEST_MODE + default: 'tier' + - string: + name: FUNCTEST_SUITE_NAME + default: 'healthcheck' - string: name: DISTRO default: '{distro}' @@ -237,8 +289,8 @@ GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true + kill-phase-on: '{kill-phase-on}' + abort-all-job: '{abort-all-job}' - multijob: name: healthcheck condition: SUCCESSFUL @@ -253,11 +305,12 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - FUNCTEST_MODE=tier - FUNCTEST_TIER=healthcheck + CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME node-parameters: true - kill-phase-on: NEVER - abort-all-job: true + kill-phase-on: '{kill-phase-on}' + abort-all-job: '{abort-all-job}' publishers: - postbuildscript: @@ -292,6 +345,9 @@ - string: name: DISTRO default: 'ubuntu' + - string: + name: CI_LOOP + default: 'merge' - string: name: FUNCTEST_MODE default: 'tier' @@ -342,6 +398,15 @@ - string: name: PROJECT default: $GERRIT_PROJECT + - string: + name: CI_LOOP + default: 'merge' + - string: + name: FUNCTEST_MODE + default: 'tier' + - string: + name: FUNCTEST_SUITE_NAME + default: 'healthcheck' - string: name: DISTRO default: 'all' diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml index d78dc82d9..e5c1a9394 100644 --- a/jjb/xci/xci-verify-jobs.yml +++ b/jjb/xci/xci-verify-jobs.yml @@ -146,6 +146,15 @@ - string: name: CLEAN_DIB_IMAGES default: 'true' + - string: + name: CI_LOOP + default: 'verify' + - string: + name: FUNCTEST_MODE + default: 'tier' + - string: + name: FUNCTEST_SUITE_NAME + default: 'healthcheck' - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT @@ -171,6 +180,7 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + CI_LOOP=$CI_LOOP node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -188,8 +198,9 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - FUNCTEST_MODE=tier - FUNCTEST_TIER=healthcheck + CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME node-parameters: true kill-phase-on: NEVER abort-all-job: true @@ -228,6 +239,9 @@ - string: name: DISTRO default: 'ubuntu' + - string: + name: CI_LOOP + default: 'verify' - string: name: FUNCTEST_MODE default: 'tier' diff --git a/jjb/yardstick/yardstick-daily-jobs.yml b/jjb/yardstick/yardstick-daily-jobs.yml index 20890fa13..24c7f319b 100644 --- a/jjb/yardstick/yardstick-daily-jobs.yml +++ b/jjb/yardstick/yardstick-daily-jobs.yml @@ -262,8 +262,7 @@ - description-setter: description: "POD: $NODE_NAME" - 'yardstick-cleanup' - - 'yardstick-fetch-os-creds' - - 'yardstick-fetch-k8s-conf' + - 'yardstick-fetch-creds' - 'yardstick-{testsuite}' - 'yardstick-store-results' @@ -282,16 +281,23 @@ !include-raw: ./yardstick-daily.sh - builder: - name: yardstick-fetch-os-creds + name: yardstick-fetch-creds builders: - - shell: - !include-raw: ../../utils/fetch_os_creds.sh - -- builder: - name: yardstick-fetch-k8s-conf - builders: - - shell: - !include-raw: ./yardstick-get-k8s-conf.sh + # yamllint disable rule:indentation + - conditional-step: + condition-kind: regex-match + regex: "os-.*" + label: '$DEPLOY_SCENARIO' + steps: + - shell: + !include-raw: ../../utils/fetch_os_creds.sh + - conditional-step: + condition-kind: regex-match + regex: "k8-.*" + label: '$DEPLOY_SCENARIO' + steps: + - shell: + !include-raw: ./yardstick-get-k8s-conf.sh - builder: name: yardstick-store-results diff --git a/jjb/yardstick/yardstick-get-k8s-conf.sh b/jjb/yardstick/yardstick-get-k8s-conf.sh index 22a6aa4e6..116d2f0d0 100755 --- a/jjb/yardstick/yardstick-get-k8s-conf.sh +++ b/jjb/yardstick/yardstick-get-k8s-conf.sh @@ -7,6 +7,7 @@ if [[ "${DEPLOY_SCENARIO:0:2}" == "k8" ]];then if [[ ${INSTALLER_TYPE} == 'joid' ]];then juju scp kubernetes-master/0:config "${dest_path}" elif [[ ${INSTALLER_TYPE} == 'compass' ]];then + echo "Copy admin.conf to ${dest_path}" docker cp compass-tasks:/opt/admin.conf "${dest_path}" fi fi