fuel, armband: Add post mortem log collection

Debugging test suite failures is often hard without physical access
to the POD, and even then the environment might be automatically
redeployed before successfully debugging the problem.

So, collect all contents of </var/log> from all cluster nodes
(including the Salt Master node) after all test suites have been
executed as part of the CI job.

Change-Id: Ie5e2526c74f4ed21be9c76c6f6101eb3f434be36
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml
index 0154c7a..64c633a 100644 (file)
--- a/jjb/armband/armband-ci-jobs.yml
+++ b/jjb/armband/armband-ci-jobs.yml
@@ -158,6 +158,16 @@
                     build-step-failure-threshold: 'never'
                     failure-threshold: 'never'
                     unstable-threshold: 'FAILURE'
+      # Armband uses Fuel's log collection project job, no need to duplicate
+      - trigger-builds:
+          - project: 'fuel-collect-logs-{pod}-daily-{stream}'
+            current-parameters: false
+            block: true
+            same-node: true
+            block-thresholds:
+              build-step-failure-threshold: 'never'
+              failure-threshold: 'never'
+              unstable-threshold: 'FAILURE'
 
 - job-template:
     name: '{installer}-deploy-{pod}-daily-{stream}'

diff --git a/jjb/fuel/fuel-daily-jobs.yml b/jjb/fuel/fuel-daily-jobs.yml
index d489f7e..b6a1bc4 100644 (file)
--- a/jjb/fuel/fuel-daily-jobs.yml
+++ b/jjb/fuel/fuel-daily-jobs.yml
@@ -115,6 +115,7 @@
     jobs:
       - 'fuel-{scenario}-{pod}-daily-{stream}'
       - 'fuel-deploy-{pod}-daily-{stream}'
+      - 'fuel-collect-logs-{pod}-daily-{stream}'
 
 ########################
 # job templates
@@ -215,6 +216,22 @@
                     build-step-failure-threshold: 'never'
                     failure-threshold: 'never'
                     unstable-threshold: 'FAILURE'
+      - conditional-step:
+          condition-kind: not
+          condition-operand:
+            condition-kind: regex-match
+            regex: 'danube'
+            label: '{stream}'
+          steps:
+            - trigger-builds:
+                - project: 'fuel-collect-logs-{pod}-daily-{stream}'
+                  current-parameters: false
+                  block: true
+                  same-node: true
+                  block-thresholds:
+                    build-step-failure-threshold: 'never'
+                    failure-threshold: 'never'
+                    unstable-threshold: 'FAILURE'
 
     publishers:
       - email:
@@ -280,6 +297,41 @@
           recipients: peter.barabas@ericsson.com fzhadaev@mirantis.com
       - email-jenkins-admins-on-failure
 
+- job-template:
+    name: 'fuel-collect-logs-{pod}-daily-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    concurrent: true
+
+    properties:
+      - logrotate-default
+
+    parameters:
+      - project-parameter:
+          project: '{project}'
+          branch: '{branch}'
+      - '{installer}-defaults':
+          gs-pathname: '{gs-pathname}'
+      - '{slave-label}-defaults':
+          installer: '{installer}'
+
+    scm:
+      - git-scm
+
+    wrappers:
+      - build-name:
+          name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO'
+
+    builders:
+      - description-setter:
+          description: "Built on $NODE_NAME"
+      - shell:
+          !include-raw-escape: ./fuel-logs.sh
+
+    publishers:
+      - email-jenkins-admins-on-failure
+
 ########################
 # trigger macros
 ########################

diff --git a/jjb/fuel/fuel-logs.sh b/jjb/fuel/fuel-logs.sh
new file mode 100755 (executable)
index 0000000..a7d8526
--- /dev/null
+++ b/jjb/fuel/fuel-logs.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2017 Ericsson AB, Mirantis Inc., Enea Software AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -o nounset
+set -o pipefail
+
+# Fuel requires deploy script to be ran with sudo, Armband does not
+SUDO='sudo -E'
+[ "${PROJECT}" = 'fuel' ] || SUDO=
+
+# Log file name
+FUEL_PM_LOG_FILENAME="${JOB_NAME}_${BUILD_NUMBER}_pm.log.tar.gz"
+
+# Construct the command
+LOG_COMMAND="${SUDO} ${WORKSPACE}/mcp/scripts/log.sh \
+             ${WORKSPACE}/${FUEL_PM_LOG_FILENAME}"
+
+# Log info to console
+echo "Collecting post mortem logs ..."
+echo "--------------------------------------------------------"
+echo "${LOG_COMMAND}"
+
+${LOG_COMMAND}
+
+# Upload logs for both baremetal and virtual deployments
+echo "Uploading deployment logs"
+echo "--------------------------------------------------------"
+gsutil cp "${WORKSPACE}/${FUEL_PM_LOG_FILENAME}" \
+    "gs://${GS_URL}/logs/${FUEL_PM_LOG_FILENAME}" > /dev/null 2>&1
+echo "Logs are available at http://${GS_URL}/logs/${FUEL_PM_LOG_FILENAME}"