[MaaS] Override failed testing by default

Some nodes fail automatic testing done by MaaS during commissioning,
although running the testing suites one more time manually works.
For now, just override all 'failed testing' nodes unconditionally.

[stable/euphrates cherry-pick additions]
Note: Since our salt formulas are pinned to 2017.12 repos, we
need to backport one salt-formula-maas patch merged upstream, which
adds support for translating status code '22' to 'Failed testing' [1].

JIRA: FUEL-333

[1] https://github.com/salt-formulas/salt-formula-maas/commit/08ffc3ff

Change-Id: I13d3ee3d82550524480aa53aa8752ab90aa940cd
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
(cherry picked from commit 81561126307f15d4f65a743ed2431ea8c713a921)

diff --git a/mcp/config/states/maas b/mcp/config/states/maas
index 39f6bad..24249d4 100755 (executable)
--- a/mcp/config/states/maas
+++ b/mcp/config/states/maas
@@ -27,12 +27,19 @@ function maas_fixup() {
 
   local fcnodes=$(echo "${statusout}" | \
     grep -Pzo 'status: Failed commissioning\n\s+system_id: \K.+\n')
+  local ftnodes=$(echo "${statusout}" | \
+    grep -Pzo 'status: Failed testing\n\s+system_id: \K.+\n')
   for node_system_id in ${fcnodes}; do
     salt -C 'mas01*' state.apply maas.machines.delete \
       pillar="{'system_id': '${node_system_id}'}"
     sleep 10
   done
-  if [ -n "${fcnodes}" ]; then
+  for node_system_id in ${ftnodes}; do
+    salt -C 'mas01*' state.apply maas.machines.override_failed_testing \
+      pillar="{'system_id': '${node_system_id}'}"
+    sleep 10
+  done
+  if [ -n "${fcnodes}" ] || [ -n "${ftnodes}" ]; then
     salt -C 'mas01*' state.apply maas.machines
     return 1
   fi

diff --git a/mcp/patches/0009-maas.py-Add-missing-status-ids.patch b/mcp/patches/0009-maas.py-Add-missing-status-ids.patch
new file mode 100644 (file)
index 0000000..648e62e
--- /dev/null
+++ b/mcp/patches/0009-maas.py-Add-missing-status-ids.patch
@@ -0,0 +1,38 @@
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+: Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+:
+: All rights reserved. This program and the accompanying materials
+: are made available under the terms of the Apache License, Version 2.0
+: which accompanies this distribution, and is available at
+: http://www.apache.org/licenses/LICENSE-2.0
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Sun, 17 Dec 2017 06:30:27 +0100
+Subject: [PATCH] maas.py: Add missing status ids
+
+Status ID list synced from MaaS 2.3 [1].
+
+[1] https://github.com/maas/maas/blob/2.3/src/maasserver/\
+    migrations/builtin/maasserver/0106_testing_status.py#L25
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ _modules/maas.py | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/_modules/maas.py b/_modules/maas.py
+index e279b70..1df4698 100644
+--- a/_modules/maas.py
++++ b/_modules/maas.py
+@@ -52,7 +52,10 @@ def __virtual__():
+     (9, 'Deploying'), (6, 'Deployed'), (7, 'Retired'), (8, 'Broken'),
+     (11, 'Failed deployment'), (12, 'Releasing'),
+     (13, 'Releasing failed'), (14, 'Disk erasing'),
+-    (15, 'Failed disk erasing')])
++    (15, 'Failed disk erasing'), (16, 'Rescue mode'),
++    (17, 'Entering rescue mode'), (18, 'Failed to enter rescue mode'),
++    (19, 'Exiting rescue mode'), (20, 'Failed to exit rescue mode'),
++    (21, 'Testing'), (22, 'Failed testing')])
+
+
+ def _format_data(data):

diff --git a/mcp/patches/patches.list b/mcp/patches/patches.list
index 2030133..62b0f60 100644 (file)
--- a/mcp/patches/patches.list
+++ b/mcp/patches/patches.list
@@ -13,6 +13,7 @@
 /usr/share/salt-formulas/env: 0006-maas-module-Add-VLAN-DHCP-enable-support.patch
 /usr/share/salt-formulas/env: 0007-network.interface-Fix-ifup-OVS-port-with-route.patch
 /usr/share/salt-formulas/env: 0008-Handle-file_recv-option.patch
+/usr/share/salt-formulas/env: 0009-maas.py-Add-missing-status-ids.patch
 /usr/share/salt-formulas/env: 0010-maas-region-allow-timeout-override.patch
 /usr/share/salt-formulas/reclass: 0011-service.horizon.server.cluster-Default-to-v2-API.patch
 /usr/share/salt-formulas/env: 0011-system.repo-Debian-Add-keyserver-proxy-support.patch

diff --git a/mcp/salt-formulas/maas/machines/delete.sls b/mcp/salt-formulas/maas/machines/delete.sls
index dbb625a..2903f92 100644 (file)
--- a/mcp/salt-formulas/maas/machines/delete.sls
+++ b/mcp/salt-formulas/maas/machines/delete.sls
@@ -10,6 +10,7 @@
 maas_login_admin:
   cmd.run:
   - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
+  - unless: 'test -e /var/lib/maas/.maas_credentials'
 
 # TODO: implement delete_machine via _modules/maas.py
 delete_machine:

diff --git a/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls b/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls
index 17a7df8..3983729 100644 (file)
--- a/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls
+++ b/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls
@@ -10,6 +10,7 @@
 maas_login_admin:
   cmd.run:
   - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
+  - unless: 'test -e /var/lib/maas/.maas_credentials'
 
 # TODO: implement mark_broken_fixed_machine via _modules/maas.py
 mark_broken_fixed_machine:

diff --git a/mcp/salt-formulas/maas/machines/override_failed_testing.sls b/mcp/salt-formulas/maas/machines/override_failed_testing.sls
new file mode 100644 (file)
index 0000000..e7fe1d2
--- /dev/null
+++ b/mcp/salt-formulas/maas/machines/override_failed_testing.sls
@@ -0,0 +1,20 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+{%- from "maas/map.jinja" import region with context %}
+
+maas_login_admin:
+  cmd.run:
+  - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
+  - unless: 'test -e /var/lib/maas/.maas_credentials'
+
+# TODO: implement override_failed_testing via _modules/maas.py
+mark_broken_fixed_machine:
+  cmd.run:
+  - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine override-failed-testing {{ pillar['system_id'] }}"
+  - require:
+    - cmd: maas_login_admin