maas: Avoid race condition in node fixups

When more than one node enters a failure state during a deploy attempt,
we recover the first one and issue another deploy request; avoid raising
an exception for the second node (which is not in 'Ready' state either),
allowing the retries to continue.

Change-Id: I4a3e037e78b5c48aebf6e700115c0bbf848c7cd5
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

diff --git a/mcp/patches/salt-formula-maas/0003-Extend-wait_for-maas.py-wait_for_-attempts-arg.patch b/mcp/patches/salt-formula-maas/0003-Extend-wait_for-maas.py-wait_for_-attempts-arg.patch
index 1be210f..efd7cc2 100644 (file)
--- a/mcp/patches/salt-formula-maas/0003-Extend-wait_for-maas.py-wait_for_-attempts-arg.patch
+++ b/mcp/patches/salt-formula-maas/0003-Extend-wait_for-maas.py-wait_for_-attempts-arg.patch
@@ -60,13 +60,13 @@ Signed-off-by: ting wu <ting.wu@enea.com>
 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
 ---
  README.rst                                   |  9 +++-
- _modules/maas.py                             | 49 ++++++++++++++++++--
+ _modules/maas.py                             | 52 +++++++++++++++++---
  maas/machines/wait_for_deployed.sls          |  2 +
  maas/machines/wait_for_ready.sls             |  3 ++
  maas/machines/wait_for_ready_or_deployed.sls | 15 ++++++
  maas/map.jinja                               |  4 ++
  tests/pillar/maas_region.sls                 |  4 ++
- 7 files changed, 79 insertions(+), 7 deletions(-)
+ 7 files changed, 81 insertions(+), 8 deletions(-)
  create mode 100644 maas/machines/wait_for_ready_or_deployed.sls
 
 diff --git a/README.rst b/README.rst
@@ -102,9 +102,18 @@ index 20da43e..78d8aef 100644
              ignore_machines:
                 - kvm01 # in case it's broken or whatever
 diff --git a/_modules/maas.py b/_modules/maas.py
-index c02f104..28e46c5 100644
+index c02f104..bb70576 100644
 --- a/_modules/maas.py
 +++ b/_modules/maas.py
+@@ -670,7 +670,7 @@ class DeployMachines(MaasObject):
+         if machine['status'] == self.DEPLOYED:
+             return
+         if machine['status'] != self.READY:
+-            raise Exception('Not in ready state')
++            return
+         data = {
+             'system_id': machine['system_id'],
+         }
 @@ -921,6 +921,7 @@ class MachinesStatus(MaasObject):
              req_status: string; Polling status
              machines:   list; machine names
@@ -122,7 +131,7 @@ index c02f104..28e46c5 100644
          if not to_discover:
              try:
                  to_discover = __salt__['config.get']('maas')['region'][
-@@ -943,11 +946,45 @@ class MachinesStatus(MaasObject):
+@@ -943,11 +946,46 @@ class MachinesStatus(MaasObject):
          while len(total) <= len(to_discover):
              for m in to_discover:
                  for discovered in MachinesStatus.execute()['machines']:
@@ -155,6 +164,7 @@ index c02f104..28e46c5 100644
 +                                LOG.info('Machine {0} mark broken'.format(sid))
 +                                cls._maas.post(u'api/2.0/machines/{0}/'
 +                                    .format(sid), 'mark_broken', **data)
++                                time.sleep(poll_time)
 +                                LOG.info('Machine {0} mark fixed'.format(sid))
 +                                cls._maas.post(u'api/2.0/machines/{0}/'
 +                                    .format(sid), 'mark_fixed', **data)
@@ -172,7 +182,7 @@ index c02f104..28e46c5 100644
              if len(total) <= 0:
                  LOG.debug(
                      "Machines:{} are:{}".format(to_discover, req_status))
-@@ -959,7 +996,9 @@ class MachinesStatus(MaasObject):
+@@ -959,7 +997,9 @@ class MachinesStatus(MaasObject):
                  "Waiting status:{} "
                  "for machines:{}"
                  "\nsleep for:{}s "

diff --git a/mcp/patches/salt-formula-maas/0005-Implement-tags-support.patch b/mcp/patches/salt-formula-maas/0005-Implement-tags-support.patch
index e59c2f2..09d21d0 100644 (file)
--- a/mcp/patches/salt-formula-maas/0005-Implement-tags-support.patch
+++ b/mcp/patches/salt-formula-maas/0005-Implement-tags-support.patch
@@ -34,7 +34,7 @@ index 78d8aef..8c35458 100644
          # domain: mydomain.local # This function broken
          http_proxy: http://192.168.0.10:3142
 diff --git a/_modules/maas.py b/_modules/maas.py
-index 28e46c5..f56ed34 100644
+index bb70576..0cda8dd 100644
 --- a/_modules/maas.py
 +++ b/_modules/maas.py
 @@ -876,6 +876,28 @@ class Domain(MaasObject):
@@ -66,7 +66,7 @@ index 28e46c5..f56ed34 100644
  class MachinesStatus(MaasObject):
      @classmethod
      def execute(cls, objects_name=None):
-@@ -1064,5 +1086,9 @@ def process_sshprefs():
+@@ -1065,5 +1087,9 @@ def process_sshprefs():
      return SSHPrefs().process()