Retry if the pacemaker_resource commands failed

Add a retry when the pacemaker_resource command
wasn't apply correctly, more info here:
https://bugzilla.redhat.com/show_bug.cgi?id=1482116

This is the same approach puppet-pacemaker uses
and provides eventual consistency when multiple
nodes change the cluster CIB concurrently.

This change depends-on :
https://review.gerrithub.io/375982

The return code is not available in the current
ansible-pacemaker package.

Change-Id: I8da03f5c4a6d442617b81be5793a9724cc8842bf
(cherry picked from commit e92430d8d03fc2ce2d0ce192b96209f2c5c04169)

diff --git a/docker/services/pacemaker/cinder-backup.yaml b/docker/services/pacemaker/cinder-backup.yaml
index c2117c0..cdb8c1b 100644 (file)
--- a/docker/services/pacemaker/cinder-backup.yaml
+++ b/docker/services/pacemaker/cinder-backup.yaml
@@ -188,6 +188,9 @@ outputs:
             resource: openstack-cinder-backup
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped openstack-cinder-backup cluster resource.
           tags: step2
@@ -195,6 +198,9 @@ outputs:
             resource: openstack-cinder-backup
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable cinder_backup service
           tags: step2

diff --git a/docker/services/pacemaker/cinder-volume.yaml b/docker/services/pacemaker/cinder-volume.yaml
index a4f6951..15c5e09 100644 (file)
--- a/docker/services/pacemaker/cinder-volume.yaml
+++ b/docker/services/pacemaker/cinder-volume.yaml
@@ -206,6 +206,9 @@ outputs:
             resource: openstack-cinder-volume
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped openstack-cinder-volume cluster resource.
           tags: step2
@@ -213,6 +216,9 @@ outputs:
             resource: openstack-cinder-volume
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable cinder_volume service from boot
           tags: step2

diff --git a/docker/services/pacemaker/database/mysql.yaml b/docker/services/pacemaker/database/mysql.yaml
index f57f779..1e1b2ba 100644 (file)
--- a/docker/services/pacemaker/database/mysql.yaml
+++ b/docker/services/pacemaker/database/mysql.yaml
@@ -272,6 +272,9 @@ outputs:
             resource: galera
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped galera cluster resource.
           tags: step2
@@ -279,6 +282,9 @@ outputs:
             resource: galera
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable mysql service
           tags: step2

diff --git a/docker/services/pacemaker/database/redis.yaml b/docker/services/pacemaker/database/redis.yaml
index ac64120..4ebeae8 100644 (file)
--- a/docker/services/pacemaker/database/redis.yaml
+++ b/docker/services/pacemaker/database/redis.yaml
@@ -228,6 +228,9 @@ outputs:
             resource: {get_attr: [RedisBase, role_data, service_name]}
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped redis cluster resource.
           tags: step2
@@ -235,6 +238,9 @@ outputs:
             resource: {get_attr: [RedisBase, role_data, service_name]}
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable redis service
           tags: step2

diff --git a/docker/services/pacemaker/haproxy.yaml b/docker/services/pacemaker/haproxy.yaml
index 3cdc525..29bd22a 100644 (file)
--- a/docker/services/pacemaker/haproxy.yaml
+++ b/docker/services/pacemaker/haproxy.yaml
@@ -230,6 +230,9 @@ outputs:
             resource: {get_attr: [HAProxyBase, role_data, service_name]}
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped haproxy cluster resource.
           tags: step2
@@ -237,4 +240,7 @@ outputs:
             resource: {get_attr: [HAProxyBase, role_data, service_name]}
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node

diff --git a/docker/services/pacemaker/rabbitmq.yaml b/docker/services/pacemaker/rabbitmq.yaml
index ba1abaf..b099374 100644 (file)
--- a/docker/services/pacemaker/rabbitmq.yaml
+++ b/docker/services/pacemaker/rabbitmq.yaml
@@ -221,6 +221,9 @@ outputs:
             resource: {get_attr: [RabbitmqBase, role_data, service_name]}
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped rabbitmq cluster resource.
           tags: step2
@@ -228,6 +231,9 @@ outputs:
             resource: {get_attr: [RabbitmqBase, role_data, service_name]}
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable rabbitmq service
           tags: step2