Rework gnocchi-upgrade to run in a separate upgrade step

gnocchi when configured with swift will require keystone
to be available to authenticate to migrate to v3. At this
step keystone is not available and gnocchi upgrade fails
with auth error. Instead start apache in step 3, start
apache first and then run gnocchi upgrade in a separate
step and let upgrade happen here.

Closes-Bug: #1634897

Change-Id: I22d02528420e4456f84b80905a7b3a80653fa7b0

diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
index b3a0098..3706151 100755 (executable)
--- a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
@@ -64,6 +64,5 @@ if [[ -n $(is_bootstrap_node) ]]; then
     nova-manage db sync
     nova-manage api_db sync
     nova-manage db online_data_migrations
-    gnocchi-upgrade
     sahara-db-manage --config-file /etc/sahara/sahara.conf upgrade head
 fi

diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh
index b653c7c..d2cb955 100755 (executable)
--- a/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh
@@ -9,18 +9,9 @@ check_resource redis started 600
 start_or_enable_service openstack-cinder-volume
 check_resource openstack-cinder-volume started 600
 
+# start httpd so keystone is available for gnocchi
+# upgrade to run.
+systemctl start httpd
 
 # Swift isn't controled by pacemaker
 systemctl_swift start
-
-# We need to start the systemd services we explicitely stopped at step _1.sh
-# FIXME: Should we let puppet during the convergence step do the service enabling or
-# should we add it here?
-services=$(services_to_migrate)
-if [[ ${keep_sahara_services_on_upgrade} =~ [Ff]alse ]] ; then
-    services=${services%%openstack-sahara*}
-fi
-for service in $services; do
-    manage_systemd_service start "${service%%-clone}"
-    check_resource_systemd "${service%%-clone}" started 600
-done

diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh
new file mode 100755 (executable)
index 0000000..fa95f1f
--- /dev/null
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eu
+
+if [[ -n $(is_bootstrap_node) ]]; then
+  # run gnocchi upgrade
+  gnocchi-upgrade
+fi

diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh
new file mode 100755 (executable)
index 0000000..d569084
--- /dev/null
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eu
+
+# We need to start the systemd services we explicitely stopped at step _1.sh
+# FIXME: Should we let puppet during the convergence step do the service enabling or
+# should we add it here?
+services=$(services_to_migrate)
+if [[ ${keep_sahara_services_on_upgrade} =~ [Ff]alse ]] ; then
+    services=${services%%openstack-sahara*}
+fi
+for service in $services; do
+    manage_systemd_service start "${service%%-clone}"
+    check_resource_systemd "${service%%-clone}" started 600
+done

diff --git a/extraconfig/tasks/major_upgrade_pacemaker.yaml b/extraconfig/tasks/major_upgrade_pacemaker.yaml
index 7c78d5a..e13aada 100644 (file)
--- a/extraconfig/tasks/major_upgrade_pacemaker.yaml
+++ b/extraconfig/tasks/major_upgrade_pacemaker.yaml
@@ -126,6 +126,44 @@ resources:
       input_values: {get_param: input_values}
 
   ControllerPacemakerUpgradeConfig_Step3:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_pacemaker_migrations.sh
+          - get_file: major_upgrade_controller_pacemaker_3.sh
+
+  ControllerPacemakerUpgradeDeployment_Step3:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: ControllerPacemakerUpgradeDeployment_Step2
+    properties:
+      servers:  {get_param: [servers, Controller]}
+      config: {get_resource: ControllerPacemakerUpgradeConfig_Step3}
+      input_values: {get_param: input_values}
+
+  ControllerPacemakerUpgradeConfig_Step4:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_pacemaker_migrations.sh
+          - get_file: major_upgrade_controller_pacemaker_4.sh
+
+  ControllerPacemakerUpgradeDeployment_Step4:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: ControllerPacemakerUpgradeDeployment_Step3
+    properties:
+      servers:  {get_param: [servers, Controller]}
+      config: {get_resource: ControllerPacemakerUpgradeConfig_Step4}
+      input_values: {get_param: input_values}
+
+  ControllerPacemakerUpgradeConfig_Step5:
     type: OS::Heat::SoftwareConfig
     properties:
       group: script
@@ -140,13 +178,12 @@ resources:
                 KEEP_SAHARA_SERVICES_ON_UPGRADE: {get_param: KeepSaharaServicesOnUpgrade}
           - get_file: pacemaker_common_functions.sh
           - get_file: major_upgrade_pacemaker_migrations.sh
-          - get_file: major_upgrade_controller_pacemaker_3.sh
+          - get_file: major_upgrade_controller_pacemaker_5.sh
 
-  ControllerPacemakerUpgradeDeployment_Step3:
+  ControllerPacemakerUpgradeDeployment_Step5:
     type: OS::Heat::SoftwareDeploymentGroup
-    depends_on: ControllerPacemakerUpgradeDeployment_Step2
+    depends_on: ControllerPacemakerUpgradeDeployment_Step4
     properties:
       servers:  {get_param: [servers, Controller]}
-      config: {get_resource: ControllerPacemakerUpgradeConfig_Step3}
+      config: {get_resource: ControllerPacemakerUpgradeConfig_Step5}
       input_values: {get_param: input_values}
-