X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=extraconfig%2Ftasks%2Fmajor_upgrade_pacemaker_migrations.sh;h=cd78f8383271bab236884b2343324b8daf846c37;hb=01198c81d29af2a90670e782adff865715241b38;hp=b63198db432a47989e30c22a9200c465424e1719;hpb=859ab124131b60aaba853b9d69c027dc1f22d6db;p=apex-tripleo-heat-templates.git

diff --git a/extraconfig/tasks/major_upgrade_pacemaker_migrations.sh b/extraconfig/tasks/major_upgrade_pacemaker_migrations.sh
index b63198db..cd78f838 100644
--- a/extraconfig/tasks/major_upgrade_pacemaker_migrations.sh
+++ b/extraconfig/tasks/major_upgrade_pacemaker_migrations.sh
@@ -13,63 +13,159 @@
 # been already applied, it should be possible to call the function
 # again without damaging the deployment or failing the upgrade.
 
-function add_missing_openstack_core_constraints {
-    # The CIBs are saved under /root as they might contain sensitive data
-    CIB="/root/migration.cib"
-    CIB_BACKUP="/root/backup.cib"
-    CIB_PUSH_NEEDED=n
-
-    rm -f "$CIB" "$CIB_BACKUP" || /bin/true
-    pcs cluster cib "$CIB"
-    cp "$CIB" "$CIB_BACKUP"
-
-    if ! pcs -f "$CIB" constraint --full | grep 'start openstack-sahara-api-clone then start openstack-sahara-engine-clone'; then
-        pcs -f "$CIB" constraint order start openstack-sahara-api-clone then start openstack-sahara-engine-clone
-        CIB_PUSH_NEEDED=y
+# If the major version of mysql is going to change after the major
+# upgrade, the database must be upgraded on disk to avoid failures
+# due to internal incompatibilities between major mysql versions
+# https://bugs.launchpad.net/tripleo/+bug/1587449
+# This function detects whether a database upgrade is required
+# after a mysql package upgrade. It returns 0 when no major upgrade
+# has to take place, 1 otherwise.
+function is_mysql_upgrade_needed {
+    # The name of the package which provides mysql might differ
+    # after the upgrade. Consider the generic package name, which
+    # should capture the major version change (e.g. 5.5 -> 10.1)
+    local name="mariadb"
+    local output
+    local ret
+    set +e
+    output=$(yum -q check-update $name)
+    ret=$?
+    set -e
+    if [ $ret -ne 100 ]; then
+        # no updates so we exit
+        echo "0"
+        return
     fi
 
-    if ! pcs -f "$CIB" constraint --full | grep 'start openstack-core-clone then start openstack-ceilometer-notification-clone'; then
-        pcs -f "$CIB" constraint order start openstack-core-clone then start openstack-ceilometer-notification-clone
-        CIB_PUSH_NEEDED=y
-    fi
-
-    if ! pcs -f "$CIB" constraint --full | grep 'start openstack-aodh-evaluator-clone then start openstack-aodh-listener-clone'; then
-        pcs -f "$CIB" constraint order start openstack-aodh-evaluator-clone then start openstack-aodh-listener-clone
-        CIB_PUSH_NEEDED=y
-    fi
+    local currentepoch=$(rpm -q --qf "%{epoch}" $name)
+    local currentversion=$(rpm -q --qf "%{version}" $name | cut -d. -f-2)
+    local currentrelease=$(rpm -q --qf "%{release}" $name)
+    local newoutput=$(repoquery -a --pkgnarrow=updates --qf "%{epoch} %{version} %{release}\n" $name)
+    local newepoch=$(echo "$newoutput" | awk '{ print $1 }')
+    local newversion=$(echo "$newoutput" | awk '{ print $2 }' | cut -d. -f-2)
+    local newrelease=$(echo "$newoutput" | awk '{ print $3 }')
 
-    if pcs -f "$CIB" constraint --full | grep 'start openstack-core-clone then start openstack-heat-api-clone'; then
-        CID=$(pcs -f "$CIB" constraint --full | grep 'start openstack-core-clone then start openstack-heat-api-clone' | sed -e 's/.*id\://g' -e 's/)//g')
-        pcs -f "$CIB" constraint remove $CID
-        CIB_PUSH_NEEDED=y
+    # With this we trigger the dump restore/path if we change either epoch or
+    # version in the package If only the release tag changes we do not do it
+    # FIXME: we could refine this by trying to parse the mariadb version
+    # into X.Y.Z and trigger the update only if X and/or Y change.
+    output=$(python -c "import rpm; rc = rpm.labelCompare((\"$currentepoch\", \"$currentversion\", None), (\"$newepoch\", \"$newversion\", None)); print rc")
+    if [ "$output" != "-1" ]; then
+        echo "0"
+        return
     fi
+    echo "1"
+}
 
-    if [ "$CIB_PUSH_NEEDED" = 'y' ]; then
-        pcs cluster cib-push "$CIB"
-    fi
+# This function returns the list of services to be migrated away from pacemaker
+# and to systemd. The reason to have these services in a separate function is because
+# this list is needed in three different places: major_upgrade_controller_pacemaker_{1,2}
+# and in the function to migrate the cluster from full HA to HA NG
+function services_to_migrate {
+    # The following PCMK resources the ones the we are going to delete
+    PCMK_RESOURCE_TODELETE="
+    httpd-clone
+    memcached-clone
+    mongod-clone
+    neutron-dhcp-agent-clone
+    neutron-l3-agent-clone
+    neutron-metadata-agent-clone
+    neutron-netns-cleanup-clone
+    neutron-openvswitch-agent-clone
+    neutron-ovs-cleanup-clone
+    neutron-server-clone
+    openstack-aodh-evaluator-clone
+    openstack-aodh-listener-clone
+    openstack-aodh-notifier-clone
+    openstack-ceilometer-api-clone
+    openstack-ceilometer-central-clone
+    openstack-ceilometer-collector-clone
+    openstack-ceilometer-notification-clone
+    openstack-cinder-api-clone
+    openstack-cinder-scheduler-clone
+    openstack-glance-api-clone
+    openstack-glance-registry-clone
+    openstack-gnocchi-metricd-clone
+    openstack-gnocchi-statsd-clone
+    openstack-heat-api-cfn-clone
+    openstack-heat-api-clone
+    openstack-heat-api-cloudwatch-clone
+    openstack-heat-engine-clone
+    openstack-nova-api-clone
+    openstack-nova-conductor-clone
+    openstack-nova-consoleauth-clone
+    openstack-nova-novncproxy-clone
+    openstack-nova-scheduler-clone
+    openstack-sahara-api-clone
+    openstack-sahara-engine-clone
+    "
+    echo $PCMK_RESOURCE_TODELETE
 }
 
-function remove_ceilometer_alarm {
-    if pcs status | grep openstack-ceilometer-alarm; then
-        # Disable pacemaker resources for ceilometer-alarms
-        pcs resource disable openstack-ceilometer-alarm-evaluator
-        check_resource openstack-ceilometer-alarm-evaluator stopped 600
-        pcs resource delete openstack-ceilometer-alarm-evaluator
-        pcs resource disable openstack-ceilometer-alarm-notifier
-        check_resource openstack-ceilometer-alarm-notifier stopped 600
-        pcs resource delete openstack-ceilometer-alarm-notifier
+# This function will migrate a mitaka system where all the resources are managed
+# via pacemaker to a newton setup where only a few services will be managed by pacemaker
+# On a high-level it will operate as follows:
+# 1. Set the cluster in maintenance-mode so no start/stop action will actually take place
+#    during the conversion
+# 2. Remove all the colocation constraints and then the ordering constraints, except the
+#    ones related to haproxy/VIPs which exist in Newton as well
+# 3. Take the cluster out of maintenance-mode
+# 4. Remove all the resources that won't be managed by pacemaker in newton. The
+#    outcome will be
+#    that they are stopped and removed from pacemakers control
+# 5. Do a resource cleanup to make sure the cluster is in a clean state
+function migrate_full_to_ng_ha {
+    if [[ -n $(pcmk_running) ]]; then
+        pcs property set maintenance-mode=true
 
-        # remove constraints
-        pcs constraint remove ceilometer-delay-then-ceilometer-alarm-evaluator-constraint
-        pcs constraint remove ceilometer-alarm-evaluator-with-ceilometer-delay-colocation
-        pcs constraint remove ceilometer-alarm-evaluator-then-ceilometer-alarm-notifier-constraint
-        pcs constraint remove ceilometer-alarm-notifier-with-ceilometer-alarm-evaluator-colocation
-        pcs constraint remove ceilometer-alarm-notifier-then-ceilometer-notification-constraint
-        pcs constraint remove ceilometer-notification-with-ceilometer-alarm-notifier-colocation
+        # First we go through all the colocation constraints (except the ones
+        # we want to keep, i.e. the haproxy/ip ones) and we remove those
+        COL_CONSTRAINTS=$(pcs config show | sed -n '/^Colocation Constraints:$/,/^$/p' | grep -v "Colocation Constraints:" | egrep -v "ip-.*haproxy" | awk '{print $NF}' | cut -f2 -d: |cut -f1 -d\))
+        for constraint in $COL_CONSTRAINTS; do
+            log_debug "Deleting colocation constraint $constraint from CIB"
+            pcs constraint remove "$constraint"
+        done
 
-    fi
+        # Now we kill all the ordering constraints (except the haproxy/ip ones)
+        ORD_CONSTRAINTS=$(pcs config show | sed -n '/^Ordering Constraints:/,/^Colocation Constraints:$/p' | grep -v "Ordering Constraints:"  | awk '{print $NF}' | cut -f2 -d: |cut -f1 -d\))
+        for constraint in $ORD_CONSTRAINTS; do
+            log_debug "Deleting ordering constraint $constraint from CIB"
+            pcs constraint remove "$constraint"
+        done
+        # At this stage all the pacemaker resources are removed from the CIB.
+        # Once we remove the maintenance-mode those systemd resources will keep
+        # on running. They shall be systemd enabled via the puppet converge
+        # step later on
+        pcs property set maintenance-mode=false
 
-    # uninstall openstack-ceilometer-alarm package
-    yum -y remove openstack-ceilometer-alarm
+        # At this stage there are no constraints whatsoever except the haproxy/ip ones
+        # which we want to keep. We now disable and then delete each resource
+        # that will move to systemd.
+        # We want the systemd resources be stopped before doing "yum update",
+        # that way "systemctl try-restart <service>" is no-op because the
+        # service was down already 
+        PCS_STATUS_OUTPUT="$(pcs status)"
+        for resource in $(services_to_migrate) "delay-clone" "openstack-core-clone"; do
+             if echo "$PCS_STATUS_OUTPUT" | grep "$resource"; then
+                 log_debug "Deleting $resource from the CIB"
+                 if ! pcs resource disable "$resource" --wait=600; then
+                     echo_error "ERROR: resource $resource failed to be disabled"
+                     exit 1
+                 fi
+                 pcs resource delete --force "$resource"
+             else
+                 log_debug "Service $service not found as a pacemaker resource, not trying to delete."
+             fi
+        done
 
+        # We need to do a pcs resource cleanup here + crm_resource --wait to
+        # make sure the cluster is in a clean state before we stop everything,
+        # upgrade and restart everything
+        pcs resource cleanup
+        # We are making sure here that the cluster is stable before proceeding
+        if ! timeout -k 10 600 crm_resource --wait; then
+            echo_error "ERROR: cluster remained unstable after resource cleanup for more than 600 seconds, exiting."
+            exit 1
+        fi
+    fi
 }