Merge "A few major-upgrade issues"

diff --combined extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
index e0d5aa7,4ceedb9..0c590a4
--- 1/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
--- 2/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
@@@ -19,7 -19,7 +19,7 @@@ STONITH_STATE=$(pcs property show stoni
  pcs property set stonith-enabled=false
  
  # Migrate to HA NG
- if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
+ if [[ -n $(is_bootstrap_node) ]]; then
      migrate_full_to_ng_ha
  fi
  
@@@ -29,9 -29,26 +29,26 @@@
  # is going to take a long time because rabbit is down. By having the service stopped
  # systemctl try-restart is a noop
  
- for $service in $(services_to_migrate); do
+ for service in $(services_to_migrate); do
      manage_systemd_service stop "${service%%-clone}"
-     check_resource_systemd "${service%%-clone}" stopped 600
+     # So the reason for not reusing check_resource_systemd is that
+     # I have observed systemctl is-active returning unknown with at least
+     # one service that was stopped (See LP 1627254)
+     timeout=600
+     tstart=$(date +%s)
+     tend=$(( $tstart + $timeout ))
+     check_interval=3
+     while (( $(date +%s) < $tend )); do
+       if [[ "$(systemctl is-active ${service%%-clone})" = "active" ]]; then
+         echo "$service still active, sleeping $check_interval seconds."
+         sleep $check_interval
+       else
+         # we do not care if it is inactive, unknown or failed as long as it is
+         # not running
+         break
+       fi
+ 
+     done
  done
  
  # In case the mysql package is updated, the database on disk must be
@@@ -46,7 -63,7 +63,7 @@@
  # on mysql package versionning, but this can be overriden manually
  # to support specific upgrade scenario
  
- if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
+ if [[ -n $(is_bootstrap_node) ]]; then
      if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
          mysqldump $backup_flags > "$MYSQL_BACKUP_DIR/openstack_database.sql"
          cp -rdp /etc/my.cnf* "$MYSQL_BACKUP_DIR"
@@@ -68,7 -85,7 +85,7 @@@
  fi
  
  
- # Swift isn't controled by pacemaker
+ # Swift isn't controlled by pacemaker
  systemctl_swift stop
  
  tstart=$(date +%s)
@@@ -151,13 -168,5 +168,13 @@@ f
  
  # Pin messages sent to compute nodes to kilo, these will be upgraded later
  crudini  --set /etc/nova/nova.conf upgrade_levels compute "$upgrade_level_nova_compute"
 +# https://bugzilla.redhat.com/show_bug.cgi?id=1284047
 +# Change-Id: Ib3f6c12ff5471e1f017f28b16b1e6496a4a4b435
 +crudini  --set /etc/ceilometer/ceilometer.conf DEFAULT rpc_backend rabbit
 +# https://bugzilla.redhat.com/show_bug.cgi?id=1284058
 +# Ifd1861e3df46fad0e44ff9b5cbd58711bbc87c97 Swift Ceilometer middleware no longer exists
 +crudini --set /etc/swift/proxy-server.conf pipeline:main pipeline "catch_errors healthcheck cache ratelimit tempurl formpost authtoken keystone staticweb proxy-logging proxy-server"
 +# LP: 1615035, required only for M/N upgrade.
 +crudini --set /etc/nova/nova.conf DEFAULT scheduler_host_manager host_manager
  
  crudini --set /etc/sahara/sahara.conf DEFAULT plugins ambari,cdh,mapr,vanilla,spark,storm

diff --combined extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
index aef54f8,7e814f5..6055a3f
--- 1/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
--- 2/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
@@@ -32,13 -32,6 +32,13 @@@ f
  
  start_or_enable_service galera
  check_resource galera started 600
 +# We need mongod which is now a systemd service up and running before calling
 +# ceilometer-dbsync. There is still a race here: mongod might not be up on all nodes
 +# so ceilometer-dbsync will fail a couple of times before that. As it retries indefinitely
 +# we should be good.
 +# Due to LP Bug https://bugs.launchpad.net/tripleo/+bug/1627254 am using systemctl directly atm
 +systemctl start mongod
 +check_resource mongod started 600
  
  if [[ -n $(is_bootstrap_node) ]]; then
      tstart=$(date +%s)
@@@ -60,7 -53,6 +60,7 @@@
      keystone-manage db_sync
      neutron-db-manage --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugin.ini upgrade head
      nova-manage db sync
 +    nova-manage api_db sync
      #TODO(marios):someone from sahara needs to check this:
      # sahara-db-manage --config-file /etc/sahara/sahara.conf upgrade head
  fi
@@@ -76,7 -68,7 +76,7 @@@ systemctl_swift star
  # We need to start the systemd services we explicitely stopped at step _1.sh
  # FIXME: Should we let puppet during the convergence step do the service enabling or
  # should we add it here?
- for $service in $(services_to_migrate); do
-     manage_systemd_service stop "${service%%-clone}"
+ for service in $(services_to_migrate); do
+     manage_systemd_service start "${service%%-clone}"
      check_resource_systemd "${service%%-clone}" started 600
  done