Bump the pacemaker service op_params to 200s for start and stop

Based on observed timeouts during updates bump the stop and start
timeouts for pacemaker service resources (via op_params) to 200.
This is based on the reasoning that the full timeout may be as
long as two elapsed timeout intervals. After an initial timeout,
the sigterm that follows is then allowed another
DefaultTimeoutStopSec seconds. The 200s is produced by allowing
this 2xDefaultTimeoutStopSec (@90s for systemd) and some
scheduling delta. Many thanks to Michele Baldessari.

Closes-Bug: 1531204
Change-Id: If6b43982c958f63bc78ad997400bf1279c23df7e

diff --git a/extraconfig/tasks/yum_update.sh b/extraconfig/tasks/yum_update.sh
index 2d6b8cc..c6313d9 100755 (executable)
--- a/extraconfig/tasks/yum_update.sh
+++ b/extraconfig/tasks/yum_update.sh
@@ -123,10 +123,10 @@ openstack-nova-scheduler"
 
     echo "Setting resource start/stop timeouts"
     for service in $SERVICES; do
-        pcs -f $pacemaker_dumpfile resource update $service op start timeout=100s op stop timeout=100s
+        pcs -f $pacemaker_dumpfile resource update $service op start timeout=200s op stop timeout=200s
     done
     # mongod start timeout is higher, setting only stop timeout
-    pcs -f $pacemaker_dumpfile resource update mongod op stop timeout=100s
+    pcs -f $pacemaker_dumpfile resource update mongod op start timeout=370s op  stop timeout=200s
 
     echo "Applying new Pacemaker config"
     if ! pcs cluster cib-push $pacemaker_dumpfile; then

diff --git a/puppet/manifests/overcloud_controller_pacemaker.pp b/puppet/manifests/overcloud_controller_pacemaker.pp
index d0d46e2..b3fe779 100644 (file)
--- a/puppet/manifests/overcloud_controller_pacemaker.pp
+++ b/puppet/manifests/overcloud_controller_pacemaker.pp
@@ -78,11 +78,11 @@ if hiera('step') >= 1 {
     Class['tripleo::fencing'] -> Class['pacemaker::stonith']
   }
 
-  # FIXME(gfidente): sets 100secs as default start timeout op
+  # FIXME(gfidente): sets 200secs as default start timeout op
   # param; until we can use pcmk global defaults we'll still
   # need to add it to every resource which redefines op params
   Pacemaker::Resource::Service {
-    op_params => 'start timeout=100s stop timeout=100s',
+    op_params => 'start timeout=200s stop timeout=200s',
   }
 
   # Only configure RabbitMQ in this step, don't start it yet to
@@ -352,7 +352,7 @@ if hiera('step') >= 2 {
 
     if downcase(hiera('ceilometer_backend')) == 'mongodb' {
       pacemaker::resource::service { $::mongodb::params::service_name :
-        op_params    => 'start timeout=120s stop timeout=100s',
+        op_params    => 'start timeout=370s stop timeout=200s',
         clone_params => true,
         require      => Class['::mongodb::server'],
       }
@@ -1319,24 +1319,24 @@ if hiera('step') >= 4 {
     # Nova
     pacemaker::resource::service { $::nova::params::api_service_name :
       clone_params => 'interleave=true',
-      op_params    => 'start timeout=100s stop timeout=100s monitor start-delay=10s',
+      op_params    => 'start timeout=200s stop timeout=200s monitor start-delay=10s',
     }
     pacemaker::resource::service { $::nova::params::conductor_service_name :
       clone_params => 'interleave=true',
-      op_params    => 'start timeout=100s stop timeout=100s monitor start-delay=10s',
+      op_params    => 'start timeout=200s stop timeout=200s monitor start-delay=10s',
     }
     pacemaker::resource::service { $::nova::params::consoleauth_service_name :
       clone_params => 'interleave=true',
-      op_params    => 'start timeout=100s stop timeout=100s monitor start-delay=10s',
+      op_params    => 'start timeout=200s stop timeout=200s monitor start-delay=10s',
       require      => Pacemaker::Resource::Service[$::keystone::params::service_name],
     }
     pacemaker::resource::service { $::nova::params::vncproxy_service_name :
       clone_params => 'interleave=true',
-      op_params    => 'start timeout=100s stop timeout=100s monitor start-delay=10s',
+      op_params    => 'start timeout=200s stop timeout=200s monitor start-delay=10s',
     }
     pacemaker::resource::service { $::nova::params::scheduler_service_name :
       clone_params => 'interleave=true',
-      op_params    => 'start timeout=100s stop timeout=100s monitor start-delay=10s',
+      op_params    => 'start timeout=200s stop timeout=200s monitor start-delay=10s',
     }
 
     pacemaker::constraint::base { 'keystone-then-nova-consoleauth-constraint':