5 DEBUG="true" # set false if the verbosity is a problem
6 SCRIPT_NAME=$(basename $0)
8 if [[ $DEBUG = "true" ]]; then
9 echo "`date` $SCRIPT_NAME tripleo-upgrade $(facter hostname) $1"
13 function is_bootstrap_node {
14 if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid | tr '[:upper:]' '[:lower:]')" = "$(facter hostname | tr '[:upper:]' '[:lower:]')" ]; then
15 log_debug "Node is bootstrap"
20 function check_resource_pacemaker {
21 if [ "$#" -ne 3 ]; then
22 echo_error "ERROR: check_resource function expects 3 parameters, $# given"
30 if [[ -z $(is_bootstrap_node) ]] ; then
31 log_debug "Node isn't bootstrap, skipping check for $service to be $state here "
34 log_debug "Node is bootstrap checking $service to be $state here"
37 if [ "$state" = "stopped" ]; then
38 match_for_incomplete='Started'
40 match_for_incomplete='Stopped'
43 nodes_local=$(pcs status | grep ^Online | sed 's/.*\[ \(.*\) \]/\1/g' | sed 's/ /\|/g')
44 if timeout -k 10 $timeout crm_resource --wait; then
45 node_states=$(pcs status --full | grep "$service" | grep -v Clone | { egrep "$nodes_local" || true; } )
46 if echo "$node_states" | grep -q "$match_for_incomplete"; then
47 echo_error "ERROR: cluster finished transition but $service was not in $state state, exiting."
50 echo "$service has $state"
53 echo_error "ERROR: cluster remained unstable for more than $timeout seconds, exiting."
59 function pcmk_running {
60 if [[ $(systemctl is-active pacemaker) = "active" ]] ; then
65 function is_systemd_unknown {
67 if [[ $(systemctl is-active "$service") = "unknown" ]]; then
68 log_debug "$service found to be unkown to systemd"
73 function grep_is_cluster_controlled {
75 if [[ -n $(systemctl status $service -l | grep Drop-In -A 5 | grep pacemaker) ||
76 -n $(systemctl status $service -l | grep "Cluster Controlled $service") ]] ; then
77 log_debug "$service is pcmk managed from systemctl grep"
83 function is_systemd_managed {
85 #if we have pcmk check to see if it is managed there
86 if [[ -n $(pcmk_running) ]]; then
87 if [[ -z $(pcs status --full | grep $service) && -z $(is_systemd_unknown $service) ]] ; then
88 log_debug "$service found to be systemd managed from pcs status"
92 # if it is "unknown" to systemd, then it is pacemaker managed
93 if [[ -n $(is_systemd_unknown $service) ]] ; then
95 elif [[ -z $(grep_is_cluster_controlled $service) ]] ; then
101 function is_pacemaker_managed {
103 #if we have pcmk check to see if it is managed there
104 if [[ -n $(pcmk_running) ]]; then
105 if [[ -n $(pcs status --full | grep $service) ]]; then
106 log_debug "$service found to be pcmk managed from pcs status"
110 # if it is unknown to systemd, then it is pcmk managed
111 if [[ -n $(is_systemd_unknown $service) ]]; then
113 elif [[ -n $(grep_is_cluster_controlled $service) ]] ; then
119 function is_managed {
121 if [[ -n $(is_pacemaker_managed $service) || -n $(is_systemd_managed $service) ]]; then
126 function check_resource_systemd {
128 if [ "$#" -ne 3 ]; then
129 echo_error "ERROR: check_resource function expects 3 parameters, $# given"
136 local check_interval=3
138 if [ "$state" = "stopped" ]; then
139 match_for_incomplete='active'
141 match_for_incomplete='inactive'
144 log_debug "Going to check_resource_systemd for $service to be $state"
146 #sanity check is systemd managed:
147 if [[ -z $(is_systemd_managed $service) ]]; then
148 echo "ERROR - $service not found to be systemd managed."
153 tend=$(( $tstart + $timeout ))
154 while (( $(date +%s) < $tend )); do
155 if [[ "$(systemctl is-active $service)" = $match_for_incomplete ]]; then
156 echo "$service not yet $state, sleeping $check_interval seconds."
157 sleep $check_interval
159 echo "$service is $state"
164 echo "Timed out waiting for $service to go to $state after $timeout seconds"
169 function check_resource {
171 local pcmk_managed=$(is_pacemaker_managed $service)
172 local systemd_managed=$(is_systemd_managed $service)
174 if [[ -n $pcmk_managed && -n $systemd_managed ]] ; then
175 log_debug "ERROR $service managed by both systemd and pcmk - SKIPPING"
179 if [[ -n $pcmk_managed ]]; then
180 check_resource_pacemaker $@
182 elif [[ -n $systemd_managed ]]; then
183 check_resource_systemd $@
186 log_debug "ERROR cannot check_resource for $service, not managed here?"
189 function manage_systemd_service {
192 log_debug "Going to systemctl $action $service"
193 systemctl $action $service
196 function manage_pacemaker_service {
199 # not if pacemaker isn't running!
200 if [[ -z $(pcmk_running) ]]; then
201 echo "$(facter hostname) pacemaker not active, skipping $action $service here"
202 elif [[ -n $(is_bootstrap_node) ]]; then
203 log_debug "Going to pcs resource $action $service"
204 pcs resource $action $service
208 function stop_or_disable_service {
210 local pcmk_managed=$(is_pacemaker_managed $service)
211 local systemd_managed=$(is_systemd_managed $service)
213 if [[ -n $pcmk_managed && -n $systemd_managed ]] ; then
214 log_debug "Skipping stop_or_disable $service due to management conflict"
218 log_debug "Stopping or disabling $service"
219 if [[ -n $pcmk_managed ]]; then
220 manage_pacemaker_service disable $service
222 elif [[ -n $systemd_managed ]]; then
223 manage_systemd_service stop $service
226 log_debug "ERROR: $service not managed here?"
229 function start_or_enable_service {
231 local pcmk_managed=$(is_pacemaker_managed $service)
232 local systemd_managed=$(is_systemd_managed $service)
234 if [[ -n $pcmk_managed && -n $systemd_managed ]] ; then
235 log_debug "Skipping start_or_enable $service due to management conflict"
239 log_debug "Starting or enabling $service"
240 if [[ -n $pcmk_managed ]]; then
241 manage_pacemaker_service enable $service
243 elif [[ -n $systemd_managed ]]; then
244 manage_systemd_service start $service
247 log_debug "ERROR $service not managed here?"
250 function restart_service {
252 local pcmk_managed=$(is_pacemaker_managed $service)
253 local systemd_managed=$(is_systemd_managed $service)
255 if [[ -n $pcmk_managed && -n $systemd_managed ]] ; then
256 log_debug "ERROR $service managed by both systemd and pcmk - SKIPPING"
260 log_debug "Restarting $service"
261 if [[ -n $pcmk_managed ]]; then
262 manage_pacemaker_service restart $service
264 elif [[ -n $systemd_managed ]]; then
265 manage_systemd_service restart $service
268 log_debug "ERROR $service not managed here?"
271 function echo_error {
272 echo "$@" | tee /dev/fd2
275 # swift is a special case because it is/was never handled by pacemaker
276 # when stand-alone swift is used, only swift-proxy is running on controllers
277 function systemctl_swift {
278 services=( openstack-swift-account-auditor openstack-swift-account-reaper openstack-swift-account-replicator openstack-swift-account \
279 openstack-swift-container-auditor openstack-swift-container-replicator openstack-swift-container-updater openstack-swift-container \
280 openstack-swift-object-auditor openstack-swift-object-replicator openstack-swift-object-updater openstack-swift-object openstack-swift-proxy )
284 services=$(systemctl | grep openstack-swift- | grep running | awk '{print $1}')
287 enable_swift_storage=$(hiera -c /etc/puppet/hiera.yaml tripleo::profile::base::swift::storage::enable_swift_storage)
288 if [[ $enable_swift_storage != "true" ]]; then
289 services=( openstack-swift-proxy )
292 *) echo "Unknown action $action passed to systemctl_swift"
294 ;; # shouldn't ever happen...
296 for service in ${services[@]}; do
297 manage_systemd_service $action $service
301 # Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
302 # Update condition and add --notriggerun for +bug/1669714
303 function special_case_ovs_upgrade_if_needed {
304 if rpm -qa | grep "^openvswitch-2.5.0-14" || rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart" ; then
305 echo "Manual upgrade of openvswitch - ovs-2.5.0-14 or restart in postun detected"
307 mkdir OVS_UPGRADE && pushd OVS_UPGRADE
308 echo "Attempting to downloading latest openvswitch with yumdownloader"
309 yumdownloader --resolve openvswitch
310 for pkg in $(ls -1 *.rpm); do
311 if rpm -U --test $pkg 2>&1 | grep "already installed" ; then
312 echo "Looks like newer version of $pkg is already installed, skipping"
314 echo "Updating $pkg with --nopostun --notriggerun"
315 rpm -U --replacepkgs --nopostun --notriggerun $pkg
320 echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
325 # This code is meant to fix https://bugs.launchpad.net/tripleo/+bug/1686357 on
326 # existing setups via a minor update workflow and be idempotent. We need to
327 # run this before the yum update because we fix this up even when there are no
328 # packages to update on the system (in which case the script exits).
329 # This code must be called with set +eu (due to the ocf scripts being sourced)
330 function fixup_wrong_ipv6_vip {
331 # This XPath query identifies of all the VIPs in pacemaker with netmask /64. Those are IPv6 only resources that have the wrong netmask
332 # This gives the address of the resource in the CIB, one address per line. For example:
333 # /cib/configuration/resources/primitive[@id='ip-2001.db8.ca2.4..10']/instance_attributes[@id='ip-2001.db8.ca2.4..10-instance_attributes']\
334 # /nvpair[@id='ip-2001.db8.ca2.4..10-instance_attributes-cidr_netmask']
335 vip_xpath_query="//resources/primitive[@type='IPaddr2']/instance_attributes/nvpair[@name='cidr_netmask' and @value='64']"
336 vip_xpath_xml_addresses=$(cibadmin --query --xpath "$vip_xpath_query" -e 2>/dev/null)
337 # The following extracts the @id value of the resource
338 vip_resources_to_fix=$(echo -e "$vip_xpath_xml_addresses" | sed -n "s/.*primitive\[@id='\([^']*\)'.*/\1/p")
339 # Runnning this in a subshell so that sourcing files cannot possibly affect the running script
341 OCF_PATH="/usr/lib/ocf/lib/heartbeat"
342 if [ -n "$vip_resources_to_fix" -a -f $OCF_PATH/ocf-shellfuncs -a -f $OCF_PATH/findif.sh ]; then
343 source $OCF_PATH/ocf-shellfuncs
344 source $OCF_PATH/findif.sh
345 for resource in $vip_resources_to_fix; do
346 echo "Updating IPv6 VIP $resource with a /128 and a correct addrlabel"
347 # The following will give us something like:
348 # <nvpair id="ip-2001.db8.ca2.4..10-instance_attributes-ip" name="ip" value="2001:db8:ca2:4::10"/>
349 ip_cib_nvpair=$(cibadmin --query --xpath "//resources/primitive[@type='IPaddr2' and @id='$resource']/instance_attributes/nvpair[@name='ip']")
350 # Let's filter out the value of the nvpair to get the ip address
351 ip_address=$(echo $ip_cib_nvpair | xmllint --xpath 'string(//nvpair/@value)' -)
352 OCF_RESKEY_cidr_netmask="64"
353 OCF_RESKEY_ip="$ip_address"
354 # Unfortunately due to https://bugzilla.redhat.com/show_bug.cgi?id=1445628
355 # we need to find out the appropiate nic given the ip address.
356 nic=$(findif $ip_address | awk '{ print $1 }')
358 if [ -z "$nic" -o $ret -ne 0 ]; then
359 echo "NIC autodetection failed for VIP $ip_address, not updating VIPs"
360 # Only exits the subshell
363 ocf_run -info pcs resource update --wait "$resource" ip="$ip_address" cidr_netmask=128 nic="$nic" lvs_ipv6_addrlabel=true lvs_ipv6_addrlabel_value=99
365 if [ $ret -ne 0 ]; then
366 echo "pcs resource update for VIP $resource failed, not updating VIPs"
367 # Only exits the subshell