Merge "Ensure access to Redis is password protected"
[apex-tripleo-heat-templates.git] / puppet / manifests / overcloud_controller_pacemaker.pp
index fd9b3b7..402a3bc 100644 (file)
@@ -39,7 +39,9 @@ $non_pcmk_start = hiera('step') >= 4
 
 if hiera('step') >= 1 {
 
+  create_resources(kmod::load, hiera('kernel_modules'), {})
   create_resources(sysctl::value, hiera('sysctl_settings'), {})
+  Exec <| tag == 'kmod::load' |>  -> Sysctl <| |>
 
   include ::timezone
 
@@ -60,6 +62,12 @@ if hiera('step') >= 1 {
   }
 
   $pacemaker_cluster_members = downcase(regsubst(hiera('controller_node_names'), ',', ' ', 'G'))
+  $corosync_ipv6 = str2bool(hiera('corosync_ipv6', false))
+  if $corosync_ipv6 {
+    $cluster_setup_extras = { '--ipv6' => '' }
+  } else {
+    $cluster_setup_extras = {}
+  }
   user { 'hacluster':
     ensure => present,
   } ->
@@ -67,8 +75,9 @@ if hiera('step') >= 1 {
     hacluster_pwd => hiera('hacluster_pwd'),
   } ->
   class { '::pacemaker::corosync':
-    cluster_members => $pacemaker_cluster_members,
-    setup_cluster   => $pacemaker_master,
+    cluster_members      => $pacemaker_cluster_members,
+    setup_cluster        => $pacemaker_master,
+    cluster_setup_extras => $cluster_setup_extras,
   }
   class { '::pacemaker::stonith':
     disable => !$enable_fencing,
@@ -91,12 +100,21 @@ if hiera('step') >= 1 {
   # avoid races where non-master nodes attempt to start without
   # config (eg. binding on 0.0.0.0)
   # The module ignores erlang_cookie if cluster_config is false
+  $rabbit_ipv6 = str2bool(hiera('rabbit_ipv6', false))
+  if $rabbit_ipv6 {
+      $rabbit_env = merge(hiera('rabbitmq_environment'), {
+        'RABBITMQ_SERVER_START_ARGS' => '"-proto_dist inet6_tcp"'
+      })
+  } else {
+    $rabbit_env = hiera('rabbitmq_environment')
+  }
+
   class { '::rabbitmq':
     service_manage          => false,
     tcp_keepalive           => false,
     config_kernel_variables => hiera('rabbitmq_kernel_variables'),
     config_variables        => hiera('rabbitmq_config_variables'),
-    environment_variables   => hiera('rabbitmq_environment'),
+    environment_variables   => $rabbit_env,
   } ->
   file { '/var/lib/rabbitmq/.erlang.cookie':
     ensure  => file,
@@ -134,6 +152,11 @@ if hiera('step') >= 1 {
   $galera_nodes = downcase(hiera('galera_node_names', $::hostname))
   $galera_nodes_count = count(split($galera_nodes, ','))
 
+  # FIXME: due to https://bugzilla.redhat.com/show_bug.cgi?id=1298671 we
+  # set bind-address to a hostname instead of an ip address; to move Mysql
+  # from internal_api on another network we'll have to customize both
+  # MysqlNetwork and ControllerHostnameResolveNetwork in ServiceNetMap
+  $mysql_bind_host = hiera('mysql_bind_host')
   $mysqld_options = {
     'mysqld' => {
       'skip-name-resolve'             => '1',
@@ -143,7 +166,7 @@ if hiera('step') >= 1 {
       'innodb_locks_unsafe_for_binlog'=> '1',
       'query_cache_size'              => '0',
       'query_cache_type'              => '0',
-      'bind-address'                  => hiera('mysql_bind_host'),
+      'bind-address'                  => $::hostname,
       'max_connections'               => hiera('mysql_max_connections'),
       'open_files_limit'              => '-1',
       'wsrep_provider'                => '/usr/lib64/galera/libgalera_smm.so',
@@ -159,6 +182,7 @@ if hiera('step') >= 1 {
       'wsrep_drupal_282555_workaround'=> '0',
       'wsrep_causal_reads'            => '0',
       'wsrep_sst_method'              => 'rsync',
+      'wsrep_provider_options'        => "gmcast.listen_addr=tcp://[${mysql_bind_host}]:4567;",
     },
   }
 
@@ -177,8 +201,19 @@ if hiera('step') >= 1 {
 if hiera('step') >= 2 {
 
   # NOTE(gfidente): the following vars are needed on all nodes so they
-  # need to stay out of pacemaker_master conditional
-  $mongo_node_ips_with_port = suffix(hiera('mongo_node_ips'), ':27017')
+  # need to stay out of pacemaker_master conditional.
+  # The addresses mangling will hopefully go away when we'll be able to
+  # configure the connection string via hostnames, until then, we need to pass
+  # the list of IPv6 addresses *with* port and without the brackets as 'members'
+  # argument for the 'mongodb_replset' resource.
+  if str2bool(hiera('mongodb::server::ipv6', false)) {
+    $mongo_node_ips_with_port_prefixed = prefix(hiera('mongo_node_ips'), '[')
+    $mongo_node_ips_with_port = suffix($mongo_node_ips_with_port_prefixed, ']:27017')
+    $mongo_node_ips_with_port_nobr = suffix(hiera('mongo_node_ips'), ':27017')
+  } else {
+    $mongo_node_ips_with_port = suffix(hiera('mongo_node_ips'), ':27017')
+    $mongo_node_ips_with_port_nobr = suffix(hiera('mongo_node_ips'), ':27017')
+  }
   $mongodb_replset = hiera('mongodb::server::replset')
 
   if $pacemaker_master {
@@ -187,6 +222,11 @@ if hiera('step') >= 2 {
 
       include ::pacemaker::resource_defaults
 
+      # Create an openstack-core dummy resource. See RHBZ 1290121
+      pacemaker::resource::ocf { 'openstack-core':
+        ocf_agent_name => 'heartbeat:Dummy',
+        clone_params   => true,
+      }
       # FIXME: we should not have to access tripleo::loadbalancer class
       # parameters here to configure pacemaker VIPs. The configuration
       # of pacemaker VIPs could move into puppet-tripleo or we should
@@ -196,8 +236,14 @@ if hiera('step') >= 2 {
       }
 
       $control_vip = hiera('tripleo::loadbalancer::controller_virtual_ip')
+      if is_ipv6_address($control_vip) {
+        $control_vip_netmask = '64'
+      } else {
+        $control_vip_netmask = '32'
+      }
       pacemaker::resource::ip { 'control_vip':
-        ip_address => $control_vip,
+        ip_address   => $control_vip,
+        cidr_netmask => $control_vip_netmask,
       }
       pacemaker::constraint::base { 'control_vip-then-haproxy':
         constraint_type   => 'order',
@@ -218,9 +264,15 @@ if hiera('step') >= 2 {
       }
 
       $public_vip = hiera('tripleo::loadbalancer::public_virtual_ip')
+      if is_ipv6_address($public_vip) {
+        $public_vip_netmask = '64'
+      } else {
+        $public_vip_netmask = '32'
+      }
       if $public_vip and $public_vip != $control_vip {
         pacemaker::resource::ip { 'public_vip':
-          ip_address => $public_vip,
+          ip_address   => $public_vip,
+          cidr_netmask => $public_vip_netmask,
         }
         pacemaker::constraint::base { 'public_vip-then-haproxy':
           constraint_type   => 'order',
@@ -242,9 +294,15 @@ if hiera('step') >= 2 {
       }
 
       $redis_vip = hiera('redis_vip')
+      if is_ipv6_address($redis_vip) {
+        $redis_vip_netmask = '64'
+      } else {
+        $redis_vip_netmask = '32'
+      }
       if $redis_vip and $redis_vip != $control_vip {
         pacemaker::resource::ip { 'redis_vip':
-          ip_address => $redis_vip,
+          ip_address   => $redis_vip,
+          cidr_netmask => $redis_vip_netmask,
         }
         pacemaker::constraint::base { 'redis_vip-then-haproxy':
           constraint_type   => 'order',
@@ -266,9 +324,15 @@ if hiera('step') >= 2 {
       }
 
       $internal_api_vip = hiera('tripleo::loadbalancer::internal_api_virtual_ip')
+      if is_ipv6_address($internal_api_vip) {
+        $internal_api_vip_netmask = '64'
+      } else {
+        $internal_api_vip_netmask = '32'
+      }
       if $internal_api_vip and $internal_api_vip != $control_vip {
         pacemaker::resource::ip { 'internal_api_vip':
-          ip_address => $internal_api_vip,
+          ip_address   => $internal_api_vip,
+          cidr_netmask => $internal_api_vip_netmask,
         }
         pacemaker::constraint::base { 'internal_api_vip-then-haproxy':
           constraint_type   => 'order',
@@ -290,9 +354,15 @@ if hiera('step') >= 2 {
       }
 
       $storage_vip = hiera('tripleo::loadbalancer::storage_virtual_ip')
+      if is_ipv6_address($storage_vip) {
+        $storage_vip_netmask = '64'
+      } else {
+        $storage_vip_netmask = '32'
+      }
       if $storage_vip and $storage_vip != $control_vip {
         pacemaker::resource::ip { 'storage_vip':
-          ip_address => $storage_vip,
+          ip_address   => $storage_vip,
+          cidr_netmask => $storage_vip_netmask,
         }
         pacemaker::constraint::base { 'storage_vip-then-haproxy':
           constraint_type   => 'order',
@@ -314,9 +384,15 @@ if hiera('step') >= 2 {
       }
 
       $storage_mgmt_vip = hiera('tripleo::loadbalancer::storage_mgmt_virtual_ip')
+      if is_ipv6_address($storage_mgmt_vip) {
+        $storage_mgmt_vip_netmask = '64'
+      } else {
+        $storage_mgmt_vip_netmask = '32'
+      }
       if $storage_mgmt_vip and $storage_mgmt_vip != $control_vip {
         pacemaker::resource::ip { 'storage_mgmt_vip':
-          ip_address => $storage_mgmt_vip,
+          ip_address   => $storage_mgmt_vip,
+          cidr_netmask => $storage_mgmt_vip_netmask,
         }
         pacemaker::constraint::base { 'storage_mgmt_vip-then-haproxy':
           constraint_type   => 'order',
@@ -348,6 +424,7 @@ if hiera('step') >= 2 {
       ocf_agent_name  => 'heartbeat:rabbitmq-cluster',
       resource_params => 'set_policy=\'ha-all ^(?!amq\.).* {"ha-mode":"all"}\'',
       clone_params    => 'ordered=true interleave=true',
+      meta_params     => 'notify=true',
       require         => Class['::rabbitmq'],
     }
 
@@ -365,7 +442,7 @@ if hiera('step') >= 2 {
         before  => Mongodb_replset[$mongodb_replset],
       }
       mongodb_replset { $mongodb_replset :
-        members => $mongo_node_ips_with_port,
+        members => $mongo_node_ips_with_port_nobr,
       }
     }
 
@@ -460,8 +537,15 @@ MYSQL_HOST=localhost\n",
   $enable_ceph = hiera('ceph_storage_count', 0) > 0 or hiera('enable_ceph_storage', false)
 
   if $enable_ceph {
+    $mon_initial_members = downcase(hiera('ceph_mon_initial_members'))
+    if str2bool(hiera('ceph_ipv6', false)) {
+      $mon_host = hiera('ceph_mon_host_v6')
+    } else {
+      $mon_host = hiera('ceph_mon_host')
+    }
     class { '::ceph::profile::params':
-      mon_initial_members => downcase(hiera('ceph_mon_initial_members')),
+      mon_initial_members => $mon_initial_members,
+      mon_host            => $mon_host,
     }
     include ::ceph::conf
     include ::ceph::profile::mon
@@ -487,6 +571,14 @@ MYSQL_HOST=localhost\n",
   }
 
   if str2bool(hiera('enable_external_ceph', false)) {
+    if str2bool(hiera('ceph_ipv6', false)) {
+      $mon_host = hiera('ceph_mon_host_v6')
+    } else {
+      $mon_host = hiera('ceph_mon_host')
+    }
+    class { '::ceph::profile::params':
+      mon_host            => $mon_host,
+    }
     include ::ceph::conf
     include ::ceph::profile::client
   }
@@ -497,9 +589,10 @@ MYSQL_HOST=localhost\n",
 if hiera('step') >= 3 {
 
   class { '::keystone':
-    sync_db        => $sync_db,
-    manage_service => false,
-    enabled        => false,
+    sync_db          => $sync_db,
+    manage_service   => false,
+    enabled          => false,
+    enable_bootstrap => $pacemaker_master,
   }
   include ::keystone::config
 
@@ -567,10 +660,18 @@ if hiera('step') >= 3 {
     manage_service => false,
     enabled        => false,
   }
+  include ::glance::notify::rabbitmq
   include join(['::glance::backend::', $glance_backend])
 
+  $nova_ipv6 = hiera('nova::use_ipv6', false)
+  if $nova_ipv6 {
+    $memcached_servers = suffix(hiera('memcache_node_ips_v6'), ':11211')
+  } else {
+    $memcached_servers = suffix(hiera('memcache_node_ips'), ':11211')
+  }
+
   class { '::nova' :
-    memcached_servers => suffix(hiera('memcache_node_ips'), ':11211'),
+    memcached_servers => $memcached_servers
   }
 
   include ::nova::config
@@ -665,6 +766,9 @@ if hiera('step') >= 3 {
   if  hiera('neutron::core_plugin') == 'neutron.plugins.nuage.plugin.NuagePlugin' {
     include ::neutron::plugins::nuage
   }
+  if  hiera('neutron::core_plugin') == 'neutron_plugin_contrail.plugins.opencontrail.contrail_plugin.NeutronPluginContrailCoreV2' {
+    include ::neutron::plugins::opencontrail
+  }
   if hiera('neutron::core_plugin') == 'midonet.neutron.plugin_v1.MidonetPluginV2' {
     class {'::neutron::plugins::midonet':
       midonet_api_ip    => hiera('tripleo::loadbalancer::public_virtual_ip'),
@@ -724,8 +828,9 @@ if hiera('step') >= 3 {
     }
   }
 
-  if hiera('neutron_enable_bigswitch_ml2', false) {
+  if 'bsn_ml2' in hiera('neutron::plugins::ml2::mechanism_drivers') {
     include ::neutron::plugins::ml2::bigswitch::restproxy
+    include ::neutron::agents::bigswitch
   }
   neutron_l3_agent_config {
     'DEFAULT/ovs_use_veth': value => hiera('neutron_ovs_use_veth', false);
@@ -733,6 +838,9 @@ if hiera('step') >= 3 {
   neutron_dhcp_agent_config {
     'DEFAULT/ovs_use_veth': value => hiera('neutron_ovs_use_veth', false);
   }
+  neutron_config {
+    'DEFAULT/notification_driver': value => 'messaging';
+  }
 
   include ::cinder
   include ::cinder::config
@@ -751,6 +859,7 @@ if hiera('step') >= 3 {
     enabled        => false,
   }
   include ::cinder::glance
+  include ::cinder::ceilometer
   class { '::cinder::setup_test_volume':
     size => join([hiera('cinder_lvm_loop_device_size'), 'M']),
   }
@@ -876,7 +985,7 @@ if hiera('step') >= 3 {
 
   $cinder_enabled_backends = delete_undef_values([$cinder_iscsi_backend, $cinder_rbd_backend, $cinder_eqlx_backend, $cinder_dellsc_backend, $cinder_netapp_backend, $cinder_nfs_backend])
   class { '::cinder::backends' :
-    enabled_backends => $cinder_enabled_backends,
+    enabled_backends => union($cinder_enabled_backends, hiera('cinder_user_enabled_backends')),
   }
 
   class { '::sahara':
@@ -977,7 +1086,8 @@ if hiera('step') >= 3 {
   # Heat
   include ::heat::config
   class { '::heat' :
-    sync_db => $sync_db,
+    sync_db             => $sync_db,
+    notification_driver => 'messaging',
   }
   class { '::heat::api' :
     manage_service => false,
@@ -1022,7 +1132,7 @@ if hiera('step') >= 3 {
   }
   class { '::snmp':
     agentaddress => ['udp:161','udp6:[::1]:161'],
-    snmpd_config => [ join(['rouser ', hiera('snmpd_readonly_user_name')]), 'proc  cron', 'includeAllDisks  10%', 'master agentx', 'trapsink localhost public', 'iquerySecName internalUser', 'rouser internalUser', 'defaultMonitors yes', 'linkUpDownNotifications yes' ],
+    snmpd_config => [ join(['createUser ', hiera('snmpd_readonly_user_name'), ' MD5 "', hiera('snmpd_readonly_user_password'), '"']), join(['rouser ', hiera('snmpd_readonly_user_name')]), 'proc  cron', 'includeAllDisks  10%', 'master agentx', 'trapsink localhost public', 'iquerySecName internalUser', 'rouser internalUser', 'defaultMonitors yes', 'linkUpDownNotifications yes' ],
   }
 
   hiera_include('controller_classes')
@@ -1033,6 +1143,7 @@ if hiera('step') >= 4 {
   $keystone_enable_db_purge = hiera('keystone_enable_db_purge', true)
   $nova_enable_db_purge = hiera('nova_enable_db_purge', true)
   $cinder_enable_db_purge = hiera('cinder_enable_db_purge', true)
+  $heat_enable_db_purge = hiera('heat_enable_db_purge', true)
 
   if $keystone_enable_db_purge {
     include ::keystone::cron::token_flush
@@ -1043,6 +1154,9 @@ if hiera('step') >= 4 {
   if $cinder_enable_db_purge {
     include ::cinder::cron::db_purge
   }
+  if $heat_enable_db_purge {
+    include ::heat::cron::purge_deleted
+  }
 
   if $pacemaker_master {
 
@@ -1050,45 +1164,55 @@ if hiera('step') >= 4 {
       pacemaker::constraint::base { 'haproxy-then-keystone-constraint':
         constraint_type => 'order',
         first_resource  => 'haproxy-clone',
-        second_resource => "${::apache::params::service_name}-clone",
+        second_resource => 'openstack-core-clone',
         first_action    => 'start',
         second_action   => 'start',
         require         => [Pacemaker::Resource::Service['haproxy'],
-                            Pacemaker::Resource::Service[$::apache::params::service_name]],
+                            Pacemaker::Resource::Ocf['openstack-core']],
       }
     }
+
+    pacemaker::constraint::base { 'openstack-core-then-httpd-constraint':
+      constraint_type => 'order',
+      first_resource  => 'openstack-core-clone',
+      second_resource => "${::apache::params::service_name}-clone",
+      first_action    => 'start',
+      second_action   => 'start',
+      require         => [Pacemaker::Resource::Service[$::apache::params::service_name],
+                          Pacemaker::Resource::Ocf['openstack-core']],
+    }
     pacemaker::constraint::base { 'rabbitmq-then-keystone-constraint':
       constraint_type => 'order',
       first_resource  => 'rabbitmq-clone',
-      second_resource => "${::apache::params::service_name}-clone",
+      second_resource => 'openstack-core-clone',
       first_action    => 'start',
       second_action   => 'start',
       require         => [Pacemaker::Resource::Ocf['rabbitmq'],
-                          Pacemaker::Resource::Service[$::apache::params::service_name]],
+                          Pacemaker::Resource::Ocf['openstack-core']],
     }
-    pacemaker::constraint::base { 'memcached-then-keystone-constraint':
+    pacemaker::constraint::base { 'memcached-then-openstack-core-constraint':
       constraint_type => 'order',
       first_resource  => 'memcached-clone',
-      second_resource => "${::apache::params::service_name}-clone",
+      second_resource => 'openstack-core-clone',
       first_action    => 'start',
       second_action   => 'start',
       require         => [Pacemaker::Resource::Service['memcached'],
-                          Pacemaker::Resource::Service[$::apache::params::service_name]],
+                          Pacemaker::Resource::Ocf['openstack-core']],
     }
-    pacemaker::constraint::base { 'galera-then-keystone-constraint':
+    pacemaker::constraint::base { 'galera-then-openstack-core-constraint':
       constraint_type => 'order',
       first_resource  => 'galera-master',
-      second_resource => "${::apache::params::service_name}-clone",
+      second_resource => 'openstack-core-clone',
       first_action    => 'promote',
       second_action   => 'start',
       require         => [Pacemaker::Resource::Ocf['galera'],
-                          Pacemaker::Resource::Service[$::apache::params::service_name]],
+                          Pacemaker::Resource::Ocf['openstack-core']],
     }
 
     # Cinder
     pacemaker::resource::service { $::cinder::params::api_service :
       clone_params => 'interleave=true',
-      require      => Pacemaker::Resource::Service[$::apache::params::service_name],
+      require      => Pacemaker::Resource::Ocf['openstack-core'],
     }
     pacemaker::resource::service { $::cinder::params::scheduler_service :
       clone_params => 'interleave=true',
@@ -1097,12 +1221,12 @@ if hiera('step') >= 4 {
 
     pacemaker::constraint::base { 'keystone-then-cinder-api-constraint':
       constraint_type => 'order',
-      first_resource  => "${::apache::params::service_name}-clone",
+      first_resource  => 'openstack-core-clone',
       second_resource => "${::cinder::params::api_service}-clone",
       first_action    => 'start',
       second_action   => 'start',
-      require         => [Pacemaker::Resource::Service[$::cinder::params::api_service],
-                          Pacemaker::Resource::Service[$::apache::params::service_name]],
+      require         => [Pacemaker::Resource::Ocf['openstack-core'],
+                          Pacemaker::Resource::Service[$::cinder::params::api_service]],
     }
     pacemaker::constraint::base { 'cinder-api-then-cinder-scheduler-constraint':
       constraint_type => 'order',
@@ -1140,25 +1264,25 @@ if hiera('step') >= 4 {
     # Sahara
     pacemaker::resource::service { $::sahara::params::api_service_name :
       clone_params => 'interleave=true',
-      require      => Pacemaker::Resource::Service[$::apache::params::service_name],
+      require      => Pacemaker::Resource::Ocf['openstack-core'],
     }
     pacemaker::resource::service { $::sahara::params::engine_service_name :
       clone_params => 'interleave=true',
     }
     pacemaker::constraint::base { 'keystone-then-sahara-api-constraint':
       constraint_type => 'order',
-      first_resource  => "${::apache::params::service_name}-clone",
+      first_resource  => 'openstack-core-clone',
       second_resource => "${::sahara::params::api_service_name}-clone",
       first_action    => 'start',
       second_action   => 'start',
       require         => [Pacemaker::Resource::Service[$::sahara::params::api_service_name],
-                          Pacemaker::Resource::Service[$::apache::params::service_name]],
+                          Pacemaker::Resource::Ocf['openstack-core']],
     }
 
     # Glance
     pacemaker::resource::service { $::glance::params::registry_service_name :
       clone_params => 'interleave=true',
-      require      => Pacemaker::Resource::Service[$::apache::params::service_name],
+      require      => Pacemaker::Resource::Ocf['openstack-core'],
     }
     pacemaker::resource::service { $::glance::params::api_service_name :
       clone_params => 'interleave=true',
@@ -1166,12 +1290,12 @@ if hiera('step') >= 4 {
 
     pacemaker::constraint::base { 'keystone-then-glance-registry-constraint':
       constraint_type => 'order',
-      first_resource  => "${::apache::params::service_name}-clone",
+      first_resource  => 'openstack-core-clone',
       second_resource => "${::glance::params::registry_service_name}-clone",
       first_action    => 'start',
       second_action   => 'start',
       require         => [Pacemaker::Resource::Service[$::glance::params::registry_service_name],
-                          Pacemaker::Resource::Service[$::apache::params::service_name]],
+                          Pacemaker::Resource::Ocf['openstack-core']],
     }
     pacemaker::constraint::base { 'glance-registry-then-glance-api-constraint':
       constraint_type => 'order',
@@ -1209,12 +1333,12 @@ if hiera('step') >= 4 {
       } ->
       pacemaker::resource::service { $::neutron::params::server_service:
         clone_params => 'interleave=true',
-        require      => Pacemaker::Resource::Service[$::apache::params::service_name]
+        require      => Pacemaker::Resource::Ocf['openstack-core']
       }
     } else {
       pacemaker::resource::service { $::neutron::params::server_service:
         clone_params => 'interleave=true',
-        require      => Pacemaker::Resource::Service[$::apache::params::service_name]
+        require      => Pacemaker::Resource::Ocf['openstack-core']
       }
     }
     if hiera('neutron::enable_l3_agent', true) {
@@ -1287,14 +1411,14 @@ if hiera('step') >= 4 {
       }
     }
     pacemaker::constraint::base { 'keystone-to-neutron-server-constraint':
-      constraint_type   => 'order',
-      first_resource    => "${::apache::params::service_name}-clone",
-        second_resource => "${::neutron::params::server_service}-clone",
-        first_action    => 'start',
-        second_action   => 'start',
-        require         => [Pacemaker::Resource::Service[$::apache::params::service_name],
-                            Pacemaker::Resource::Service[$::neutron::params::server_service]],
-      }
+      constraint_type => 'order',
+      first_resource  => 'openstack-core-clone',
+      second_resource => "${::neutron::params::server_service}-clone",
+      first_action    => 'start',
+      second_action   => 'start',
+      require         => [Pacemaker::Resource::Ocf['openstack-core'],
+                          Pacemaker::Resource::Service[$::neutron::params::server_service]],
+    }
     if hiera('neutron::enable_ovs_agent',true) {
       pacemaker::constraint::base { 'neutron-openvswitch-agent-to-dhcp-agent-constraint':
         constraint_type => 'order',
@@ -1402,34 +1526,29 @@ if hiera('step') >= 4 {
     # Nova
     pacemaker::resource::service { $::nova::params::api_service_name :
       clone_params => 'interleave=true',
-      op_params    => 'start timeout=200s stop timeout=200s monitor start-delay=10s',
     }
     pacemaker::resource::service { $::nova::params::conductor_service_name :
       clone_params => 'interleave=true',
-      op_params    => 'start timeout=200s stop timeout=200s monitor start-delay=10s',
     }
     pacemaker::resource::service { $::nova::params::consoleauth_service_name :
       clone_params => 'interleave=true',
-      op_params    => 'start timeout=200s stop timeout=200s monitor start-delay=10s',
-      require      => Pacemaker::Resource::Service[$::apache::params::service_name],
+      require      => Pacemaker::Resource::Ocf['openstack-core'],
     }
     pacemaker::resource::service { $::nova::params::vncproxy_service_name :
       clone_params => 'interleave=true',
-      op_params    => 'start timeout=200s stop timeout=200s monitor start-delay=10s',
     }
     pacemaker::resource::service { $::nova::params::scheduler_service_name :
       clone_params => 'interleave=true',
-      op_params    => 'start timeout=200s stop timeout=200s monitor start-delay=10s',
     }
 
     pacemaker::constraint::base { 'keystone-then-nova-consoleauth-constraint':
       constraint_type => 'order',
-      first_resource  => "${::apache::params::service_name}-clone",
+      first_resource  => 'openstack-core-clone',
       second_resource => "${::nova::params::consoleauth_service_name}-clone",
       first_action    => 'start',
       second_action   => 'start',
       require         => [Pacemaker::Resource::Service[$::nova::params::consoleauth_service_name],
-                          Pacemaker::Resource::Service[$::apache::params::service_name]],
+                          Pacemaker::Resource::Ocf['openstack-core']],
     }
     pacemaker::constraint::base { 'nova-consoleauth-then-nova-vncproxy-constraint':
       constraint_type => 'order',
@@ -1499,16 +1618,16 @@ if hiera('step') >= 4 {
     # Ceilometer
     case downcase(hiera('ceilometer_backend')) {
       /mysql/: {
-        pacemaker::resource::service { $::ceilometer::params::agent_central_service_name :
+        pacemaker::resource::service { $::ceilometer::params::agent_central_service_name:
           clone_params => 'interleave=true',
-          require      => Pacemaker::Resource::Service[$::apache::params::service_name],
+          require      => Pacemaker::Resource::Ocf['openstack-core'],
         }
       }
       default: {
-        pacemaker::resource::service { $::ceilometer::params::agent_central_service_name :
+        pacemaker::resource::service { $::ceilometer::params::agent_central_service_name:
           clone_params => 'interleave=true',
-          require      => [Pacemaker::Resource::Service[$::apache::params::service_name],
-          Pacemaker::Resource::Service[$::mongodb::params::service_name]],
+          require      => [Pacemaker::Resource::Ocf['openstack-core'],
+                          Pacemaker::Resource::Service[$::mongodb::params::service_name]],
         }
       }
     }
@@ -1544,12 +1663,12 @@ if hiera('step') >= 4 {
     }
     pacemaker::constraint::base { 'keystone-then-ceilometer-central-constraint':
       constraint_type => 'order',
-      first_resource  => "${::apache::params::service_name}-clone",
+      first_resource  => 'openstack-core-clone',
       second_resource => "${::ceilometer::params::agent_central_service_name}-clone",
       first_action    => 'start',
       second_action   => 'start',
       require         => [Pacemaker::Resource::Service[$::ceilometer::params::agent_central_service_name],
-                          Pacemaker::Resource::Service[$::apache::params::service_name]],
+                          Pacemaker::Resource::Ocf['openstack-core']],
     }
     pacemaker::constraint::base { 'ceilometer-central-then-ceilometer-collector-constraint':
       constraint_type => 'order',
@@ -1619,12 +1738,12 @@ if hiera('step') >= 4 {
     }
     pacemaker::constraint::base { 'keystone-then-heat-api-constraint':
       constraint_type => 'order',
-      first_resource  => "${::apache::params::service_name}-clone",
+      first_resource  => 'openstack-core-clone',
       second_resource => "${::heat::params::api_service_name}-clone",
       first_action    => 'start',
       second_action   => 'start',
       require         => [Pacemaker::Resource::Service[$::heat::params::api_service_name],
-                          Pacemaker::Resource::Service[$::apache::params::service_name]],
+                          Pacemaker::Resource::Ocf['openstack-core']],
     }
     pacemaker::constraint::base { 'heat-api-then-heat-api-cfn-constraint':
       constraint_type => 'order',
@@ -1732,6 +1851,16 @@ if hiera('step') >= 5 {
     class {'::keystone::endpoint' :
       require => Pacemaker::Resource::Service[$::apache::params::service_name],
     }
+    include ::heat::keystone::domain
+    Class['::keystone::roles::admin'] -> Class['::heat::keystone::domain']
+
+  } else {
+    # On non-master controller we don't need to create Keystone resources again
+    class { '::heat::keystone::domain':
+      manage_domain => false,
+      manage_user   => false,
+      manage_role   => false,
+    }
   }
 
 } #END STEP 5