puppet controller role: per service VIP settings

This patch refactors the puppet controller role so that it
makes use of per service VIP settings for each service.

Previously the VIP for the ctlplane was hard wired to
many of the controller service. With this patch we have
the ability to isolate traffic for services which
made use of the ctlplane and public VIPs for their
settings.

The implementation includes:

 * stops the use of the VirtualIP and PublicVirtualIP within the
   controller role. These parameters have now been replaced with
   per service heat parameters for the controller nested stack which
   are determined via VipMap based on per service settings in the heat
   environment.

 * All VIP configuration is now moved into puppet/vip-config.yaml.
   This made sense so we could deprecate the use of the VirtualIP
   and PublicVirtualIP settings above.

 * The puppet manifests for the controller were cleaned up for several
   to use Hiera directly instead of constructing URLs based on the
   static controller and public network VIPs. This improvement
   was something we wanted to do anyways and made the implementation
   cleaner.

Change-Id: I9b9a15be67f74bec97366408f7047acfd6ea0ec6

diff --git a/controller.yaml b/controller.yaml
index 4e541e9..8c1b93d 100644 (file)
--- a/controller.yaml
+++ b/controller.yaml
@@ -418,6 +418,21 @@ parameters:
   VirtualIP:
     type: string
     default: ''  # Has to be here because of the ignored empty value bug
+  HeatApiVirtualIP:
+    type: string
+    default: ''
+  GlanceApiVirtualIP:
+    type: string
+    default: ''
+  MysqlVirtualIP:
+    type: string
+    default: ''
+  KeystonePublicApiVirtualIP:
+    type: string
+    default: ''
+  NeutronApiVirtualIP:
+    type: string
+    default: ''
   ServiceNetMap:
     default: {}
     description: Mapping of service_name -> network name. Typically set

diff --git a/overcloud-without-mergepy.yaml b/overcloud-without-mergepy.yaml
index 4b969df..4b2e0b9 100644 (file)
--- a/overcloud-without-mergepy.yaml
+++ b/overcloud-without-mergepy.yaml
@@ -662,9 +662,14 @@ resources:
           SwiftPartPower: {get_param: SwiftPartPower}
           SwiftPassword: {get_param: SwiftPassword}
           SwiftReplicas: { get_param: SwiftReplicas}
-          VirtualIP: {get_attr: [ControlVirtualIP, fixed_ips, 0, ip_address]}
-          PublicVirtualIP: {get_attr: [PublicVirtualIP, fixed_ips, 0, ip_address]}
+          VirtualIP: {get_attr: [ControlVirtualIP, fixed_ips, 0, ip_address]} # deprecated. Use per service VIP settings instead now.
+          PublicVirtualIP: {get_attr: [PublicVirtualIP, fixed_ips, 0, ip_address]} # deprecated. Use per service VIP settings instead now.
           ServiceNetMap: {get_param: ServiceNetMap}
+          HeatApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, HeatApiNetwork]}]}
+          GlanceApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, GlanceApiNetwork]}]}
+          MysqlVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, MysqlNetwork]}]}
+          KeystonePublicApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, KeystonePublicApiNetwork]}]}
+          NeutronApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, NeutronApiNetwork]}]}
 
   Compute:
     type: OS::Heat::ResourceGroup
@@ -910,6 +915,8 @@ resources:
         mysql_vip: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, MysqlNetwork]}]}
         rabbit_vip: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, RabbitMqNetwork]}]}
         # direct configuration of Virtual IPs for each network
+        control_virtual_ip: {get_attr: [ControlVirtualIP, fixed_ips, 0, ip_address]}
+        public_virtual_ip: {get_attr: [PublicVirtualIP, fixed_ips, 0, ip_address]}
         internal_api_virtual_ip: {get_attr: [InternalApiVirtualIP, ip_address]}
         storage_virtual_ip: {get_attr: [StorageVirtualIP, ip_address]}
         storage_mgmt_virtual_ip: {get_attr: [StorageMgmtVirtualIP, ip_address]}

diff --git a/puppet/controller-puppet.yaml b/puppet/controller-puppet.yaml
index 0da10fc..aa33cbf 100644 (file)
--- a/puppet/controller-puppet.yaml
+++ b/puppet/controller-puppet.yaml
@@ -342,7 +342,7 @@ parameters:
         Specifies the interface where the public-facing virtual ip will be assigned.
         This should be int_public when a VLAN is being used.
     type: string
-  PublicVirtualIP:
+  PublicVirtualIP: # DEPRECATED: use per service settings instead
     type: string
     default: ''  # Has to be here because of the ignored empty value bug
   RabbitCookie:
@@ -419,9 +419,24 @@ parameters:
     type: number
     default: 3
     description: How many replicas to use in the swift rings.
-  VirtualIP:
+  VirtualIP: # DEPRECATED: use per service settings instead
     type: string
     default: ''  # Has to be here because of the ignored empty value bug
+  HeatApiVirtualIP:
+    type: string
+    default: ''
+  GlanceApiVirtualIP:
+    type: string
+    default: ''
+  MysqlVirtualIP:
+    type: string
+    default: ''
+  KeystonePublicApiVirtualIP:
+    type: string
+    default: ''
+  NeutronApiVirtualIP:
+    type: string
+    default: ''
   EnablePackageInstall:
     default: 'false'
     description: Set to true to enable package installation via Puppet
@@ -510,25 +525,24 @@ resources:
       server: {get_resource: Controller}
       input_values:
         bootstack_nodeid: {get_attr: [Controller, name]}
-        controller_virtual_ip: {get_param: VirtualIP}
         neutron_enable_tunneling: {get_param: NeutronEnableTunnelling}
         heat.watch_server_url:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: HeatApiVirtualIP}
               - ':8003'
         heat.metadata_server_url:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: HeatApiVirtualIP}
               - ':8000'
         heat.waitcondition_server_url:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: HeatApiVirtualIP}
               - ':8000/v1/waitcondition'
         heat_auth_encryption_key: {get_param: HeatAuthEncryptionKey}
         horizon_secret: {get_param: HorizonSecret}
@@ -547,13 +561,11 @@ resources:
             - - 'mysql://cinder:'
               - {get_param: CinderPassword}
               - '@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/cinder'
         glance_port: {get_param: GlancePort}
-        glance_protocol: {get_param: GlanceProtocol}
         glance_password: {get_param: GlancePassword}
         glance_backend: {get_param: GlanceBackend}
-        glance_swift_store_auth_address: {list_join: ['', ['http://', {get_param: VirtualIP} , ':5000/v2.0']]}
         glance_notifier_strategy: {get_param: GlanceNotifierStrategy}
         glance_log_file: {get_param: GlanceLogFile}
         glance_dsn:
@@ -562,7 +574,7 @@ resources:
             - - 'mysql://glance:'
               - {get_param: GlancePassword}
               - '@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/glance'
         heat_password: {get_param: HeatPassword}
         heat_stack_domain_admin_password: {get_param: HeatStackDomainAdminPassword}
@@ -572,8 +584,9 @@ resources:
             - - 'mysql://heat:'
               - {get_param: HeatPassword}
               - '@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/heat'
+        keystone_auth_address: {list_join: ['', ['http://', {get_param: KeystonePublicApiVirtualIP} , ':5000/v2.0']]}
         keystone_ca_certificate: {get_param: KeystoneCACertificate}
         keystone_signing_key: {get_param: KeystoneSigningKey}
         keystone_signing_certificate: {get_param: KeystoneSigningCertificate}
@@ -585,19 +598,19 @@ resources:
             - - 'mysql://keystone:'
               - {get_param: AdminToken}
               - '@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/keystone'
         keystone_identity_uri:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: KeystonePublicApiVirtualIP}
               - ':35357/'
         keystone_auth_uri:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: KeystonePublicApiVirtualIP}
               - ':5000/v2.0/'
         enable_galera: {get_param: EnableGalera}
         enable_ceph_storage: {get_param: EnableCephStorage}
@@ -632,19 +645,19 @@ resources:
             - - 'mysql://neutron:'
               - {get_param: NeutronPassword}
               - '@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/ovs_neutron?charset=utf8'
         neutron_url:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: NeutronApiVirtualIP}
               - ':9696'
         neutron_admin_auth_url:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: KeystonePublicApiVirtualIP}
               - ':35357/v2.0'
         ceilometer_backend: {get_param: CeilometerBackend}
         ceilometer_metering_secret: {get_param: CeilometerMeteringSecret}
@@ -653,7 +666,7 @@ resources:
           list_join:
             - ''
             - - 'mysql://ceilometer:unset@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/ceilometer'
         snmpd_readonly_user_name: {get_param: SnmpdReadonlyUserName}
         snmpd_readonly_user_password: {get_param: SnmpdReadonlyUserPassword}
@@ -664,7 +677,7 @@ resources:
             - - 'mysql://nova:'
               - {get_param: NovaPassword}
               - '@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/nova'
         pcsd_password: {get_param: PcsdPassword}
         rabbit_username: {get_param: RabbitUserName}
@@ -679,7 +692,6 @@ resources:
               server: {get_param: NtpServer}
         control_virtual_interface: {get_param: ControlVirtualInterface}
         public_virtual_interface: {get_param: PublicVirtualInterface}
-        public_virtual_ip: {get_param: PublicVirtualIP}
         swift_hash_suffix: {get_param: SwiftHashSuffix}
         swift_password: {get_param: SwiftPassword}
         swift_part_power: {get_param: SwiftPartPower}
@@ -693,6 +705,14 @@ resources:
         cinder_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, CinderApiNetwork]}]}
         glance_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, GlanceApiNetwork]}]}
         glance_registry_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, GlanceRegistryNetwork]}]}
+        glance_api_servers:
+          list_join:
+            - ''
+            - - {get_param: GlanceProtocol}
+              - '://'
+              - {get_param: GlanceApiVirtualIP}
+              - ':'
+              - {get_param: GlancePort}
         heat_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, HeatApiNetwork]}]}
         keystone_public_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, KeystonePublicApiNetwork]}]}
         keystone_admin_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, KeystoneAdminApiNetwork]}]}
@@ -724,6 +744,7 @@ resources:
             - ceph
             - bootstrap_node # provided by BootstrapNodeConfig
             - all_nodes # provided by allNodesConfig
+            - vip_data # provided by vip-config
             - '"%{::osfamily}"'
             - common
           datafiles:
@@ -782,9 +803,6 @@ resources:
                 glance::api::registry_host: {get_input: glance_registry_network}
                 glance::api::keystone_password: {get_input: glance_password}
                 glance::api::debug: {get_input: debug}
-                # used to construct glance_api_servers
-                glance_port: {get_input: glance_port}
-                glance_protocol: {get_input: glance_protocol}
                 glance_notifier_strategy: {get_input: glance_notifier_strategy}
                 glance_log_file: {get_input: glance_log_file}
                 glance_log_file: {get_input: glance_log_file}
@@ -795,7 +813,7 @@ resources:
                 glance::registry::auth_uri: {get_input: keystone_auth_uri}
                 glance::registry::identity_uri: {get_input: keystone_identity_uri}
                 glance::registry::debug: {get_input: debug}
-                glance::backend::swift::swift_store_auth_address: {get_input: glance_swift_store_auth_address}
+                glance::backend::swift::swift_store_auth_address: {get_input: keystone_auth_address}
                 glance::backend::swift::swift_store_user: service:glance
                 glance::backend::swift::swift_store_key: {get_input: glance_password}
                 glance_backend: {get_input: glance_backend}
@@ -857,7 +875,7 @@ resources:
                 neutron::agents::ml2::ovs::local_ip: {get_input: neutron_local_ip}
                 neutron_flat_networks: {get_input: neutron_flat_networks}
                 neutron::agents::metadata::shared_secret: {get_input: neutron_metadata_proxy_shared_secret}
-                neutron::agents::metadata::metadata_ip: {get_input: controller_virtual_ip}
+                neutron::agents::metadata::metadata_ip: {get_input: neutron_api_network}
                 neutron_agent_mode: {get_input: neutron_agent_mode}
                 neutron_router_distributed: {get_input: neutron_router_distributed}
                 neutron_mechanism_drivers: {get_input: neutron_mechanism_drivers}
@@ -875,6 +893,7 @@ resources:
                 neutron::agents::metadata::auth_password: {get_input: neutron_password}
                 neutron_dnsmasq_options: {get_input: neutron_dnsmasq_options}
                 neutron_dsn: {get_input: neutron_dsn}
+                neutron::agents::metadata::auth_url: {get_input: keystone_identity_uri}
 
                 # Ceilometer
                 ceilometer_backend: {get_input: ceilometer_backend}
@@ -890,6 +909,7 @@ resources:
                 ceilometer::api::keystone_auth_uri: {get_input: keystone_auth_uri}
                 ceilometer::api::keystone_identity_uri: {get_input: keystone_identity_uri}
                 ceilometer::agent::auth::auth_password: {get_input: ceilometer_password}
+                ceilometer::agent::auth::auth_url: {get_input: keystone_auth_address}
                 snmpd_readonly_user_name: {get_input: snmpd_readonly_user_name}
                 snmpd_readonly_user_password: {get_input: snmpd_readonly_user_password}
 
@@ -905,6 +925,7 @@ resources:
                 nova::api::metadata_listen: {get_input: nova_metadata_network}
                 nova::api::admin_password: {get_input: nova_password}
                 nova::database_connection: {get_input: nova_dsn}
+                nova::glance_api_servers: {get_input: glance_api_servers}
                 nova::api::neutron_metadata_proxy_shared_secret: {get_input: neutron_metadata_proxy_shared_secret}
                 nova::network::neutron::neutron_admin_password: {get_input: neutron_password}
                 nova::network::neutron::neutron_url: {get_input: neutron_url}
@@ -926,13 +947,9 @@ resources:
                 neutron_public_interface_ip: {get_input: neutron_public_interface_ip}
                 ntp::servers: {get_input: ntp_servers}
                 control_virtual_interface: {get_input: control_virtual_interface}
-                controller_virtual_ip: {get_input: controller_virtual_ip}
                 public_virtual_interface: {get_input: public_virtual_interface}
-                public_virtual_ip: {get_input: public_virtual_ip}
                 tripleo::loadbalancer::control_virtual_interface: {get_input: control_virtual_interface}
-                tripleo::loadbalancer::controller_virtual_ip: {get_input: controller_virtual_ip}
                 tripleo::loadbalancer::public_virtual_interface: {get_input: public_virtual_interface}
-                tripleo::loadbalancer::public_virtual_ip: {get_input: public_virtual_ip}
                 enable_package_install: {get_input: enable_package_install}
 
 outputs:
@@ -984,11 +1001,11 @@ outputs:
       str_replace:
         template: 'r1z1-IP:%PORT%/d1'
         params:
-          IP: {get_attr: [Controller, networks, ctlplane, 0]}
+          IP: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftMgmtNetwork]}]}
   swift_proxy_memcache:
     description: Swift proxy-memcache value
     value:
       str_replace:
         template: "IP:11211"
         params:
-          IP: {get_attr: [Controller, networks, ctlplane, 0]}
+          IP: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, MemcachedNetwork]}]}

diff --git a/puppet/manifests/overcloud_controller.pp b/puppet/manifests/overcloud_controller.pp
index 9316ae1..7070e05 100644 (file)
--- a/puppet/manifests/overcloud_controller.pp
+++ b/puppet/manifests/overcloud_controller.pp
@@ -76,9 +76,7 @@ if hiera('step') >= 2 {
   if count($redis_node_ips) > 1 {
     Class['::tripleo::redis_notification'] -> Service['redis-sentinel']
     include ::redis::sentinel
-    class {'::tripleo::redis_notification' :
-      haproxy_monitor_ip => hiera('tripleo::loadbalancer::controller_virtual_ip'),
-    }
+    include ::tripleo::redis_notification
   }
 
   if str2bool(hiera('enable_galera', 'true')) {
@@ -253,10 +251,7 @@ if hiera('step') >= 3 {
   include ::glance::registry
   include join(['::glance::backend::', $glance_backend])
 
-  class { 'nova':
-    glance_api_servers     => join([hiera('glance_protocol'), '://', hiera('controller_virtual_ip'), ':', hiera('glance_port')]),
-  }
-
+  include ::nova
   include ::nova::api
   include ::nova::cert
   include ::nova::conductor
@@ -269,9 +264,7 @@ if hiera('step') >= 3 {
   include ::neutron::server
   include ::neutron::agents::l3
   include ::neutron::agents::dhcp
-  class { 'neutron::agents::metadata':
-    auth_url => join(['http://', hiera('controller_virtual_ip'), ':35357/v2.0']),
-  }
+  include ::neutron::agents::metadata
 
   file { '/etc/neutron/dnsmasq-neutron.conf':
     content => hiera('neutron_dnsmasq_options'),
@@ -398,12 +391,10 @@ if hiera('step') >= 3 {
   include ::ceilometer::alarm::evaluator
   include ::ceilometer::expirer
   include ::ceilometer::collector
+  include ceilometer::agent::auth
   class { '::ceilometer::db' :
     database_connection => $ceilometer_database_connection,
   }
-  class { 'ceilometer::agent::auth':
-    auth_url => join(['http://', hiera('controller_virtual_ip'), ':5000/v2.0']),
-  }
 
   Cron <| title == 'ceilometer-expirer' |> { command => "sleep $((\$(od -A n -t d -N 3 /dev/urandom) % 86400)) && ${::ceilometer::params::expirer_command}" }
 

diff --git a/puppet/manifests/overcloud_controller_pacemaker.pp b/puppet/manifests/overcloud_controller_pacemaker.pp
index e30cda8..05ea875 100644 (file)
--- a/puppet/manifests/overcloud_controller_pacemaker.pp
+++ b/puppet/manifests/overcloud_controller_pacemaker.pp
@@ -237,9 +237,7 @@ if hiera('step') >= 2 {
   if count($redis_node_ips) > 1 {
     Class['::tripleo::redis_notification'] -> Service['redis-sentinel']
     include ::redis::sentinel
-    class {'::tripleo::redis_notification' :
-      haproxy_monitor_ip => hiera('tripleo::loadbalancer::controller_virtual_ip'),
-    }
+    include ::tripleo::redis_notification
   }
 
   exec { 'galera-ready' :
@@ -426,9 +424,7 @@ if hiera('step') >= 3 {
   }
   include join(['::glance::backend::', $glance_backend])
 
-  class { 'nova':
-    glance_api_servers     => join([hiera('glance_protocol'), '://', hiera('controller_virtual_ip'), ':', hiera('glance_port')]),
-  }
+  include ::nova
 
   class { '::nova::api' :
     sync_db => $sync_db,
@@ -473,7 +469,6 @@ if hiera('step') >= 3 {
     enabled => false,
   }
   class { 'neutron::agents::metadata':
-    auth_url => join(['http://', hiera('controller_virtual_ip'), ':35357/v2.0']),
     manage_service => false,
     enabled => false,
   }
@@ -496,7 +491,6 @@ if hiera('step') >= 3 {
     tunnel_types     => split(hiera('neutron_tunnel_types'), ','),
   }
 
-
   include ::cinder
   class { '::cinder::api':
     sync_db => $sync_db,
@@ -645,9 +639,7 @@ if hiera('step') >= 3 {
     database_connection => $ceilometer_database_connection,
     sync_db             => $sync_db,
   }
-  class { 'ceilometer::agent::auth':
-    auth_url => join(['http://', hiera('controller_virtual_ip'), ':5000/v2.0']),
-  }
+  include ceilometer::agent::auth
 
   Cron <| title == 'ceilometer-expirer' |> { command => "sleep $((\$(od -A n -t d -N 3 /dev/urandom) % 86400)) && ${::ceilometer::params::expirer_command}" }
 

diff --git a/puppet/vip-config.yaml b/puppet/vip-config.yaml
index a84def5..c083821 100644 (file)
--- a/puppet/vip-config.yaml
+++ b/puppet/vip-config.yaml
@@ -28,9 +28,13 @@ resources:
                 redis_vip: {get_input: redis_vip}
                 mysql_vip: {get_input: mysql_vip}
                 mysql_vip: {get_input: mysql_vip}
+                tripleo::loadbalancer::public_virtual_ip: {get_input: public_virtual_ip}
+                tripleo::loadbalancer::controller_virtual_ip: {get_input: control_virtual_ip}
                 tripleo::loadbalancer::internal_api_virtual_ip: {get_input: internal_api_virtual_ip}
                 tripleo::loadbalancer::storage_virtual_ip: {get_input: storage_virtual_ip}
                 tripleo::loadbalancer::storage_mgmt_virtual_ip: {get_input: storage_mgmt_virtual_ip}
+                tripleo::redis_notification::haproxy_monitor_ip: {get_input: control_virtual_ip}
+
 
 outputs:
   OS::stack_id: