build/f_repos/patch/fuel-library/0002-Rework-processorcount-limits-to-os_workers.patch

   1 From 15284786b55e844fcdcc95d5865c91be03871487 Mon Sep 17 00:00:00 2001
   2 From: Bogdan Dobrelya <bdobrelia@mirantis.com>
   3 Date: Fri, 30 Sep 2016 12:03:34 +0200
   4 Subject: [PATCH] Rework processorcount limits to os_workers
   5
   6 * reduce workers_max default limits of a 16 to the $::os_workers, which is limited by [2, 8] interval.
   7 * reduce workers/pool size/overflow limited by F($::processorcount) to F($::os_workers)
   8
   9 Closes-bug: 1629238
  10
  11 Change-Id: I5cc4d70b902eeaa1c9cf42911606eba13dd84aa2
  12 Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
  13
  14 diff --git a/deployment/puppet/openstack/manifests/cinder.pp b/deployment/puppet/openstack/manifests/cinder.pp
  15 index 36747a61e..502bc94b8 100644
  16 --- a/deployment/puppet/openstack/manifests/cinder.pp
  17 +++ b/deployment/puppet/openstack/manifests/cinder.pp
  18 @@ -44,7 +44,7 @@ class openstack::cinder(
  19    $keystone_user          = 'cinder',
  20    $region                 = 'RegionOne',
  21    $notification_driver    = undef,
  22 -  $service_workers        = $::processorcount,
  23 +  $service_workers        = $::os_workers,
  24    $vmware_host_ip         = '10.10.10.10',
  25    $vmware_host_username   = 'administrator@vsphere.local',
  26    $vmware_host_password   = 'password',
  27 diff --git a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp
  28 index 72fbe3271..e6fcf4f88 100644
  29 --- a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp
  30 +++ b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp
  31 @@ -35,7 +35,7 @@ class openstack_tasks::ceilometer::controller {
  32    $service_endpoint           = hiera('service_endpoint', $management_vip)
  33    $ha_mode                    = pick($ceilometer_hash['ha_mode'], true)
  34    $ssl_hash                   = hiera_hash('use_ssl', {})
  35 -  $workers_max                = hiera('workers_max', 16)
  36 +  $workers_max                = hiera('workers_max', $::os_workers)
  37    $service_workers            = pick($ceilometer_hash['workers'],
  38    min(max($::processorcount, 2), $workers_max))
  39
  40 diff --git a/deployment/puppet/openstack_tasks/manifests/glance/glance.pp b/deployment/puppet/openstack_tasks/manifests/glance/glance.pp
  41 index 4c951999d..f3a20c5a7 100644
  42 --- a/deployment/puppet/openstack_tasks/manifests/glance/glance.pp
  43 +++ b/deployment/puppet/openstack_tasks/manifests/glance/glance.pp
  44 @@ -21,7 +21,7 @@ class openstack_tasks::glance::glance {
  45    $max_overflow        = hiera('max_overflow')
  46    $ceilometer_hash     = hiera_hash('ceilometer', {})
  47    $region              = hiera('region','RegionOne')
  48 -  $workers_max         = hiera('workers_max', 16)
  49 +  $workers_max         = hiera('workers_max', $::os_workers)
  50    $service_workers     = pick($glance_hash['glance_workers'],
  51                                min(max($::processorcount, 2), $workers_max))
  52    $ironic_hash         = hiera_hash('ironic', {})
  53 diff --git a/deployment/puppet/openstack_tasks/manifests/horizon/horizon.pp b/deployment/puppet/openstack_tasks/manifests/horizon/horizon.pp
  54 index 1a70af711..824c700bb 100644
  55 --- a/deployment/puppet/openstack_tasks/manifests/horizon/horizon.pp
  56 +++ b/deployment/puppet/openstack_tasks/manifests/horizon/horizon.pp
  57 @@ -134,7 +134,7 @@ class openstack_tasks::horizon::horizon {
  58      $wsgi_processes = 2
  59      $wsgi_threads = 9
  60    } else {
  61 -    $wsgi_processes = $::processorcount
  62 +    $wsgi_processes = $::os_workers
  63      $wsgi_threads = 15
  64    }
  65
  66 diff --git a/deployment/puppet/openstack_tasks/manifests/ironic/ironic_compute.pp b/deployment/puppet/openstack_tasks/manifests/ironic/ironic_compute.pp
  67 index 2d26e1f19..6e7d8ca2b 100644
  68 --- a/deployment/puppet/openstack_tasks/manifests/ironic/ironic_compute.pp
  69 +++ b/deployment/puppet/openstack_tasks/manifests/ironic/ironic_compute.pp
  70 @@ -35,8 +35,8 @@ class openstack_tasks::ironic::ironic_compute {
  71    $db_name                        = pick($nova_hash['db_name'], 'nova')
  72    $db_password                    = pick($nova_hash['db_password'], 'nova')
  73
  74 -  $max_pool_size = hiera('max_pool_size', min($::processorcount * 5 + 0, 30 + 0))
  75 -  $max_overflow = hiera('max_overflow', min($::processorcount * 5 + 0, 60 + 0))
  76 +  $max_pool_size = hiera('max_pool_size', min($::os_workers * 5 + 0, 30 + 0))
  77 +  $max_overflow = hiera('max_overflow', min($::os_workers * 5 + 0, 60 + 0))
  78    $idle_timeout = hiera('idle_timeout', '3600')
  79    $max_retries = hiera('max_retries', '-1')
  80
  81 diff --git a/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp b/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp
  82 index 0d96b177a..0d3d33bff 100644
  83 --- a/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp
  84 +++ b/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp
  85 @@ -22,7 +22,7 @@ class openstack_tasks::keystone::keystone {
  86    $syslog_log_facility   = hiera('syslog_log_facility_keystone')
  87    $rabbit_hash           = hiera_hash('rabbit', {})
  88    $neutron_user_password = hiera('neutron_user_password', false)
  89 -  $workers_max           = hiera('workers_max', 16)
  90 +  $workers_max           = hiera('workers_max', $::os_workers)
  91    $service_workers       = pick($keystone_hash['workers'],
  92                                  min(max($::processorcount, 2), $workers_max))
  93    $default_log_levels    = hiera_hash('default_log_levels')
  94 diff --git a/deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp b/deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp
  95 index 6eccfc37b..644dd2440 100644
  96 --- a/deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp
  97 +++ b/deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp
  98 @@ -13,7 +13,7 @@ class openstack_tasks::openstack_cinder::openstack_cinder {
  99    $sahara_hash            = hiera_hash('sahara', {})
 100    $rabbit_hash            = hiera_hash('rabbit', {})
 101    $service_endpoint       = hiera('service_endpoint')
 102 -  $workers_max            = hiera('workers_max', 16)
 103 +  $workers_max            = hiera('workers_max', $::os_workers)
 104    $service_workers        = pick($cinder_hash['workers'], min(max($::processorcount, 2), $workers_max))
 105    $cinder_user_password   = $cinder_hash[user_password]
 106    $keystone_user          = pick($cinder_hash['user'], 'cinder')
 107 @@ -84,8 +84,8 @@ class openstack_tasks::openstack_cinder::openstack_cinder {
 108    }
 109
 110    # SQLAlchemy backend configuration
 111 -  $max_pool_size = min($::processorcount * 5 + 0, 30 + 0)
 112 -  $max_overflow = min($::processorcount * 5 + 0, 60 + 0)
 113 +  $max_pool_size = min($::os_workers * 5 + 0, 30 + 0)
 114 +  $max_overflow = min($::os_workers * 5 + 0, 60 + 0)
 115    $max_retries = '-1'
 116    $idle_timeout = '3600'
 117
 118 diff --git a/deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp b/deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp
 119 index 458c34c8f..89fcb20b3 100644
 120 --- a/deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp
 121 +++ b/deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp
 122 @@ -48,7 +48,7 @@ class openstack_tasks::openstack_controller::openstack_controller {
 123    $keystone_user                = pick($nova_hash['user'], 'nova')
 124    $keystone_tenant              = pick($nova_hash['tenant'], 'services')
 125    $region_name                  = hiera('region', 'RegionOne')
 126 -  $workers_max                  = hiera('workers_max', 16)
 127 +  $workers_max                  = hiera('workers_max', $::os_workers)
 128    $service_workers              = pick($nova_hash['workers'],
 129                                          min(max($::processorcount, 2), $workers_max))
 130    $compute_nodes                = get_nodes_hash_by_roles($network_metadata, ['compute'])
 131 @@ -117,8 +117,8 @@ class openstack_tasks::openstack_controller::openstack_controller {
 132    })
 133
 134    # SQLAlchemy backend configuration
 135 -  $max_pool_size = hiera('max_pool_size', min($::processorcount * 5 + 0, 30 + 0))
 136 -  $max_overflow = hiera('max_overflow', min($::processorcount * 5 + 0, 60 + 0))
 137 +  $max_pool_size = hiera('max_pool_size', min($::os_workers * 5 + 0, 30 + 0))
 138 +  $max_overflow = hiera('max_overflow', min($::os_workers * 5 + 0, 60 + 0))
 139    $idle_timeout = hiera('idle_timeout', '3600')
 140    $max_retries = hiera('max_retries', '-1')
 141
 142 diff --git a/deployment/puppet/openstack_tasks/manifests/openstack_network/agents/metadata.pp b/deployment/puppet/openstack_tasks/manifests/openstack_network/agents/metadata.pp
 143 index 9492493d1..240a378b7 100644
 144 --- a/deployment/puppet/openstack_tasks/manifests/openstack_network/agents/metadata.pp
 145 +++ b/deployment/puppet/openstack_tasks/manifests/openstack_network/agents/metadata.pp
 146 @@ -9,7 +9,7 @@ class openstack_tasks::openstack_network::agents::metadata {
 147    $neutron_advanced_config  = hiera_hash('neutron_advanced_configuration', { })
 148    $neutron_config           = hiera_hash('neutron_config')
 149    $dvr                      = pick($neutron_advanced_config['neutron_dvr'], false)
 150 -  $workers_max              = hiera('workers_max', 16)
 151 +  $workers_max              = hiera('workers_max', $::os_workers)
 152
 153    if $compute {
 154      $metadata_workers = pick($neutron_config['workers'],
 155 diff --git a/deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp b/deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp
 156 index 1d5c2d0e1..822e41750 100644
 157 --- a/deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp
 158 +++ b/deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp
 159 @@ -64,7 +64,7 @@ class openstack_tasks::openstack_network::server_config {
 160    $auth_url                = "${internal_auth_protocol}://${internal_auth_endpoint}:35357/"
 161    $nova_admin_auth_url     = "${admin_auth_protocol}://${admin_auth_endpoint}:35357/"
 162
 163 -  $workers_max             = hiera('workers_max', 16)
 164 +  $workers_max             = hiera('workers_max', $::os_workers)
 165    $service_workers         = pick($neutron_config['workers'], min(max($::processorcount, 1), $workers_max))
 166
 167    $neutron_advanced_config = hiera_hash('neutron_advanced_configuration', { })
 168 diff --git a/deployment/puppet/openstack_tasks/manifests/roles/cinder.pp b/deployment/puppet/openstack_tasks/manifests/roles/cinder.pp
 169 index b36619437..b6958b64f 100644
 170 --- a/deployment/puppet/openstack_tasks/manifests/roles/cinder.pp
 171 +++ b/deployment/puppet/openstack_tasks/manifests/roles/cinder.pp
 172 @@ -81,8 +81,8 @@ class openstack_tasks::roles::cinder {
 173    }
 174
 175    # SQLAlchemy backend configuration
 176 -  $max_pool_size = min($::processorcount * 5 + 0, 30 + 0)
 177 -  $max_overflow = min($::processorcount * 5 + 0, 60 + 0)
 178 +  $max_pool_size = min($::os_workers * 5 + 0, 30 + 0)
 179 +  $max_overflow = min($::os_workers * 5 + 0, 60 + 0)
 180    $max_retries = '-1'
 181    $idle_timeout = '3600'
 182
 183 diff --git a/deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp b/deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp
 184 index 9d011ceb5..0366659f0 100644
 185 --- a/deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp
 186 +++ b/deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp
 187 @@ -51,8 +51,8 @@ class openstack_tasks::sahara::sahara {
 188      $sahara_user     = pick($sahara_hash['user'], 'sahara')
 189      $sahara_password = pick($sahara_hash['user_password'])
 190      $tenant          = pick($sahara_hash['tenant'], 'services')
 191 -    $max_pool_size   = min($::processorcount * 5 + 0, 30 + 0)
 192 -    $max_overflow    = min($::processorcount * 5 + 0, 60 + 0)
 193 +    $max_pool_size   = min($::os_workers * 5 + 0, 30 + 0)
 194 +    $max_overflow    = min($::os_workers * 5 + 0, 60 + 0)
 195      $max_retries     = '-1'
 196      $idle_timeout    = '3600'
 197
 198 diff --git a/deployment/puppet/openstack_tasks/manifests/swift/parts/proxy.pp b/deployment/puppet/openstack_tasks/manifests/swift/parts/proxy.pp
 199 index 81d64a2d8..8a68acdb3 100644
 200 --- a/deployment/puppet/openstack_tasks/manifests/swift/parts/proxy.pp
 201 +++ b/deployment/puppet/openstack_tasks/manifests/swift/parts/proxy.pp
 202 @@ -26,7 +26,7 @@ class openstack_tasks::swift::parts::proxy (
 203      'account_quotas',
 204      'slo',
 205      'proxy-server'],
 206 -  $proxy_workers                     = $::processorcount,
 207 +  $proxy_workers                     = $::os_workers,
 208    $proxy_port                        = '8080',
 209    $proxy_allow_account_management    = true,
 210    $proxy_account_autocreate          = true,
 211 diff --git a/deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp b/deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp
 212 index a221baf36..77f4ba1a0 100644
 213 --- a/deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp
 214 +++ b/deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp
 215 @@ -30,7 +30,7 @@ class openstack_tasks::swift::proxy_storage {
 216    $keystone_user              = pick($swift_hash['user'], 'swift')
 217    $keystone_password          = pick($swift_hash['user_password'], 'passsword')
 218    $keystone_tenant            = pick($swift_hash['tenant'], 'services')
 219 -  $workers_max                = hiera('workers_max', 16)
 220 +  $workers_max                = hiera('workers_max', $::os_workers)
 221    $service_workers            = pick($swift_hash['workers'], min(max($::processorcount, 2), $workers_max))
 222    $ssl_hash                   = hiera_hash('use_ssl', {})
 223    $rabbit_hash                = hiera_hash('rabbit')
 224 diff --git a/deployment/puppet/osnailyfacter/lib/facter/os_workers.rb b/deployment/puppet/osnailyfacter/lib/facter/os_workers.rb
 225 new file mode 100644
 226 index 000000000..404fb9018
 227 --- /dev/null
 228 +++ b/deployment/puppet/osnailyfacter/lib/facter/os_workers.rb
 229 @@ -0,0 +1,20 @@
 230 +#
 231 +# We've found that using $::processorcount for workers/threads can lead to
 232 +# unexpected memory or process counts for people deploying on baremetal or
 233 +# if they have large number of cpus. This fact allows us to tweak the formula
 234 +# used to determine number of workers in a single place but use it across all
 235 +# modules.
 236 +#
 237 +# The value for os_workers is max between '(<# processors> / 4)' and '2' with
 238 +# a cap of 8.
 239 +#
 240 +# This fact can be overloaded by an external fact from /etc/factor/facts.d if
 241 +# a user would like to provide their own default value.
 242 +#
 243 +Facter.add(:os_workers) do
 244 +  has_weight 100
 245 +  setcode do
 246 +    processors = Facter.value('processorcount')
 247 +    [ [ (processors.to_i / 4), 2 ].max, 8 ].min
 248 +  end
 249 +end
 250 diff --git a/deployment/puppet/osnailyfacter/manifests/apache_mpm.pp b/deployment/puppet/osnailyfacter/manifests/apache_mpm.pp
 251 index 51bc3274d..70a654ae2 100644
 252 --- a/deployment/puppet/osnailyfacter/manifests/apache_mpm.pp
 253 +++ b/deployment/puppet/osnailyfacter/manifests/apache_mpm.pp
 254 @@ -11,7 +11,7 @@ class osnailyfacter::apache_mpm inherits ::osnailyfacter::apache {
 255    if ($::processorcount + 0) <= 2 {
 256      $startservers = 2
 257    } else {
 258 -    $startservers = $::processorcount
 259 +    $startservers = $::os_workers
 260    }
 261
 262    $maxrequestsperchild = 0
 263 diff --git a/deployment/puppet/osnailyfacter/manifests/globals/globals.pp b/deployment/puppet/osnailyfacter/manifests/globals/globals.pp
 264 index 99e0fa4cd..2e38b10ee 100644
 265 --- a/deployment/puppet/osnailyfacter/manifests/globals/globals.pp
 266 +++ b/deployment/puppet/osnailyfacter/manifests/globals/globals.pp
 267 @@ -311,14 +311,14 @@ class osnailyfacter::globals::globals {
 268    # The default value assumes there are 20 different types of workers limited by 100Mb each.
 269    $workers_ratio = hiera('workers_ratio', 2000)
 270    # Defines the maximum allowed number of workers for each service by RAM limits. Cannot exceed the value of 30.
 271 -  $workers_max = inline_template("<%= [(@memorysize_mb.to_i / @workers_ratio.to_i).floor + 1, 30].min %>")
 272 +  $workers_max = inline_template("<%= [(@memorysize_mb.to_i / @workers_ratio.to_i).floor + 1, $::os_workers].min %>")
 273
 274    $node_name_prefix_for_messaging = hiera('node_name_prefix_for_messaging', 'messaging-')
 275
 276    # MySQL and SQLAlchemy backend configuration
 277    $custom_mysql_setup_class = hiera('custom_mysql_setup_class', 'galera')
 278 -  $max_pool_size            = hiera('max_pool_size', min($::processorcount * 5 + 0, 30 + 0))
 279 -  $max_overflow             = hiera('max_overflow', min($::processorcount * 5 + 0, 60 + 0))
 280 +  $max_pool_size            = hiera('max_pool_size', min($::os_workers * 5 + 0, 30 + 0))
 281 +  $max_overflow             = hiera('max_overflow', min($::os_workers * 5 + 0, 60 + 0))
 282    $max_retries              = hiera('max_retries', '-1')
 283    $idle_timeout             = hiera('idle_timeout','3600')
 284    $nova_db_password         = $nova_hash['db_password']