Add fluentd client service

This implements support for installing fluentd agents as a composable
service on the overcloud.

Depends-On: I2e1abe4d8c8359e56ff626255ee50c9cacca1940

Implements: tripleo-opstools-centralized-logging
Change-Id: I23b0e23881b742158fcfb6b8c145a3211d45086e

diff --git a/capabilities-map.yaml b/capabilities-map.yaml
index 962dfb9..14d50f1 100644 (file)
--- a/capabilities-map.yaml
+++ b/capabilities-map.yaml
@@ -236,3 +236,11 @@ topics:
             description:
             requires:
               - overcloud-resource-registry-puppet.yaml
+      - title: Centralized logging support
+        description: Enable centralized logging clients (fluentd)
+        environments:
+          - file: environments/logging-environment.yaml
+            title: Enable fluentd client
+            description:
+            requires:
+              - overcloud-resource-registry-puppet.yaml

diff --git a/environments/logging-environment.yaml b/environments/logging-environment.yaml
new file mode 100644 (file)
index 0000000..eefa702
--- /dev/null
+++ b/environments/logging-environment.yaml
@@ -0,0 +1,29 @@
+## A Heat environment file which can be used to set up
+## logging agents
+
+resource_registry:
+  OS::TripleO::Services::FluentdClient: ../puppet/services/logging/fluentd-client.yaml
+
+parameter_defaults:
+
+## Simple configuration
+#
+# LoggingServers:
+#   - host: log0.example.com
+#     port: 24224
+#   - host: log1.example.com
+#     port: 24224
+#
+## Example SSL configuration
+## (note the use of port 24284 for ssl connections)
+#
+# LoggingServers:
+#   - host: 192.0.2.11
+#     port: 24284
+# LoggingUsesSSL: true
+# LoggingSharedKey: secret
+# LoggingSSLCertificate: |
+#   -----BEGIN CERTIFICATE-----
+#   ...certificate data here...
+#   -----END CERTIFICATE-----
+

diff --git a/overcloud-resource-registry-puppet.yaml b/overcloud-resource-registry-puppet.yaml
index d5fdaa0..a15e08a 100644 (file)
--- a/overcloud-resource-registry-puppet.yaml
+++ b/overcloud-resource-registry-puppet.yaml
@@ -212,6 +212,8 @@ resource_registry:
   OS::TripleO::Services::GnocchiStatsd: puppet/services/gnocchi-statsd.yaml
   OS::TripleO::Services::VipHosts: puppet/services/vip-hosts.yaml
   # Services that are disabled by default (use relevant environment files):
+  OS::TripleO::Services::FluentdClient: OS::Heat::None
+  OS::TripleO::LoggingConfiguration: puppet/services/logging/fluentd-config.yaml
   OS::Tripleo::Services::ManilaApi: OS::Heat::None
   OS::Tripleo::Services::ManilaScheduler: OS::Heat::None
   OS::Tripleo::Services::ManilaShare: OS::Heat::None

diff --git a/overcloud.j2.yaml b/overcloud.j2.yaml
index 5c0d914..6ca4a14 100644 (file)
--- a/overcloud.j2.yaml
+++ b/overcloud.j2.yaml
@@ -265,6 +265,8 @@ resources:
           {% endfor %}
           ServiceNames: {get_attr: [{{role.name}}ServiceChain, role_data, service_names]}
           MonitoringSubscriptions: {get_attr: [{{role.name}}ServiceChain, role_data, monitoring_subscriptions]}
+          LoggingSources: {get_attr: [{{role.name}}ServiceChain, role_data, logging_sources]}
+          LoggingGroups: {get_attr: [{{role.name}}ServiceChain, role_data, logging_groups]}
 {% endfor %}
 
   allNodesConfig:

diff --git a/puppet/ceph-storage.yaml b/puppet/ceph-storage.yaml
index 62748f9..03a53b0 100644 (file)
--- a/puppet/ceph-storage.yaml
+++ b/puppet/ceph-storage.yaml
@@ -102,6 +102,12 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
+  LoggingSources:
+    type: json
+    default: []
+  LoggingGroups:
+    type: comma_delimited_list
+    default: []
 
 resources:
   CephStorage:
@@ -275,6 +281,8 @@ resources:
             ceph:
               mapped_data:
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
+                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
+                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
   # Resource for site-specific injection of root certificate
   NodeTLSCAData:

diff --git a/puppet/cinder-storage.yaml b/puppet/cinder-storage.yaml
index f5118c2..a66ea08 100644 (file)
--- a/puppet/cinder-storage.yaml
+++ b/puppet/cinder-storage.yaml
@@ -96,6 +96,12 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
+  LoggingSources:
+    type: json
+    default: []
+  LoggingGroups:
+    type: comma_delimited_list
+    default: []
 
 resources:
   BlockStorage:
@@ -271,6 +277,8 @@ resources:
             volume:
               mapped_data:
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
+                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
+                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
   # Resource for site-specific injection of root certificate
   NodeTLSCAData:

diff --git a/puppet/compute.yaml b/puppet/compute.yaml
index 05b8d06..0205d0a 100644 (file)
--- a/puppet/compute.yaml
+++ b/puppet/compute.yaml
@@ -114,6 +114,12 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
+  LoggingSources:
+    type: json
+    default: []
+  LoggingGroups:
+    type: comma_delimited_list
+    default: []
 
 resources:
 
@@ -289,6 +295,8 @@ resources:
             compute:
               mapped_data:
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
+                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
+                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
   NovaComputeDeployment:
     type: OS::TripleO::SoftwareDeployment

diff --git a/puppet/controller.yaml b/puppet/controller.yaml
index 33ed51c..ccb517f 100644 (file)
--- a/puppet/controller.yaml
+++ b/puppet/controller.yaml
@@ -128,6 +128,12 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
+  LoggingSources:
+    type: json
+    default: []
+  LoggingGroups:
+    type: comma_delimited_list
+    default: []
 
 parameter_groups:
 - label: deprecated
@@ -349,6 +355,8 @@ resources:
                 # Misc
                 tripleo::haproxy::service_certificate: {get_attr: [NodeTLSData, deployed_ssl_certificate_path]}
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
+                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
+                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
   # Hook for site-specific additional pre-deployment config, e.g extra hieradata
   ControllerExtraConfigPre:

diff --git a/puppet/services/ceilometer-agent-central.yaml b/puppet/services/ceilometer-agent-central.yaml
index 5d980d7..2ae46d0 100644 (file)
--- a/puppet/services/ceilometer-agent-central.yaml
+++ b/puppet/services/ceilometer-agent-central.yaml
@@ -25,6 +25,11 @@ parameters:
   MonitoringSubscriptionCeilometerCentral:
     default: 'overcloud-ceilometer-agent-central'
     type: string
+  CeilometerAgentCentralLoggingSource:
+    type: json
+    default:
+      tag: openstack.ceilometer.agent.central
+      path: /var/log/ceilometer/central.log
 
 resources:
   CeilometerServiceBase:
@@ -40,6 +45,9 @@ outputs:
     value:
       service_name: ceilometer_agent_central
       monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerCentral}
+      logging_source: {get_param: CeilometerAgentCentralLoggingSource}
+      logging_groups:
+        - ceilometer
       config_settings:
         map_merge:
           - get_attr: [CeilometerServiceBase, role_data, config_settings]

diff --git a/puppet/services/ceilometer-agent-notification.yaml b/puppet/services/ceilometer-agent-notification.yaml
index bedb8b0..ea403aa 100644 (file)
--- a/puppet/services/ceilometer-agent-notification.yaml
+++ b/puppet/services/ceilometer-agent-notification.yaml
@@ -21,6 +21,11 @@ parameters:
   MonitoringSubscriptionCeilometerNotification:
     default: 'overcloud-ceilometer-agent-notification'
     type: string
+  CeilometerAgentNotificationLoggingSource:
+    type: json
+    default:
+      tag: openstack.ceilometer.agent.notification
+      path: /var/log/ceilometer/agent-notification.log
 
 
 resources:
@@ -37,6 +42,9 @@ outputs:
     value:
       service_name: ceilometer_agent_notification
       monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerNotification}
+      logging_source: {get_param: CeilometerAgentNotificationLoggingSource}
+      logging_groups:
+        - ceilometer
       config_settings:
         get_attr: [CeilometerServiceBase, role_data, config_settings]
       step_config: |

diff --git a/puppet/services/ceilometer-api.yaml b/puppet/services/ceilometer-api.yaml
index 5df9f2b..c6e3650 100644 (file)
--- a/puppet/services/ceilometer-api.yaml
+++ b/puppet/services/ceilometer-api.yaml
@@ -21,6 +21,11 @@ parameters:
   MonitoringSubscriptionCeilometerApi:
     default: 'overcloud-ceilometer-api'
     type: string
+  CeilometerApiLoggingSource:
+    type: json
+    default:
+      tag: openstack.ceilometer.api
+      path: /var/log/ceilometer/api.log
 
 
 resources:
@@ -44,6 +49,9 @@ outputs:
     value:
       service_name: ceilometer_api
       monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerApi}
+      logging_source: {get_param: CeilometerApiLoggingSource}
+      logging_groups:
+        - ceilometer
       config_settings:
         map_merge:
           - get_attr: [ApacheServiceBase, role_data, config_settings]

diff --git a/puppet/services/ceilometer-collector.yaml b/puppet/services/ceilometer-collector.yaml
index 9dbb275..2fc9aec 100644 (file)
--- a/puppet/services/ceilometer-collector.yaml
+++ b/puppet/services/ceilometer-collector.yaml
@@ -21,6 +21,11 @@ parameters:
   MonitoringSubscriptionCeilometerCollector:
     default: 'overcloud-ceilometer-collector'
     type: string
+  CeilometerCollectorLoggingSource:
+    type: json
+    default:
+      tag: openstack.ceilometer.collector
+      path: /var/log/ceilometer/collector.log
 
 resources:
   CeilometerServiceBase:
@@ -36,6 +41,9 @@ outputs:
     value:
       service_name: ceilometer_collector
       monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerCollector}
+      logging_source: {get_param: CeilometerCollectorLoggingSource}
+      logging_groups:
+        - ceilometer
       config_settings:
         get_attr: [CeilometerServiceBase, role_data, config_settings]
       step_config: |

diff --git a/puppet/services/cinder-api.yaml b/puppet/services/cinder-api.yaml
index 94c94a6..1dae9f1 100644 (file)
--- a/puppet/services/cinder-api.yaml
+++ b/puppet/services/cinder-api.yaml
@@ -34,6 +34,11 @@ parameters:
   MonitoringSubscriptionCinderApi:
     default: 'overcloud-cinder-api'
     type: string
+  CinderApiLoggingSource:
+    type: json
+    default:
+      tag: openstack.cinder.api
+      path: /var/log/cinder/cinder-api.log
 
 resources:
 
@@ -50,6 +55,9 @@ outputs:
     value:
       service_name: cinder_api
       monitoring_subscription: {get_param: MonitoringSubscriptionCinderApi}
+      logging_source: {get_param: CinderApiLoggingSource}
+      logging_groups:
+        - cinder
       config_settings:
         map_merge:
           - get_attr: [CinderBase, role_data, config_settings]

diff --git a/puppet/services/cinder-scheduler.yaml b/puppet/services/cinder-scheduler.yaml
index 1326e26..94c263e 100644 (file)
--- a/puppet/services/cinder-scheduler.yaml
+++ b/puppet/services/cinder-scheduler.yaml
@@ -21,6 +21,11 @@ parameters:
   MonitoringSubscriptionCinderScheduler:
     default: 'overcloud-cinder-scheduler'
     type: string
+  CinderSchedulerLoggingSource:
+    type: json
+    default:
+      tag: openstack.cinder.scheduler
+      path: /var/log/cinder/cinder-scheduler.log
 
 resources:
 
@@ -37,6 +42,9 @@ outputs:
     value:
       service_name: cinder_scheduler
       monitoring_subscription: {get_param: MonitoringSubscriptionCinderScheduler}
+      logging_source: {get_param: CinderSchedulerLoggingSource}
+      logging_groups:
+        - cinder
       config_settings:
         map_merge:
           - get_attr: [CinderBase, role_data, config_settings]

diff --git a/puppet/services/cinder-volume.yaml b/puppet/services/cinder-volume.yaml
index c84c784..82e16f3 100644 (file)
--- a/puppet/services/cinder-volume.yaml
+++ b/puppet/services/cinder-volume.yaml
@@ -59,6 +59,11 @@ parameters:
   MonitoringSubscriptionCinderVolume:
     default: 'overcloud-cinder-volume'
     type: string
+  CinderVolumeLoggingSource:
+    type: json
+    default:
+      tag: openstack.cinder.volume
+      path: /var/log/cinder/cinder-volume.log
 
 resources:
 
@@ -75,6 +80,9 @@ outputs:
     value:
       service_name: cinder_volume
       monitoring_subscription: {get_param: MonitoringSubscriptionCinderVolume}
+      logging_source: {get_param: CinderVolumeLoggingSource}
+      logging_groups:
+        - cinder
       config_settings:
         map_merge:
           - get_attr: [CinderBase, role_data, config_settings]

diff --git a/puppet/services/database/mongodb.yaml b/puppet/services/database/mongodb.yaml
index 36962a3..01daeaf 100644 (file)
--- a/puppet/services/database/mongodb.yaml
+++ b/puppet/services/database/mongodb.yaml
@@ -19,6 +19,15 @@ parameters:
     description: Mapping of service endpoint -> protocol. Typically set
                  via parameter_defaults in the resource registry.
     type: json
+  MongoDbLoggingSource:
+    type: json
+    description: Fluentd logging configuration for mongodb.
+    default:
+      tag: database.mongodb
+      path: /var/log/mongodb/mongodb.log
+      format: >-
+        /(?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+\+\d{4})
+        (?<message>.*)$/
 
 resources:
   MongoDbBase:
@@ -33,6 +42,9 @@ outputs:
     description: Service mongodb using composable services.
     value:
       service_name: mongodb
+      logging_groups:
+        - mongodb
+      logging_source: {get_param: MongoDbLoggingSource}
       config_settings:
         map_merge:
           - get_attr: [MongoDbBase, role_data, config_settings]

diff --git a/puppet/services/glance-api.yaml b/puppet/services/glance-api.yaml
index 03abe79..f0ec823 100644 (file)
--- a/puppet/services/glance-api.yaml
+++ b/puppet/services/glance-api.yaml
@@ -76,6 +76,11 @@ parameters:
   MonitoringSubscriptionGlanceApi:
     default: 'overcloud-glance-api'
     type: string
+  GlanceApiLoggingSource:
+    type: json
+    default:
+      tag: openstack.glance.api
+      path: /var/log/glance/api.log
 
 outputs:
   role_data:
@@ -83,6 +88,9 @@ outputs:
     value:
       service_name: glance_api
       monitoring_subscription: {get_param: MonitoringSubscriptionGlanceApi}
+      logging_source: {get_param: GlanceApiLoggingSource}
+      logging_groups:
+        - glance
       config_settings:
         glance::api::database_connection:
           list_join:

diff --git a/puppet/services/glance-registry.yaml b/puppet/services/glance-registry.yaml
index d5f01d4..864e970 100644 (file)
--- a/puppet/services/glance-registry.yaml
+++ b/puppet/services/glance-registry.yaml
@@ -33,6 +33,11 @@ parameters:
   MonitoringSubscriptionGlanceRegistry:
     default: 'overcloud-glance-registry'
     type: string
+  GlanceRegistryLoggingSource:
+    type: json
+    default:
+      tag: openstack.glance.registry
+      path: /var/log/glance/registry.log
 
 outputs:
   role_data:
@@ -40,6 +45,9 @@ outputs:
     value:
       service_name: glance_registry
       monitoring_subscription: {get_param: MonitoringSubscriptionGlanceRegistry}
+      logging_source: {get_param: GlanceRegistryLoggingSource}
+      logging_groups:
+        - glance
       config_settings:
         glance::registry::database_connection:
           list_join:

diff --git a/puppet/services/gnocchi-api.yaml b/puppet/services/gnocchi-api.yaml
index 650865e..e8f25bb 100644 (file)
--- a/puppet/services/gnocchi-api.yaml
+++ b/puppet/services/gnocchi-api.yaml
@@ -36,6 +36,11 @@ parameters:
   MonitoringSubscriptionGnocchiApi:
     default: 'overcloud-gnocchi-api'
     type: string
+  GnocchiApiLoggingSource:
+    type: json
+    default:
+      tag: openstack.gnocchi.api
+      path: /var/log/gnocchi/app.log
 
 resources:
 
@@ -59,6 +64,9 @@ outputs:
     value:
       service_name: gnocchi_api
       monitoring_subscription: {get_param: MonitoringSubscriptionGnocchiApi}
+      logging_source: {get_param: GnocchiApiLoggingSource}
+      logging_groups:
+        - gnocchi
       config_settings:
         map_merge:
           - get_attr: [ApacheServiceBase, role_data, config_settings]

diff --git a/puppet/services/heat-api-cfn.yaml b/puppet/services/heat-api-cfn.yaml
index 61a6907..4e9c45e 100644 (file)
--- a/puppet/services/heat-api-cfn.yaml
+++ b/puppet/services/heat-api-cfn.yaml
@@ -33,6 +33,11 @@ parameters:
   MonitoringSubscriptionHeatApiCnf:
     default: 'overcloud-heat-api-cfn'
     type: string
+  HeatApiCfnLoggingSource:
+    type: json
+    default:
+      tag: openstack.heat.api.cfn
+      path: /var/log/heat/heat-api-cfn.log
 
 resources:
   HeatBase:
@@ -48,6 +53,9 @@ outputs:
     value:
       service_name: heat_api_cfn
       monitoring_subscription: {get_param: MonitoringSubscriptionHeatApiCnf}
+      logging_source: {get_param: HeatApiCfnLoggingSource}
+      logging_groups:
+        - heat
       config_settings:
         map_merge:
           - get_attr: [HeatBase, role_data, config_settings]

diff --git a/puppet/services/heat-api-cloudwatch.yaml b/puppet/services/heat-api-cloudwatch.yaml
index c12e56e..6dfeaaf 100644 (file)
--- a/puppet/services/heat-api-cloudwatch.yaml
+++ b/puppet/services/heat-api-cloudwatch.yaml
@@ -25,6 +25,11 @@ parameters:
   MonitoringSubscriptionHeatApiCloudwatch:
     default: 'overcloud-heat-api-cloudwatch'
     type: string
+  HeatApiCloudwatchLoggingSource:
+    type: json
+    default:
+      tag: openstack.heat.api.cloudwatch
+      path: /var/log/heat/heat-api-cloudwatch.log
 
 resources:
   HeatBase:
@@ -40,6 +45,9 @@ outputs:
     value:
       service_name: heat_api_cloudwatch
       monitoring_subscription: {get_param: MonitoringSubscriptionHeatApiCloudwatch}
+      logging_source: {get_param: HeatApiCloudwatchLoggingSource}
+      logging_groups:
+        - heat
       config_settings:
         map_merge:
           - get_attr: [HeatBase, role_data, config_settings]

diff --git a/puppet/services/heat-api.yaml b/puppet/services/heat-api.yaml
index 64b0c53..edaff77 100644 (file)
--- a/puppet/services/heat-api.yaml
+++ b/puppet/services/heat-api.yaml
@@ -33,6 +33,11 @@ parameters:
   MonitoringSubscriptionHeatApi:
     default: 'overcloud-heat-api'
     type: string
+  HeatApiLoggingSource:
+    type: json
+    default:
+      tag: openstack.heat.api
+      path: /var/log/heat/heat-api.log
 
 resources:
   HeatBase:
@@ -48,6 +53,9 @@ outputs:
     value:
       service_name: heat_api
       monitoring_subscription: {get_param: MonitoringSubscriptionHeatApi}
+      logging_source: {get_param: HeatApiLoggingSource}
+      logging_groups:
+        - heat
       config_settings:
         map_merge:
           - get_attr: [HeatBase, role_data, config_settings]

diff --git a/puppet/services/heat-engine.yaml b/puppet/services/heat-engine.yaml
index 089bf53..1e7bec2 100644 (file)
--- a/puppet/services/heat-engine.yaml
+++ b/puppet/services/heat-engine.yaml
@@ -43,6 +43,11 @@ parameters:
   MonitoringSubscriptionHeatEngine:
     default: 'overcloud-heat-engine'
     type: string
+  HeatEngineLoggingSource:
+    type: json
+    default:
+      tag: openstack.heat.engine
+      path: /var/log/heat/heat-engine.log
 
 resources:
   HeatBase:
@@ -58,6 +63,9 @@ outputs:
     value:
       service_name: heat_engine
       monitoring_subscription: {get_param: MonitoringSubscriptionHeatEngine}
+      logging_source: {get_param: HeatEngineLoggingSource}
+      logging_groups:
+        - heat
       config_settings:
         map_merge:
           - get_attr: [HeatBase, role_data, config_settings]

diff --git a/puppet/services/keystone.yaml b/puppet/services/keystone.yaml
index 18fc915..b488fb4 100644 (file)
--- a/puppet/services/keystone.yaml
+++ b/puppet/services/keystone.yaml
@@ -93,6 +93,11 @@ parameters:
   KeystoneCredential1:
     type: string
     description: The second Keystone credential key. Must be a valid key.
+  KeystoneLoggingSource:
+    type: json
+    default:
+      tag: openstack.keystone
+      path: /var/log/keystone/keystone.log
 
 resources:
 
@@ -109,6 +114,9 @@ outputs:
     value:
       service_name: keystone
       monitoring_subscription: {get_param: MonitoringSubscriptionKeystone}
+      logging_source: {get_param: KeystoneLoggingSource}
+      logging_groups:
+        - keystone
       config_settings:
       config_settings:
         map_merge:

diff --git a/puppet/services/logging/fluentd-base.yaml b/puppet/services/logging/fluentd-base.yaml
new file mode 100644 (file)
index 0000000..583dca3
--- /dev/null
+++ b/puppet/services/logging/fluentd-base.yaml
@@ -0,0 +1,36 @@
+heat_template_version: 2016-04-08
+
+description: Fluentd base service
+
+parameters:
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: >
+      Mapping of service endpoint -> protocol. Typically set
+      via parameter_defaults in the resource registry.
+    type: json
+
+
+outputs:
+  role_data:
+    description: Role data for the Fluentd role.
+    value:
+      service_name: fluentd_base
+      config_settings:
+        fluentd::package_name: fluentd
+        fluentd::service_name: fluentd
+        fluentd::config_file: /etc/fluentd/fluent.conf
+        fluentd::config_owner: fluentd
+        fluentd::config_group: fluentd
+        fluentd::config_path: /etc/fluentd/config.d
+        fluentd::plugin_provider: yum
+        fluentd::repo_install: false

diff --git a/puppet/services/logging/fluentd-client.yaml b/puppet/services/logging/fluentd-client.yaml
new file mode 100644 (file)
index 0000000..3ae7110
--- /dev/null
+++ b/puppet/services/logging/fluentd-client.yaml
@@ -0,0 +1,64 @@
+heat_template_version: 2016-10-14
+
+description: Fluentd client configured with Puppet
+
+parameters:
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: >
+      Mapping of service endpoint -> protocol. Typically set
+      via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+  FluentdBase:
+    type: ./fluentd-base.yaml
+    properties:
+      EndpointMap: {get_param: EndpointMap}
+      ServiceNetMap: {get_param: ServiceNetMap}
+      DefaultPasswords: {get_param: DefaultPasswords}
+
+  LoggingConfiguration:
+    type: OS::TripleO::LoggingConfiguration
+
+outputs:
+  role_data:
+    description: Role data for the Fluentd client role.
+    value:
+      service_name: fluentd_client
+      config_settings:
+        map_merge:
+          - get_attr: [FluentdBase, role_data, config_settings]
+          - tripleo::profile::base::logging::fluentd::fluentd_servers:
+              get_attr: [LoggingConfiguration, LoggingServers]
+            tripleo::profile::base::logging::fluentd::fluentd_filters:
+              yaql:
+                expression: >
+                  $.data.filters.flatten().where($)
+                data:
+                  filters:
+                    - get_attr: [LoggingConfiguration, LoggingDefaultFilters]
+                    - get_attr: [LoggingConfiguration, LoggingExtraFilters]
+            tripleo::profile::base::logging::fluentd::fluentd_pos_file_path:
+              get_attr: [LoggingConfiguration, LoggingPosFilePath]
+            tripleo::profile::base::logging::fluentd::fluentd_use_ssl:
+              get_attr: [LoggingConfiguration, LoggingUsesSSL]
+            tripleo::profile::base::logging::fluentd::fluentd_ssl_certificate:
+              get_attr: [LoggingConfiguration, LoggingSSLCertificate]
+            tripleo::profile::base::logging::fluentd::fluentd_ssl_key:
+              get_attr: [LoggingConfiguration, LoggingSSLKey]
+            tripleo::profile::base::logging::fluentd::fluentd_ssl_key_passphrase:
+              get_attr: [LoggingConfiguration, LoggingSSLKeyPassphrase]
+            tripleo::profile::base::logging::fluentd::fluentd_shared_key:
+              get_attr: [LoggingConfiguration, LoggingSharedKey]
+      step_config: |
+        include ::tripleo::profile::base::logging::fluentd

diff --git a/puppet/services/logging/fluentd-config.yaml b/puppet/services/logging/fluentd-config.yaml
new file mode 100644 (file)
index 0000000..e051781
--- /dev/null
+++ b/puppet/services/logging/fluentd-config.yaml
@@ -0,0 +1,154 @@
+heat_template_version: 2016-10-14
+
+description: Fluentd logging configuration
+
+parameters:
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: >
+      Mapping of service endpoint -> protocol. Typically set
+      via parameter_defaults in the resource registry.
+    type: json
+  LoggingDefaultFormat:
+    description: >
+      Default format used to parse messages from log files.
+    type: string
+    default: >-
+      /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+)
+      (?<pid>\d+)
+      (?<priority>\S+)
+      (?<message>.*)$/
+  LoggingPosFilePath:
+    description: >
+      Directory in which to place fluentd pos_file files (used to track
+      file position for the 'tail' input type).
+    type: string
+    default: /var/cache/fluentd
+  LoggingDefaultGroups:
+    description: >
+      Make fluentd user a member of these groups. Only override this parameter
+      if you want to modify the default list of groups.  Use
+      LoggingExtraGroups to add the fluentd user to additional groups.
+    type: comma_delimited_list
+    default:
+      - root
+  LoggingExtraGroups:
+    description: >
+      Make fluentd user a member of these groups (in addition to
+      LoggingDefaultGroups and the groups provided by individual
+      composable services).
+    type: comma_delimited_list
+    default: []
+  LoggingServers:
+    description: |
+      A list of destinations to which fluentd will forward log messages.  Expects
+      a list of dictionaries of the form:
+
+          - host: loghost1.example.com
+            port: 24224
+          - host: loghost2.example.com
+            port: 24224
+    type: json
+    default: []
+  LoggingDefaultFilters:
+    description: >
+      A list of fluentd default filters. This will be passed verbatim
+      to the 'filter' key of a fluentd::config resource.  Only override this
+      if you do not want the default set of filters; use LoggingExtraFilters
+      if you just want to add additional servers.
+    type: json
+    default:
+      - tag_pattern: '**'
+        type: record_transformer
+        record:
+          nodename: '${hostname}'
+
+      - tag_pattern: 'openstack.**'
+        type: record_transformer
+        record:
+          component: '${tag_parts[1]}'
+  LoggingExtraFilters:
+    description: >
+      A list of additional fluentd filters. This will be passed
+      verbatim to the 'filter' key of a fluentd::config resource.
+    type: json
+    default: []
+  LoggingUsesSSL:
+    description: >
+      A boolean value indicating whether or not we should forward log messages
+      use the secure_forward plugin.
+    type: boolean
+    default: false
+  LoggingSSLCertificate:
+    description: >
+      PEM-encoded SSL CA certificate for fluentd.
+    type: string
+    default: ""
+  LoggingSSLKey:
+    description: >
+      PEM-encoded key for fluentd CA certificate (used by in_secure_forward).
+    type: string
+    default: ""
+  LoggingSSLKeyPassphrase:
+    description: >
+      Passphrase for LoggingSSLKey (used by in_secure_forward).
+    type: string
+    default: ""
+  LoggingSharedKey:
+    description: >
+      Shared secret for fluentd secure-forward plugin.
+    type: string
+    default: ""
+  LoggingDefaultSources:
+    description: >
+      A list of default logging sources for fluentd.  You should only override
+      this parameter if you wish to disable the default logging sources.  Use
+      LoggingExtraSources to define additional source configurations.
+    type: json
+    default: []
+  LoggingExtraSources:
+    description: >
+      A list of additional logging sources for fluentd.  These will be combined
+      with the LoggingDefaultSources and any logging sources defined by
+      composable services.
+    type: json
+    default: []
+
+outputs:
+  LoggingDefaultFormat:
+    value: {get_param: LoggingDefaultFormat}
+  LoggingDefaultFilters:
+    value: {get_param: LoggingDefaultFilters}
+  LoggingExtraFilters:
+    value: {get_param: LoggingExtraFilters}
+  LoggingDefaultGroups:
+    value: {get_param: LoggingDefaultGroups}
+  LoggingExtraGroups:
+    value: {get_param: LoggingExtraGroups}
+  LoggingPosFilePath:
+    value: {get_param: LoggingPosFilePath}
+  LoggingSSLCertificate:
+    value: {get_param: LoggingSSLCertificate}
+  LoggingSSLKey:
+    value: {get_param: LoggingSSLKey}
+  LoggingSSLKeyPassphrase:
+    value: {get_param: LoggingSSLKeyPassphrase}
+  LoggingServers:
+    value: {get_param: LoggingServers}
+  LoggingSharedKey:
+    value: {get_param: LoggingSharedKey}
+  LoggingUsesSSL:
+    value: {get_param: LoggingUsesSSL}
+  LoggingDefaultSources:
+    value: {get_param: LoggingDefaultSources}
+  LoggingExtraSources:
+    value: {get_param: LoggingExtraSources}

diff --git a/puppet/services/neutron-api.yaml b/puppet/services/neutron-api.yaml
index e4ca489..35934f7 100644 (file)
--- a/puppet/services/neutron-api.yaml
+++ b/puppet/services/neutron-api.yaml
@@ -56,6 +56,11 @@ parameters:
   MonitoringSubscriptionNeutronServer:
     default: 'overcloud-neutron-server'
     type: string
+  NeutronApiLoggingSource:
+    type: json
+    default:
+      tag: openstack.neutron.api
+      path: /var/log/neutron/server.log
 
 resources:
 
@@ -72,6 +77,9 @@ outputs:
     value:
       service_name: neutron_api
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronServer}
+      logging_source: {get_param: NeutronApiLoggingSource}
+      logging_groups:
+        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]

diff --git a/puppet/services/neutron-dhcp.yaml b/puppet/services/neutron-dhcp.yaml
index b2ad5da..2cd08f9 100644 (file)
--- a/puppet/services/neutron-dhcp.yaml
+++ b/puppet/services/neutron-dhcp.yaml
@@ -34,6 +34,11 @@ parameters:
   MonitoringSubscriptionNeutronDhcp:
     default: 'overcloud-neutron-dhcp'
     type: string
+  NeutronDhcpAgentLoggingSource:
+    type: json
+    default:
+      tag: openstack.neutron.agent.dhcp
+      path: /var/log/neutron/dhcp-agent.log
 
 resources:
 
@@ -50,6 +55,9 @@ outputs:
     value:
       service_name: neutron_dhcp
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronDhcp}
+      logging_source: {get_param: NeutronDhcpAgentLoggingSource}
+      logging_groups:
+        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]

diff --git a/puppet/services/neutron-l3-compute-dvr.yaml b/puppet/services/neutron-l3-compute-dvr.yaml
index 5eb3e25..b6c2911 100644 (file)
--- a/puppet/services/neutron-l3-compute-dvr.yaml
+++ b/puppet/services/neutron-l3-compute-dvr.yaml
@@ -29,6 +29,11 @@ parameters:
   MonitoringSubscriptionNeutronL3Dvr:
     default: 'overcloud-neutron-l3-dvr'
     type: string
+  NeutronL3ComputeAgentLoggingSource:
+    type: json
+    default:
+      tag: openstack.neutron.agent.l3-compute
+      path: /var/log/neutron/l3-agent.log
 
 resources:
 
@@ -45,6 +50,9 @@ outputs:
     value:
       service_name: neutron_l3_compute_dvr
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronL3Dvr}
+      logging_source: {get_param: NeutronL3ComputeAgentLoggingSource}
+      logging_groups:
+        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]

diff --git a/puppet/services/neutron-l3.yaml b/puppet/services/neutron-l3.yaml
index de62a50..9e22337 100644 (file)
--- a/puppet/services/neutron-l3.yaml
+++ b/puppet/services/neutron-l3.yaml
@@ -37,6 +37,11 @@ parameters:
   MonitoringSubscriptionNeutronL3:
     default: 'overcloud-neutron-l3-agent'
     type: string
+  NeutronL3AgentLoggingSource:
+    type: json
+    default:
+      tag: openstack.neutron.agent.l3
+      path: /var/log/neutron/l3-agent.log
 
 resources:
 
@@ -53,6 +58,9 @@ outputs:
     value:
       service_name: neutron_l3
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronL3}
+      logging_source: {get_param: NeutronL3AgentLoggingSource}
+      logging_groups:
+        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]

diff --git a/puppet/services/neutron-metadata.yaml b/puppet/services/neutron-metadata.yaml
index 320ae0c..0479231 100644 (file)
--- a/puppet/services/neutron-metadata.yaml
+++ b/puppet/services/neutron-metadata.yaml
@@ -33,6 +33,11 @@ parameters:
   MonitoringSubscriptionNeutronMetadata:
     default: 'overcloud-neutron-metadata'
     type: string
+  NeutronMetadataAgentLoggingSource:
+    type: json
+    default:
+      tag: openstack.neutron.agent.metadata
+      path: /var/log/neutron/metadata-agent.log
 
 resources:
 
@@ -49,6 +54,9 @@ outputs:
     value:
       service_name: neutron_metadata
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronMetadata}
+      logging_source: {get_param: NeutronMetadataAgentLoggingSource}
+      logging_groups:
+        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]

diff --git a/puppet/services/neutron-ovs-agent.yaml b/puppet/services/neutron-ovs-agent.yaml
index ade322e..cbe6563 100644 (file)
--- a/puppet/services/neutron-ovs-agent.yaml
+++ b/puppet/services/neutron-ovs-agent.yaml
@@ -64,6 +64,11 @@ parameters:
       examples are: noop, openvswitch, iptables_hybrid. The default value of an
       empty string will result in a default supported configuration.
     type: string
+  NeutronOpenVswitchAgentLoggingSource:
+    type: json
+    default:
+      tag: openstack.neutron.agent.openvswitch
+      path: /var/log/neutron/openvswitch-agent.log
 
 resources:
 
@@ -80,6 +85,9 @@ outputs:
     value:
       service_name: neutron_ovs_agent
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronOvs}
+      logging_source: {get_param: NeutronOpenVswitchAgentLoggingSource}
+      logging_groups:
+        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]

diff --git a/puppet/services/nova-api.yaml b/puppet/services/nova-api.yaml
index e1dbd8e..9d42fe6 100644 (file)
--- a/puppet/services/nova-api.yaml
+++ b/puppet/services/nova-api.yaml
@@ -46,6 +46,11 @@ parameters:
   MonitoringSubscriptionNovaApi:
     default: 'overcloud-nova-api'
     type: string
+  NovaApiLoggingSource:
+    type: json
+    default:
+      tag: openstack.nova.api
+      path: /var/log/nova/nova-api.log
 
 resources:
   NovaBase:
@@ -61,6 +66,9 @@ outputs:
     value:
       service_name: nova_api
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaApi}
+      logging_source: {get_param: NovaApiLoggingSource}
+      logging_groups:
+        - nova
       config_settings:
         map_merge:
           - get_attr: [NovaBase, role_data, config_settings]

diff --git a/puppet/services/nova-compute.yaml b/puppet/services/nova-compute.yaml
index d1d7ae6..c9a9534 100644 (file)
--- a/puppet/services/nova-compute.yaml
+++ b/puppet/services/nova-compute.yaml
@@ -70,6 +70,11 @@ parameters:
   MonitoringSubscriptionNovaCompute:
     default: 'overcloud-nova-compute'
     type: string
+  NovaComputeLoggingSource:
+    type: json
+    default:
+      tag: openstack.nova.compute
+      path: /var/log/nova/nova-compute.log
 
 resources:
   NovaBase:
@@ -85,6 +90,9 @@ outputs:
     value:
       service_name: nova_compute
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaCompute}
+      logging_source: {get_param: NovaComputeLoggingSource}
+      logging_groups:
+        - nova
       config_settings:
         map_merge:
           - get_attr: [NovaBase, role_data, config_settings]

diff --git a/puppet/services/nova-conductor.yaml b/puppet/services/nova-conductor.yaml
index 5dbc7ca..2671cdd 100644 (file)
--- a/puppet/services/nova-conductor.yaml
+++ b/puppet/services/nova-conductor.yaml
@@ -25,6 +25,11 @@ parameters:
   MonitoringSubscriptionNovaConductor:
     default: 'overcloud-nova-conductor'
     type: string
+  NovaSchedulerLoggingSource:
+    type: json
+    default:
+      tag: openstack.nova.scheduler
+      path: /var/log/nova/nova-scheduler.log
 
 resources:
   NovaBase:
@@ -40,6 +45,9 @@ outputs:
     value:
       service_name: nova_conductor
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaConductor}
+      logging_source: {get_param: NovaSchedulerLoggingSource}
+      logging_groups:
+        - nova
       config_settings:
         map_merge:
           - get_attr: [NovaBase, role_data, config_settings]

diff --git a/puppet/services/nova-consoleauth.yaml b/puppet/services/nova-consoleauth.yaml
index 13e3a26..85e6042 100644 (file)
--- a/puppet/services/nova-consoleauth.yaml
+++ b/puppet/services/nova-consoleauth.yaml
@@ -21,6 +21,11 @@ parameters:
   MonitoringSubscriptionNovaConsoleauth:
     default: 'overcloud-nova-consoleauth'
     type: string
+  NovaConsoleauthLoggingSource:
+    type: json
+    default:
+      tag: openstack.nova.consoleauth
+      path: /var/log/nova/nova-consoleauth.log
 
 resources:
   NovaBase:
@@ -36,6 +41,9 @@ outputs:
     value:
       service_name: nova_consoleauth
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaConsoleauth}
+      logging_source: {get_param: NovaConsoleauthLoggingSource}
+      logging_groups:
+        - nova
       config_settings:
         get_attr: [NovaBase, role_data, config_settings]
       step_config: |

diff --git a/puppet/services/nova-scheduler.yaml b/puppet/services/nova-scheduler.yaml
index 3ffc9c5..d89e3e1 100644 (file)
--- a/puppet/services/nova-scheduler.yaml
+++ b/puppet/services/nova-scheduler.yaml
@@ -32,6 +32,11 @@ parameters:
   MonitoringSubscriptionNovaScheduler:
     default: 'overcloud-nova-scheduler'
     type: string
+  NovaSchedulerLoggingSource:
+    type: json
+    default:
+      tag: openstack.nova.scheduler
+      path: /var/log/nova/nova-scheduler.log
 
 resources:
   NovaBase:
@@ -47,6 +52,9 @@ outputs:
     value:
       service_name: nova_scheduler
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaScheduler}
+      logging_source: {get_param: NovaSchedulerLoggingSource}
+      logging_groups:
+        - nova
       config_settings:
         map_merge:
           - get_attr: [NovaBase, role_data, config_settings]

diff --git a/puppet/services/nova-vnc-proxy.yaml b/puppet/services/nova-vnc-proxy.yaml
index 899fa35..85d59ae 100644 (file)
--- a/puppet/services/nova-vnc-proxy.yaml
+++ b/puppet/services/nova-vnc-proxy.yaml
@@ -21,6 +21,11 @@ parameters:
   MonitoringSubscriptionNovaVNCProxy:
     default: 'overcloud-nova-vncproxy'
     type: string
+  NovaVncproxyLoggingSource:
+    type: json
+    default:
+      tag: openstack.nova.vncproxy
+      path: /var/log/nova/nova-vncproxy.log
 
 resources:
   NovaBase:
@@ -36,6 +41,9 @@ outputs:
     value:
       service_name: nova_vnc_proxy
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaVNCProxy}
+      logging_source: {get_param: NovaVncproxyLoggingSource}
+      logging_groups:
+        - nova
       config_settings:
         map_merge:
           - get_attr: [NovaBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker.yaml b/puppet/services/pacemaker.yaml
index 5d1d666..abfb9c8 100644 (file)
--- a/puppet/services/pacemaker.yaml
+++ b/puppet/services/pacemaker.yaml
@@ -66,6 +66,16 @@ parameters:
           ]
         }
     type: json
+  PacemakerLoggingSource:
+    type: json
+    default:
+      tag: system.pacemaker
+      path: /var/log/pacemaker.log,/var/log/cluster/corosync.log
+      format: >-
+        /^(?<time>[^ ]*\s*[^ ]* [^ ]*)
+        \[(?<pid>[^ ]*)\]
+        (?<host>[^ ]*)
+        (?<message>.*)$/
 
 outputs:
   role_data:
@@ -73,6 +83,9 @@ outputs:
     value:
       service_name: pacemaker
       monitoring_subscription: {get_param: MonitoringSubscriptionPacemaker}
+      logging_groups:
+        - haclient
+      logging_source: {get_param: PacemakerLoggingSource}
       config_settings:
         pacemaker::corosync::cluster_name: 'tripleo_cluster'
         pacemaker::corosync::manage_fw: false

diff --git a/puppet/services/pacemaker/cinder-api.yaml b/puppet/services/pacemaker/cinder-api.yaml
index e4bcfc3..6823789 100644 (file)
--- a/puppet/services/pacemaker/cinder-api.yaml
+++ b/puppet/services/pacemaker/cinder-api.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: cinder_api
       monitoring_subscription: {get_attr: [CinderApiBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [CinderApiBase, role_data, logging_source]}
+      logging_groups: {get_attr: [CinderApiBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [CinderApiBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/cinder-scheduler.yaml b/puppet/services/pacemaker/cinder-scheduler.yaml
index eb578e5..15e44be 100644 (file)
--- a/puppet/services/pacemaker/cinder-scheduler.yaml
+++ b/puppet/services/pacemaker/cinder-scheduler.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: cinder_scheduler
       monitoring_subscription: {get_attr: [CinderSchedulerBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [CinderSchedulerBase, role_data, logging_source]}
+      logging_groups: {get_attr: [CinderSchedulerBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [CinderSchedulerBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/cinder-volume.yaml b/puppet/services/pacemaker/cinder-volume.yaml
index d5dedf3..11b9bf8 100644 (file)
--- a/puppet/services/pacemaker/cinder-volume.yaml
+++ b/puppet/services/pacemaker/cinder-volume.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: cinder_volume
       monitoring_subscription: {get_attr: [CinderVolumeBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [CinderVolumeBase, role_data, logging_source]}
+      logging_groups: {get_attr: [CinderVolumeBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [CinderVolumeBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/glance-api.yaml b/puppet/services/pacemaker/glance-api.yaml
index 684785a..20a439f 100644 (file)
--- a/puppet/services/pacemaker/glance-api.yaml
+++ b/puppet/services/pacemaker/glance-api.yaml
@@ -58,6 +58,8 @@ outputs:
     value:
       service_name: glance_api
       monitoring_subscription: {get_attr: [GlanceApiBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [GlanceApiBase, role_data, logging_source]}
+      logging_groups: {get_attr: [GlanceApiBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [GlanceApiBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/glance-registry.yaml b/puppet/services/pacemaker/glance-registry.yaml
index 5bcabca..41f89fd 100644 (file)
--- a/puppet/services/pacemaker/glance-registry.yaml
+++ b/puppet/services/pacemaker/glance-registry.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: glance_registry
       monitoring_subscription: {get_attr: [GlanceRegistryBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [GlanceRegistryBase, role_data, logging_source]}
+      logging_groups: {get_attr: [GlanceRegistryBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [GlanceRegistryBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/heat-api-cfn.yaml b/puppet/services/pacemaker/heat-api-cfn.yaml
index eae01b5..dd25905 100644 (file)
--- a/puppet/services/pacemaker/heat-api-cfn.yaml
+++ b/puppet/services/pacemaker/heat-api-cfn.yaml
@@ -33,6 +33,8 @@ outputs:
     value:
       service_name: heat_api_cfn
       monitoring_subscription: {get_attr: [HeatApiCfnBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [HeatApiCfnBase, role_data, logging_source]}
+      logging_groups: {get_attr: [HeatApiCfnBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [HeatApiCfnBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/heat-api-cloudwatch.yaml b/puppet/services/pacemaker/heat-api-cloudwatch.yaml
index 5608ae9..18d2a0d 100644 (file)
--- a/puppet/services/pacemaker/heat-api-cloudwatch.yaml
+++ b/puppet/services/pacemaker/heat-api-cloudwatch.yaml
@@ -33,6 +33,8 @@ outputs:
     value:
       service_name: heat_api_cloudwatch
       monitoring_subscription: {get_attr: [HeatApiCloudwatchBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [HeatApiCloudwatchBase, role_data, logging_source]}
+      logging_groups: {get_attr: [HeatApiCloudwatchBase, role_data, logging_groups]}
       config_settings:
         map_merge:
            - get_attr: [HeatApiCloudwatchBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/heat-api.yaml b/puppet/services/pacemaker/heat-api.yaml
index 6fd790c..43122cb 100644 (file)
--- a/puppet/services/pacemaker/heat-api.yaml
+++ b/puppet/services/pacemaker/heat-api.yaml
@@ -33,6 +33,8 @@ outputs:
     value:
       service_name: heat_api
       monitoring_subscription: {get_attr: [HeatApiBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [HeatApiBase, role_data, logging_source]}
+      logging_groups: {get_attr: [HeatApiBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [HeatApiBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/heat-engine.yaml b/puppet/services/pacemaker/heat-engine.yaml
index b8c962a..54bfdad 100644 (file)
--- a/puppet/services/pacemaker/heat-engine.yaml
+++ b/puppet/services/pacemaker/heat-engine.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: heat_engine
       monitoring_subscription: {get_attr: [HeatEngineBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [HeatEngineBase, role_data, logging_source]}
+      logging_groups: {get_attr: [HeatEngineBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [HeatEngineBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/keystone.yaml b/puppet/services/pacemaker/keystone.yaml
index 0a479c9..908b9bb 100644 (file)
--- a/puppet/services/pacemaker/keystone.yaml
+++ b/puppet/services/pacemaker/keystone.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: keystone
       monitoring_subscription: {get_attr: [KeystoneServiceBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [KeystoneServiceBase, role_data, logging_source]}
+      logging_groups: {get_attr: [KeystoneServiceBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [KeystoneServiceBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/neutron-dhcp.yaml b/puppet/services/pacemaker/neutron-dhcp.yaml
index 9b9e584..7fca73d 100644 (file)
--- a/puppet/services/pacemaker/neutron-dhcp.yaml
+++ b/puppet/services/pacemaker/neutron-dhcp.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: neutron_dhcp
       monitoring_subscription: {get_attr: [NeutronDhcpBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [NeutronDhcpBase, role_data, logging_source]}
+      logging_groups: {get_attr: [NeutronDhcpBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [NeutronDhcpBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/neutron-l3.yaml b/puppet/services/pacemaker/neutron-l3.yaml
index 21ac02d..cdb87f5 100644 (file)
--- a/puppet/services/pacemaker/neutron-l3.yaml
+++ b/puppet/services/pacemaker/neutron-l3.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: neutron_l3
       monitoring_subscription: {get_attr: [NeutronL3Base, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [NeutronL3Base, role_data, logging_source]}
+      logging_groups: {get_attr: [NeutronL3Base, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [NeutronL3Base, role_data, config_settings]

diff --git a/puppet/services/pacemaker/neutron-metadata.yaml b/puppet/services/pacemaker/neutron-metadata.yaml
index 8c22d42..49a31eb 100644 (file)
--- a/puppet/services/pacemaker/neutron-metadata.yaml
+++ b/puppet/services/pacemaker/neutron-metadata.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: neutron_metadata
       monitoring_subscription: {get_attr: [NeutronMetadataBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [NeutronMetadataBase, role_data, logging_source]}
+      logging_groups: {get_attr: [NeutronMetadataBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [NeutronMetadataBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/neutron-ovs-agent.yaml b/puppet/services/pacemaker/neutron-ovs-agent.yaml
index 18d6073..a2bd7c8 100644 (file)
--- a/puppet/services/pacemaker/neutron-ovs-agent.yaml
+++ b/puppet/services/pacemaker/neutron-ovs-agent.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: neutron_ovs_agent
       monitoring_subscription: {get_attr: [NeutronOvsBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [NeutronOvsBase, role_data, logging_source]}
+      logging_groups: {get_attr: [NeutronOvsBase, role_data, logging_groups]}
       config_settings:
         get_attr: [NeutronOvsBase, role_data, config_settings]
       step_config: |

diff --git a/puppet/services/pacemaker/nova-api.yaml b/puppet/services/pacemaker/nova-api.yaml
index 3d56534..b86e438 100644 (file)
--- a/puppet/services/pacemaker/nova-api.yaml
+++ b/puppet/services/pacemaker/nova-api.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: nova_api
       monitoring_subscription: {get_attr: [NovaApiBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [NovaApiBase, role_data, logging_source]}
+      logging_groups: {get_attr: [NovaApiBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [NovaApiBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/nova-conductor.yaml b/puppet/services/pacemaker/nova-conductor.yaml
index 9d55a48..a0a766e 100644 (file)
--- a/puppet/services/pacemaker/nova-conductor.yaml
+++ b/puppet/services/pacemaker/nova-conductor.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: nova_conductor
       monitoring_subscription: {get_attr: [NovaConductorBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [NovaConductorBase, role_data, logging_source]}
+      logging_groups: {get_attr: [NovaConductorBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [NovaConductorBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/nova-consoleauth.yaml b/puppet/services/pacemaker/nova-consoleauth.yaml
index 814505f..5d51eb4 100644 (file)
--- a/puppet/services/pacemaker/nova-consoleauth.yaml
+++ b/puppet/services/pacemaker/nova-consoleauth.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: nova_consoleauth
       monitoring_subscription: {get_attr: [NovaConsoleauthBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [NovaConsoleauthBase, role_data, logging_source]}
+      logging_groups: {get_attr: [NovaConsoleauthBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [NovaConsoleauthBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/nova-scheduler.yaml b/puppet/services/pacemaker/nova-scheduler.yaml
index 2769226..8828ee1 100644 (file)
--- a/puppet/services/pacemaker/nova-scheduler.yaml
+++ b/puppet/services/pacemaker/nova-scheduler.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: nova_scheduler
       monitoring_subscription: {get_attr: [NovaSchedulerBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [NovaSchedulerBase, role_data, logging_source]}
+      logging_groups: {get_attr: [NovaSchedulerBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [NovaSchedulerBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/nova-vnc-proxy.yaml b/puppet/services/pacemaker/nova-vnc-proxy.yaml
index d0c4f1d..ebe84a0 100644 (file)
--- a/puppet/services/pacemaker/nova-vnc-proxy.yaml
+++ b/puppet/services/pacemaker/nova-vnc-proxy.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: nova_vnc_proxy
       monitoring_subscription: {get_attr: [NovaVncproxyBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [NovaVncproxyBase, role_data, logging_source]}
+      logging_groups: {get_attr: [NovaVncproxyBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [NovaVncproxyBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/sahara-api.yaml b/puppet/services/pacemaker/sahara-api.yaml
index 214e8db..3dfb7d9 100644 (file)
--- a/puppet/services/pacemaker/sahara-api.yaml
+++ b/puppet/services/pacemaker/sahara-api.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: sahara_api
       monitoring_subscription: {get_attr: [SaharaApiBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [SaharaApiBase, role_data, logging_source]}
+      logging_groups: {get_attr: [SaharaApiBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [SaharaApiBase, role_data, config_settings]

diff --git a/puppet/services/pacemaker/sahara-engine.yaml b/puppet/services/pacemaker/sahara-engine.yaml
index aa85115..a06d11b 100644 (file)
--- a/puppet/services/pacemaker/sahara-engine.yaml
+++ b/puppet/services/pacemaker/sahara-engine.yaml
@@ -34,6 +34,8 @@ outputs:
     value:
       service_name: sahara_engine
       monitoring_subscription: {get_attr: [SaharaEngineBase, role_data, monitoring_subscription]}
+      logging_source: {get_attr: [SaharaEngineBase, role_data, logging_source]}
+      logging_groups: {get_attr: [SaharaEngineBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [SaharaEngineBase, role_data, config_settings]

diff --git a/puppet/services/sahara-api.yaml b/puppet/services/sahara-api.yaml
index 7f15ca7..8085d54 100644 (file)
--- a/puppet/services/sahara-api.yaml
+++ b/puppet/services/sahara-api.yaml
@@ -33,6 +33,11 @@ parameters:
   MonitoringSubscriptionSaharaApi:
     default: 'overcloud-sahara-api'
     type: string
+  SaharaApiLoggingSource:
+    type: json
+    default:
+      tag: openstack.sahara.api
+      path: /var/log/sahara/sahara-api.log
 
 resources:
   SaharaBase:
@@ -48,6 +53,9 @@ outputs:
     value:
       service_name: sahara_api
       monitoring_subscription: {get_param: MonitoringSubscriptionSaharaApi}
+      logging_source: {get_param: SaharaApiLoggingSource}
+      logging_groups:
+        - sahara
       config_settings:
         map_merge:
           - get_attr: [SaharaBase, role_data, config_settings]

diff --git a/puppet/services/sahara-engine.yaml b/puppet/services/sahara-engine.yaml
index 9224fd5..287c1c0 100644 (file)
--- a/puppet/services/sahara-engine.yaml
+++ b/puppet/services/sahara-engine.yaml
@@ -21,6 +21,11 @@ parameters:
   MonitoringSubscriptionSaharaEngine:
     default: 'overcloud-sahara-engine'
     type: string
+  SaharaEngineLoggingSource:
+    type: json
+    default:
+      tag: openstack.sahara.engine
+      path: /var/log/sahara/sahara-engine.log
 
 resources:
   SaharaBase:
@@ -36,6 +41,9 @@ outputs:
     value:
       service_name: sahara_engine
       monitoring_subscription: {get_param: MonitoringSubscriptionSaharaEngine}
+      logging_source: {get_param: SaharaEngineLoggingSource}
+      logging_groups:
+        - sahara
       config_settings:
         map_merge:
           - get_attr: [SaharaBase, role_data, config_settings]

diff --git a/puppet/services/services.yaml b/puppet/services/services.yaml
index b54a6d7..c8d5642 100644 (file)
--- a/puppet/services/services.yaml
+++ b/puppet/services/services.yaml
@@ -39,6 +39,9 @@ resources:
         EndpointMap: {get_param: EndpointMap}
         DefaultPasswords: {get_param: DefaultPasswords}
 
+  LoggingConfiguration:
+    type: OS::TripleO::LoggingConfiguration
+
 outputs:
   role_data:
     description: Combined Role data for this set of services.
@@ -53,6 +56,42 @@ outputs:
         yaql:
           expression: list($.data.subscriptions.where($ != null))
           data: {subscriptions: {get_attr: [ServiceChain, role_data, monitoring_subscription]}}
+      logging_sources:
+        # Transform the individual logging_source configuration from
+        # each service in the chain into a global list, adding some
+        # default configuration at the same time.
+        yaql:
+          expression: >
+            let(
+            default_format => $.data.default_format,
+            pos_file_path => $.data.pos_file_path,
+            sources => $.data.sources.flatten()
+            ) ->
+            $sources.where($ != null).select({
+            'type' => 'tail',
+            'tag' => $.tag,
+            'path' => $.path,
+            'format' => $.get('format', $default_format),
+            'pos_file' => $.get('pos_file', $pos_file_path + '/' + $.tag + '.pos')
+            })
+          data:
+            sources:
+              - {get_attr: [LoggingConfiguration, LoggingDefaultSources]}
+              - {get_attr: [ServiceChain, role_data, logging_source]}
+              - {get_attr: [LoggingConfiguration, LoggingExtraSources]}
+            default_format: {get_attr: [LoggingConfiguration, LoggingDefaultFormat]}
+            pos_file_path: {get_attr: [LoggingConfiguration, LoggingPosFilePath]}
+      logging_groups:
+        # Build a list of unique groups to which we should add the
+        # fluentd user.
+        yaql:
+          expression: >
+            set($.data.groups.flatten()).where($)
+          data:
+            groups:
+              - [{get_attr: [LoggingConfiguration, LoggingDefaultGroups]}]
+              - {get_attr: [ServiceChain, role_data, logging_groups]}
+              - [{get_attr: [LoggingConfiguration, LoggingExtraGroups]}]
       config_settings: {map_merge: {get_attr: [ServiceChain, role_data, config_settings]}}
       global_config_settings:
         map_merge:

diff --git a/puppet/swift-storage.yaml b/puppet/swift-storage.yaml
index 9eb6631..899ba66 100644 (file)
--- a/puppet/swift-storage.yaml
+++ b/puppet/swift-storage.yaml
@@ -96,6 +96,12 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
+  LoggingSources:
+    type: json
+    default: []
+  LoggingGroups:
+    type: comma_delimited_list
+    default: []
 
 resources:
 
@@ -260,6 +266,8 @@ resources:
             object:
               mapped_data: # data supplied directly to this deployment configuration, etc
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
+                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
+                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
 
   SwiftStorageHieraDeploy:

diff --git a/roles_data.yaml b/roles_data.yaml
index be96cac..35c1af8 100644 (file)
--- a/roles_data.yaml
+++ b/roles_data.yaml
@@ -69,6 +69,7 @@
     - OS::TripleO::Services::TripleoFirewall
     - OS::TripleO::Services::OpenDaylight
     - OS::TripleO::Services::SensuClient
+    - OS::TripleO::Services::FluentdClient
     - OS::TripleO::Services::VipHosts
 
 - name: Compute
@@ -94,6 +95,7 @@
     - OS::TripleO::Services::NeutronSriovAgent
     - OS::TripleO::Services::OpenDaylightOvs
     - OS::TripleO::Services::SensuClient
+    - OS::TripleO::Services::FluentdClient
     - OS::TripleO::Services::VipHosts
 
 - name: BlockStorage
@@ -109,6 +111,7 @@
     - OS::TripleO::Services::TripleoPackages
     - OS::TripleO::Services::TripleoFirewall
     - OS::TripleO::Services::SensuClient
+    - OS::TripleO::Services::FluentdClient
     - OS::TripleO::Services::VipHosts
 
 - name: ObjectStorage
@@ -125,6 +128,7 @@
     - OS::TripleO::Services::TripleoPackages
     - OS::TripleO::Services::TripleoFirewall
     - OS::TripleO::Services::SensuClient
+    - OS::TripleO::Services::FluentdClient
     - OS::TripleO::Services::VipHosts
 
 - name: CephStorage
@@ -139,4 +143,5 @@
     - OS::TripleO::Services::TripleoPackages
     - OS::TripleO::Services::TripleoFirewall
     - OS::TripleO::Services::SensuClient
+    - OS::TripleO::Services::FluentdClient
     - OS::TripleO::Services::VipHosts