From bfb1d7536e376244a48275afdbb5693c80571ed1 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Wed, 29 Nov 2017 11:48:19 +0000 Subject: [PATCH] docker: add kafka support to collectd Adds kafka support to collectd, cleaned up configuration files and updates documentation. Change-Id: Ie66ed22e8904e08e59aa7047d6afc9d176f3fb02 Signed-off-by: Maryam Tahhan --- docs/release/userguide/docker.userguide.rst | 11 ++-- src/Makefile | 1 + src/collectd/Makefile | 1 + src/collectd/collectd_sample_configs/csv.conf | 2 +- .../collectd_sample_configs/default_plugins.conf | 31 +++++++++ src/collectd/collectd_sample_configs/dpdkstat.conf | 2 +- src/collectd/collectd_sample_configs/exec.conf | 5 +- .../collectd_sample_configs/hugepages.conf | 2 +- src/collectd/collectd_sample_configs/kafka.conf | 21 ++++++ .../collectd_sample_configs/ovs_events.conf | 3 +- src/collectd/collectd_sample_configs/rdt.conf | 2 +- .../collectd_sample_configs/write_notification.sh | 2 +- src/librdkafka/Makefile | 75 ++++++++++++++++++++++ src/package-list.mk | 3 + systems/centos/7/build_base_machine.sh | 1 - 15 files changed, 146 insertions(+), 16 deletions(-) create mode 100644 src/collectd/collectd_sample_configs/default_plugins.conf create mode 100644 src/collectd/collectd_sample_configs/kafka.conf create mode 100644 src/librdkafka/Makefile diff --git a/docs/release/userguide/docker.userguide.rst b/docs/release/userguide/docker.userguide.rst index c6f56b2b..46590eb0 100644 --- a/docs/release/userguide/docker.userguide.rst +++ b/docs/release/userguide/docker.userguide.rst @@ -47,6 +47,7 @@ to the standard collectd plugins): * RDT plugin * virt * SNMP Agent +* Kafka_write plugin Plugins and third party applications in Barometer repository that will be available in the docker image: @@ -251,13 +252,13 @@ Run the barometer docker image: The docker barometer image contains configuration for all the collectd plugins. In the command above we are overriding /opt/collectd/etc/collectd.conf.d by mounting a host directory `pwd`/../src/collectd_sample_configs thta contains only the sample configurations we are interested - in running. + in running. It's important to do this if you don't have DPDK, or RDT installed on the host. To make some changes run: .. code:: bash - sudo docker exec -tid barometer_image /bin/bash + sudo docker exec -ti barometer_image /bin/bash Check your docker image is running @@ -267,8 +268,8 @@ Check your docker image is running Build the influxdb + Grafana docker images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Start by installing docker compose: +On the node where you want to run influxdb + grafana: +1. Start by installing docker compose .. code:: bash @@ -309,7 +310,7 @@ Start by installing docker compose: .. code:: bash - $ ./configure_grafana.sh + $ cd dashboards && ./configure_grafana.sh 8. Connect to :3000 with a browser and log into grafana: admin/admin diff --git a/src/Makefile b/src/Makefile index 4785e854..7c6c17cc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -40,6 +40,7 @@ SUBDIRS += dpdk endif SUBDIRS += libpqos SUBDIRS += pmu-tools +SUBDIRS += librdkafka SUBDIRS += collectd SUBDIRS += collectd-openstack-plugins diff --git a/src/collectd/Makefile b/src/collectd/Makefile index decb8e84..41fe89aa 100644 --- a/src/collectd/Makefile +++ b/src/collectd/Makefile @@ -48,6 +48,7 @@ CONFIG_CMD += --enable-exec CONFIG_CMD += --enable-python CONFIG_CMD += --enable-threshold CONFIG_CMD += --disable-perl +CONFIG_CMD += --with-librdkafka=/usr .PHONY: install force_install config force_make diff --git a/src/collectd/collectd_sample_configs/csv.conf b/src/collectd/collectd_sample_configs/csv.conf index e2350039..99a8498d 100644 --- a/src/collectd/collectd_sample_configs/csv.conf +++ b/src/collectd/collectd_sample_configs/csv.conf @@ -1,4 +1,4 @@ -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/collectd/collectd_sample_configs/default_plugins.conf b/src/collectd/collectd_sample_configs/default_plugins.conf new file mode 100644 index 00000000..c96b0076 --- /dev/null +++ b/src/collectd/collectd_sample_configs/default_plugins.conf @@ -0,0 +1,31 @@ +# Copyright 2017 OPNFV +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#Hostname "" + +LoadPlugin cpufreq +LoadPlugin disk +LoadPlugin ethstat +LoadPlugin ipc +LoadPlugin ipmi +LoadPlugin load +LoadPlugin memory +LoadPlugin numa +LoadPlugin processes +LoadPlugin df +LoadPlugin turbostat +LoadPlugin uptime +LoadPlugin contextswitch +LoadPlugin irq +LoadPlugin df +LoadPlugin swap diff --git a/src/collectd/collectd_sample_configs/dpdkstat.conf b/src/collectd/collectd_sample_configs/dpdkstat.conf index 201f9d37..919e6e6e 100644 --- a/src/collectd/collectd_sample_configs/dpdkstat.conf +++ b/src/collectd/collectd_sample_configs/dpdkstat.conf @@ -1,4 +1,4 @@ -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/collectd/collectd_sample_configs/exec.conf b/src/collectd/collectd_sample_configs/exec.conf index e36409e5..0a291bdb 100644 --- a/src/collectd/collectd_sample_configs/exec.conf +++ b/src/collectd/collectd_sample_configs/exec.conf @@ -1,4 +1,4 @@ -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,9 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - LoadPlugin exec + # For OVS PMD stats plugin Exec "collectd_exec" "/opt/collectd/etc/collectd.conf.d/ovs_pmd_stats.sh" diff --git a/src/collectd/collectd_sample_configs/hugepages.conf b/src/collectd/collectd_sample_configs/hugepages.conf index 4e4d9152..558b6f50 100644 --- a/src/collectd/collectd_sample_configs/hugepages.conf +++ b/src/collectd/collectd_sample_configs/hugepages.conf @@ -1,4 +1,4 @@ -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/collectd/collectd_sample_configs/kafka.conf b/src/collectd/collectd_sample_configs/kafka.conf new file mode 100644 index 00000000..f81e87fd --- /dev/null +++ b/src/collectd/collectd_sample_configs/kafka.conf @@ -0,0 +1,21 @@ +# Copyright 2017 OPNFV +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +LoadPlugin write_kafka + + + Property "metadata.broker.list" "localhost:9092" + + Format JSON + + diff --git a/src/collectd/collectd_sample_configs/ovs_events.conf b/src/collectd/collectd_sample_configs/ovs_events.conf index a52842ca..250e1e43 100644 --- a/src/collectd/collectd_sample_configs/ovs_events.conf +++ b/src/collectd/collectd_sample_configs/ovs_events.conf @@ -1,4 +1,4 @@ -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - Interval 1 diff --git a/src/collectd/collectd_sample_configs/rdt.conf b/src/collectd/collectd_sample_configs/rdt.conf index 4ff0a77a..ae983dc0 100644 --- a/src/collectd/collectd_sample_configs/rdt.conf +++ b/src/collectd/collectd_sample_configs/rdt.conf @@ -1,4 +1,4 @@ -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/collectd/collectd_sample_configs/write_notification.sh b/src/collectd/collectd_sample_configs/write_notification.sh index 28198f9d..ed6ed9e2 100755 --- a/src/collectd/collectd_sample_configs/write_notification.sh +++ b/src/collectd/collectd_sample_configs/write_notification.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/librdkafka/Makefile b/src/librdkafka/Makefile new file mode 100644 index 00000000..ea5e34e2 --- /dev/null +++ b/src/librdkafka/Makefile @@ -0,0 +1,75 @@ +# makefile to manage collectd package +# Copyright 2017 OPNFV +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Contributors: +# Aihua Li, Huawei Technologies. +# Maryam Tahhan, Intel Corporation. + +include ../mk/master.mk +include ../package-list.mk + +WORK_DIR = librdkafka +TAG_DONE_FLAG = $(WORK_DIR)/.$(KAFKA_TAG).done +CONFIG_CMD = +CONFIG_CMD += ./configure +CONFIG_CMD += --prefix=/usr + +.PHONY: install force_install force_make + +# install depends on make +force_install: force_make + +all: force_make + @echo "Finished making $(WORK_DIR) " + +INSTALL_TARGET = force_install force_make + +force_make: $(WORK_DIR) + $(AT)cd $(WORK_DIR); git checkout -b $(KAFKA_TAG) $(KAFKA_TAG); $(CONFIG_CMD); cd - + $(AT)$(MAKE) -C $(WORK_DIR) + @echo "Make done" + +force_install: + $(AT)sudo $(MAKE) -C $(WORK_DIR) install + +install: $(INSTALL_TARGET) + +# hard way to clean and clobber +clean: + $(AT)cd $(WORK_DIR) && git clean -xfd *.o +clobber: + $(AT)rm -rf $(WORK_DIR) + +# distclean is for developer who would like to keep the +# clone git repo, saving time to fetch again from url +distclean: + $(AT)cd $(WORK_DIR) && git clean -xfd && git checkout -f + +test: + @echo "Make test in $(WORK_DIR) (stub) " + +sanity: + @echo "Make sanity in $(WORK_DIR) (stub) " + +$(WORK_DIR): + $(AT)git clone $(KAFKA_URL) + +$(TAG_DONE_FLAG): $(WORK_DIR) + $(AT)cd $(WORK_DIR); git checkout $(KAFKA_TAG) +ifneq ($(PATCH_FILE),) + $(AT)cd $(WORK_DIR); patch -p1 < ../$(PATCH_FILE) +endif + $(AT)touch $@ diff --git a/src/package-list.mk b/src/package-list.mk index a059bd55..3a92a1f7 100644 --- a/src/package-list.mk +++ b/src/package-list.mk @@ -14,6 +14,9 @@ LIBPQOS_TAG ?= master PMUTOOLS_URL ?= https://github.com/andikleen/pmu-tools PMUTOOLS_TAG ?= master +KAFKA_URL ?= https://github.com/edenhill/librdkafka.git +KAFKA_TAG ?= v0.9.5 + # collectd section COLLECTD_URL ?= https://github.com/collectd/collectd COLLECTD_TAG ?= collectd-5.8 diff --git a/systems/centos/7/build_base_machine.sh b/systems/centos/7/build_base_machine.sh index 9c49d493..18539315 100755 --- a/systems/centos/7/build_base_machine.sh +++ b/systems/centos/7/build_base_machine.sh @@ -58,4 +58,3 @@ net-snmp-devel # install epel release required for git-review epel-release " | grep -v ^#) - -- 2.16.6