gerrit.opnfv Code Review - ves.git/blob

1 #!/bin/bash

3 #

4 # Licensed under the Apache License, Version 2.0 (the "License");

5 # you may not use this file except in compliance with the License.

6 # You may obtain a copy of the License at

7 #

8 # http://www.apache.org/licenses/LICENSE-2.0

9 #

10 # Unless required by applicable law or agreed to in writing, software

11 # distributed under the License is distributed on an "AS IS" BASIS,

12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

13 # See the License for the specific language governing permissions and

14 # limitations under the License.

15 #

16 #. What this is: Setup script for the VES monitoring framework.

17 #. With this script VES support can be installed in one or more hosts, with:

18 #. - a dedicated or shared Kafka server for collection of events from collectd

19 #. - VES collectd agents running in host or guest mode

20 #. - VES monitor (test collector)

21 #. A typical multi-node install could involve these steps:

22 #. - Install the VES collector (for testing) on one of the hosts, or use a

23 #. pre-installed VES collector e.g. from the ONAP project.

24 #. - Install Kafka server on one of the hosts, or use a pre-installed server

25 #. accessible from the agent hosts.

26 #. - Install collectd on each host.

27 #. - Install the VES agent on each host.

28 #. - As needed, install the VES agent on each virtual host. This could include

29 #. pre-installed VES agents in VM or container images, which are configured

30 #. upon virtual host deployment, or agent install/config as part of virtual

31 #. host deploy. NOTE: support for pre-installed VES agents is a WIP.

32 #.

33 #. Prerequisites:

34 #. - Ubuntu Xenial (Centos support to be provided)

35 #. - passwordless sudo setup for user running this script

36 #. - shell environment variables setup as below (for non-default setting)

37 #. ves_mode: install mode (node|guest) for VES collectd plugin (default: node)

38 #. ves_host: VES collector IP or hostname (default: 127.0.0.1)

39 #. ves_port: VES collector port (default: 30000)

40 #. ves_path: REST path optionalRoutingPath element (default: empty)

41 #. ves_topic: REST path topicName element (default: empty)

42 #. ves_https: use HTTPS instead of HTTP (default: false)

43 #. ves_user: username for basic auth with collector (default: empty)

44 #. ves_pass: password for basic auth with collector (default: empty)

45 #. ves_interval: frequency in sec for collectd data reports (default: 20)

46 #. ves_version: VES API version (default: 5.1)

47 #. ves_kafka_port: kafka port (default: 9092)

48 #. ves_kafka_host: kafka host IP or hostname (default: 127.0.0.1)

49 #.

50 #. Usage:

51 #. git clone https://gerrit.opnfv.org/gerrit/ves /tmp/ves

52 #. bash /tmp/ves/ves-setup.sh <collector|kafka|collectd|agent>

53 #. collector: setup VES collector (test collector)

54 #. kafka: setup kafka server for VES events from collect agent(s)

55 #. collectd: setup collectd with libvirt plugin, as a kafka publisher

56 #. agent: setup VES agent in host or guest mode, as a kafka consumer

57 #. See demo_deploy.sh in this repo for a recommended sequence of the above.

58 #.

59 #. Status: this is a work in progress, under test.

61 # http://docs.opnfv.org/en/latest/submodules/barometer/docs/release/userguide/collectd.ves.userguide.html

63 function fail() {

64 log "$1"

65 exit 1

66 }

68 function log() {

69 f=$(caller 0 | awk '{print $2}')

70 l=$(caller 0 | awk '{print $1}')

71 echo ""

72 echo "$f:$l ($(date)) $1"

73 }

75 function common_prereqs() {

76 log "install common prerequisites"

77 if [[ ! -f /.dockerenv ]]; then dosudo="sudo"; fi

78 $dosudo apt-get update

79 $dosudo apt-get install -y git

80 # Required for kafka

81 $dosudo apt-get install -y default-jre

82 $dosudo apt-get install -y zookeeperd

83 $dosudo apt-get install -y python-pip

84 $dosudo pip install kafka-python

85 # Required for building collectd

86 $dosudo apt-get install -y pkg-config

87 }

89 function setup_env() {

90 if [[ ! -d /tmp/ves ]]; then mkdir /tmp/ves; fi

91 cp $0 /tmp/ves

92 if [[ ! -f /tmp/ves/ves_env.sh ]]; then

93 cat <<EOF >/tmp/ves/ves_env.sh

94 #!/bin/bash

95 ves_mode="${ves_mode:=node}"

96 ves_host="${ves_host:=127.0.0.1}"

97 ves_hostname="${ves_hostname:=localhost}"

98 ves_port="${ves_port:=30000}"

99 ves_path="${ves_path:=}"

100 ves_topic="${ves_topic:=}"

101 ves_https="${ves_https:=false}"

102 ves_user="${ves_user:=}"

103 ves_pass="${ves_pass:=}"

104 ves_interval="${ves_interval:=20}"

105 ves_version="${ves_version:=5.1}"

106 ves_kafka_host="${ves_kafka_host:=127.0.0.1}"

107 ves_kafka_port="${ves_kafka_port:=9092}"

108 export ves_mode

109 export ves_host

110 export ves_hostname

111 export ves_port

112 export ves_path

113 export ves_topic

114 export ves_https

115 export ves_user

116 export ves_pass

117 export ves_interval

118 export ves_kafka_hostame

119 export ves_kafka_port

120 EOF

121 fi

122 source /tmp/ves/ves_env.sh

123 }

124

125 function setup_kafka() {

126 log "setup kafka server"

127 common_prereqs

128 setup_env

129

130 cd /tmp/ves

131 ver="0.11.0.2"

132 log "get and unpack kafka_2.11-$ver.tgz"

133 wget "http://www-eu.apache.org/dist/kafka/$ver/kafka_2.11-$ver.tgz"

134 tar -xvzf kafka_2.11-$ver.tgz

135

136 log "set delete.topic.enable=true"

137 sed -i -- 's/#delete.topic.enable=true/delete.topic.enable=true/' \

138 kafka_2.11-$ver/config/server.properties

139 grep delete.topic.enable kafka_2.11-$ver/config/server.properties

140 # TODO: Barometer VES guide to clarify hostname must be in /etc/hosts

141 sudo nohup kafka_2.11-$ver/bin/kafka-server-start.sh \

142 kafka_2.11-$ver/config/server.properties \

143 > kafka_2.11-$ver/kafka.log 2>&1 &

144 }

145

146 function setup_kafka_client() {

147 log "Install Apache Kafka C/C++ client library"

148 if [[ ! -f /.dockerenv ]]; then dosudo="sudo"; fi

149 $dosudo apt-get install -y build-essential

150 git clone https://github.com/edenhill/librdkafka.git ~/librdkafka

151 cd ~/librdkafka

152 git checkout -b v0.9.5 v0.9.5

153 # TODO: Barometer VES guide to clarify specific prerequisites for Ubuntu

154 $dosudo apt-get install -y libpthread-stubs0-dev

155 $dosudo apt-get install -y libssl-dev

156 $dosudo apt-get install -y libsasl2-dev

157 $dosudo apt-get install -y liblz4-dev

158 ./configure --prefix=/usr

159 make

160 $dosudo make install

161 }

162

163 function setup_collectd() {

164 log "setup collectd"

165 common_prereqs

166 setup_env

167

168 log "cleanup any previous failed install"

169 sudo rm -rf ~/collectd-virt

170 sudo rm -rf ~/librdkafka

171 sudo rm -rf ~/collectd

172

173 setup_kafka_client

174

175 log "Build collectd with Kafka support"

176 git clone https://github.com/collectd/collectd.git ~/collectd

177 cd ~/collectd

178 # TODO: Barometer VES guide to clarify specific prerequisites for Ubuntu

179 sudo apt-get install -y flex bison

180 sudo apt-get install -y autoconf

181 sudo apt-get install -y libtool

182 ./build.sh

183 ./configure --with-librdkafka=/usr --without-perl-bindings --enable-perl=no

184 make

185 sudo make install

186

187 # TODO: Barometer VES guide to clarify collectd.service is correct

188 log "install collectd as a service"

189 sed -i -- 's~ExecStart=/usr/sbin/collectd~ExecStart=/opt/collectd/sbin/collectd~'\

190 contrib/systemd.collectd.service

191 sed -i -- 's~EnvironmentFile=-/etc/sysconfig/collectd~EnvironmentFile=-/opt/collectd/etc/~'\

192 contrib/systemd.collectd.service

193 sed -i -- 's~EnvironmentFile=-/etc/default/collectd~EnvironmentFile=-/opt/collectd/etc/~'\

194 contrib/systemd.collectd.service

195 sed -i -- 's~CapabilityBoundingSet=~CapabilityBoundingSet=CAP_SETUID CAP_SETGID~'\

196 contrib/systemd.collectd.service

197

198 sudo cp contrib/systemd.collectd.service /etc/systemd/system/

199 cd /etc/systemd/system/

200 sudo mv systemd.collectd.service collectd.service

201 sudo chmod +x collectd.service

202 sudo systemctl daemon-reload

203 sudo systemctl start collectd.service

204

205 log "setup VES collectd config for VES $ves_mode mode"

206 if [[ "$ves_mode" == "node" ]]; then

207 # TODO: Barometer VES guide to clarify prerequisites install for Ubuntu

208 log "setup additional prerequisites for VES host mode"

209 sudo apt-get install -y libxml2-dev libpciaccess-dev libyajl-dev \

210 libdevmapper-dev

211

212 # TODO: install libvirt from source to enable all features per

213 # http://docs.opnfv.org/en/latest/submodules/barometer/docs/release/userguide/feature.userguide.html#virt-plugin

214 sudo systemctl start libvirtd

215

216 # TODO: supposed to be merged with main collectd repo, but without this

217 # collectd still fails "plugin_load: Could not find plugin "virt" in /opt/collectd/lib/collectd"

218 rm -rf /tmp/ves/collectd-virt

219 git clone https://github.com/maryamtahhan/collectd /tmp/ves/collectd-virt

220 cd /tmp/ves/collectd-virt

221 ./build.sh

222 ./configure --enable-syslog --enable-logfile --enable-debug

223 make

224 sudo make install

225

226 # TODO: fix for journalctl -xe report "... is marked executable"

227 sudo chmod 744 /etc/systemd/system/collectd.service

228

229 cat <<EOF | sudo tee -a /opt/collectd/etc/collectd.conf

230 # for VES plugin

231 LoadPlugin logfile

232 <Plugin logfile>

233 LogLevel debug

234 File STDOUT

235 Timestamp true

236 PrintSeverity false

237 </Plugin>

238

239 LoadPlugin csv

240 <Plugin csv>

241 DataDir "/work-dir/collectd/install/var/lib/csv"

242 StoreRates false

243 </Plugin>

244

245 #LoadPlugin virt

246 #<Plugin virt>

247 # Connection "qemu:///system"

248 # RefreshInterval 60

249 # HostnameFormat uuid

250 # PluginInstanceFormat name

251 # ExtraStats "cpu_util"

252 #</Plugin>

253

254 LoadPlugin target_set

255 LoadPlugin match_regex

256 <Chain "PreCache">

257 <Rule "mark_memory_as_host">

258 <Match "regex">

259 Plugin "^memory$"

260 </Match>

261 <Target "set">

262 PluginInstance "host"

263 </Target>

264 </Rule>

265 </Chain>

266

267 LoadPlugin cpu

268 <Plugin cpu>

269 ReportByCpu true

270 ReportByState true

271 ValuesPercentage true

272 </Plugin>

273

274 LoadPlugin interface

275 LoadPlugin memory

276 LoadPlugin load

277 LoadPlugin disk

278 LoadPlugin uuid

279

280 LoadPlugin write_kafka

281 <Plugin write_kafka>

282 Property "metadata.broker.list" "$ves_kafka_host:$ves_kafka_port"

283 <Topic "collectd">

284 Format JSON

285 </Topic>

286 </Plugin>

287 EOF

288 else

289 cat <<EOF | sudo tee -a /opt/collectd/etc/collectd.conf

290 # for VES plugin

291 LoadPlugin logfile

292 <Plugin logfile>

293 LogLevel debug

294 File STDOUT

295 Timestamp true

296 PrintSeverity false

297 </Plugin>

298

299 LoadPlugin cpu

300 <Plugin cpu>

301 ReportByCpu true

302 ReportByState true

303 ValuesPercentage true

304 </Plugin>

305

306 LoadPlugin csv

307 <Plugin csv>

308 DataDir "/tmp"

309 </Plugin>

310

311 LoadPlugin interface

312 LoadPlugin memory

313 LoadPlugin load

314 LoadPlugin disk

315 LoadPlugin uuid

316

317 LoadPlugin write_kafka

318 <Plugin write_kafka>

319 Property "metadata.broker.list" "$ves_kafka_host:$ves_kafka_port"

320 <Topic "collectd">

321 Format JSON

322 </Topic>

323 </Plugin>

324

325 LoadPlugin target_set

326 LoadPlugin match_regex

327 <Chain "PreCache">

328 <Rule "mark_memory_as_guest">

329 <Match "regex">

330 Plugin "^memory$"

331 </Match>

332 <Target "set">

333 PluginInstance "guest"

334 </Target>

335 </Rule>

336 </Chain>

337 EOF

338 fi

339

340 if [[ $(grep -c $ves_hostname /etc/hosts) -eq 0 ]]; then

341 log "add to /etc/hosts: $ves_kafka_host $ves_hostname"

342 echo "$ves_kafka_host $ves_hostname" | sudo tee -a /etc/hosts

343 fi

344 log "restart collectd to apply updated config"

345 sudo systemctl restart collectd

346 }

347

348 function setup_agent() {

349 log "setup VES agent"

350 if [[ ! -f /.dockerenv ]]; then

351 log "start the ves-agent container"

352 sudo docker run -it -d -v /tmp/ves:/opt/ves --name=ves-agent \

353 ubuntu:xenial /bin/bash

354 log "execute the agent setup script in the container"

355 sudo docker exec ves-agent /bin/bash /opt/ves/ves-setup.sh agent

356 else

357 common_prereqs

358 log "setup the VES environment"

359 source /opt/ves/ves_env.sh

360 log "install agent prerequisites"

361 pip install pyaml

362

363 setup_kafka_client

364

365 log "clone OPNFV Barometer"

366 rm -rf /opt/ves/barometer

367 git clone https://gerrit.opnfv.org/gerrit/barometer /opt/ves/barometer

368 # Test patch

369 cd /opt/ves/barometer

370 git fetch https://gerrit.opnfv.org/gerrit/barometer refs/changes/27/47427/1 && git checkout FETCH_HEAD

371

372 log "setup ves_app_config.conf"

373 source /opt/ves/ves_env.sh

374 cd /opt/ves/barometer/3rd_party/collectd-ves-app/ves_app

375 cat <<EOF >ves_app_config.conf

376 [config]

377 Domain = $ves_host

378 Port = $ves_port

379 Path = $ves_path

380 Topic = $ves_topic

381 UseHttps = $ves_https

382 Username = $ves_user

383 Password = $ves_pass

384 SendEventInterval = $ves_interval

385 ApiVersion = $ves_version

386 KafkaPort = $ves_kafka_port

387 KafkaBroker = $ves_kafka_host

388 EOF

389

390 # log "add guest.yaml measurements to host.yaml (enables actual host data)"

391 # tail --lines=+24 guest.yaml >>host.yaml

392

393 log "start VES agent"

394 echo "$ves_kafka_host $ves_hostname">>/etc/hosts

395 nohup python ves_app.py --events-schema=$ves_mode.yaml --loglevel ERROR \

396 --config=ves_app_config.conf > /opt/ves/ves_app.stdout.log 2>&1 &

397 fi

398 }

399

400 function setup_collector() {

401 log "setup collector"

402 log "install prerequistes"

403 sudo apt-get install -y jq

404

405 ves_hostname=$HOSTNAME

406 export ves_hostname

407 ves_host=$(ip route get 8.8.8.8 | awk '{print $NF; exit}')

408 export ves_host

409 setup_env

410

411 log "setup influxdb container"

412 sudo docker run -d --name=ves-influxdb -p 8086:8086 influxdb

413 status=$(sudo docker inspect ves-influxdb | jq -r '.[0].State.Status')

414 while [[ "x$status" != "xrunning" ]]; do

415 log "InfluxDB container state is ($status)"

416 sleep 10

417 status=$(sudo docker inspect ves-influxdb | jq -r '.[0].State.Status')

418 done

419 log "InfluxDB container state is $status"

420

421 log "wait for InfluxDB API to be active"

422 while ! curl http://$ves_host:8086/ping ; do

423 log "InfluxDB API is not yet responding... waiting 10 seconds"

424 sleep 10

425 done

426

427 log "setup InfluxDB database"

428 curl -X POST http://$ves_host:8086/query \

429 --data-urlencode "q=CREATE DATABASE veseventsdb"

430

431 log "install Grafana container"

432 sudo docker run -d --name ves-grafana -p 3001:3000 grafana/grafana

433 status=$(sudo docker inspect ves-grafana | jq -r '.[0].State.Status')

434 while [[ "x$status" != "xrunning" ]]; do

435 log "Grafana container state is ($status)"

436 sleep 10

437 status=$(sudo docker inspect ves-grafana | jq -r '.[0].State.Status')

438 done

439 log "Grafana container state is $status"

440

441 log "wait for Grafana API to be active"

442 while ! curl http://$ves_host:3001 ; do

443 log "Grafana API is not yet responding... waiting 10 seconds"

444 sleep 10

445 done

446

447 log "add VESEvents datasource to Grafana"

448 cat <<EOF >/tmp/ves/datasource.json

449 { "name":"VESEvents",

450 "type":"influxdb",

451 "access":"direct",

452 "url":"http://$ves_host:8086",

453 "password":"root",

454 "user":"root",

455 "database":"veseventsdb",

456 "basicAuth":false,

457 "basicAuthUser":"",

458 "basicAuthPassword":"",

459 "withCredentials":false,

460 "isDefault":false,

461 "jsonData":null

462 }

463 EOF

464

465 curl -H "Accept: application/json" -H "Content-type: application/json" \

466 -X POST -d @/tmp/ves/datasource.json \

467 http://admin:admin@$ves_host:3001/api/datasources

468

469 log "add VES dashboard to Grafana"

470 curl -H "Accept: application/json" -H "Content-type: application/json" \

471 -X POST \

472 -d @/tmp/ves/tools/grafana/Dashboard.json\

473 http://admin:admin@$ves_host:3001/api/dashboards/db

474

475 log "setup collector container"

476 cd /tmp/ves

477 touch monitor.log

478 rm -rf /tmp/ves/evel-test-collector

479 git clone https://github.com/att/evel-test-collector.git

480 sed -i -- \

481 "s~log_file = /var/log/att/collector.log~log_file = /opt/ves/collector.log~" \

482 evel-test-collector/config/collector.conf

483 sed -i -- "s/vel_domain = 127.0.0.1/vel_domain = $ves_host/g" \

484 evel-test-collector/config/collector.conf

485 sed -i -- "s/vel_username =/vel_username = $ves_user/g" \

486 evel-test-collector/config/collector.conf

487 sed -i -- "s/vel_password =/vel_password = $ves_pass/g" \

488 evel-test-collector/config/collector.conf

489 sed -i -- "s~vel_path = vendor_event_listener/~vel_path = $ves_path~g" \

490 evel-test-collector/config/collector.conf

491 sed -i -- "s~vel_topic_name = example_vnf~vel_topic_name = $ves_topic~g" \

492 evel-test-collector/config/collector.conf

493 sed -i -- "/vel_topic_name = /a influxdb = $ves_host" \

494 evel-test-collector/config/collector.conf

495

496 cp /tmp/ves/tools/monitor.py \

497 evel-test-collector/code/collector/monitor.py

498

499 # Note below: python (2.7) is required due to dependency on module 'ConfigParser'

500 cat <<EOF >/tmp/ves/setup-collector.sh

501 apt-get update

502 apt-get upgrade -y

503 apt-get install -y python python-jsonschema python-pip

504 pip install requests

505 python /opt/ves/evel-test-collector/code/collector/monitor.py \

506 --config /opt/ves/evel-test-collector/config/collector.conf \

507 --influxdb $ves_host \

508 --section default > /opt/ves/monitor.log 2>&1 &

509 EOF

510

511 sudo docker run -it -d -v /tmp/ves:/opt/ves --name=ves-collector \

512 -p 30000:30000 ubuntu:xenial /bin/bash

513 sudo docker exec ves-collector /bin/bash /opt/ves/setup-collector.sh

514 # debug hints

515 # sudo docker exec -it ves-collector apt-get install -y tcpdump

516 # sudo docker exec -it ves-collector tcpdump -A -v -s 0 -i any port 30000

517 # curl http://$ves_host:30000

518 # sudo docker exec -it ves-collector /bin/bash

519 # ~/kafka_2.11-0.11.0.2/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic collectd

520 }

521

522 function clean() {

523 log "clean installation for $1 at $2"

524 if [[ "$1" == "master" ]]; then

525 cs="ves-agent ves-collector ves-grafana ves-influxdb"

526 for c in $cs; do

527 log "stop and remove container $c"

528 sudo docker stop $c

529 sudo docker rm -v $c

530 done

531 fi

532 log "remove collectd config for VES"

533 sudo sed -i -- '/VES plugin/,$d' /opt/collectd/etc/collectd.conf

534 sudo systemctl restart collectd

535 sudo rm -rf /tmp/ves

536 }

537

538 dist=`grep DISTRIB_ID /etc/*-release | awk -F '=' '{print $2}'`

539 if [[ $(grep -c $HOSTNAME /etc/hosts) -eq 0 ]]; then

540 echo "$(ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME" |\

541 sudo tee -a /etc/hosts

542 fi

543

544 case "$1" in

545 "collectd")

546 setup_collectd

547 ;;

548 "agent")

549 setup_agent

550 ;;

551 "collector")

552 setup_collector

553 ;;

554 "kafka")

555 setup_kafka

556 ;;

557 "clean")

558 clean $2 $3

559 ;;

560 *)

561 grep '#. ' $0

562 esac