Add more collectd data to influx/grafana

JIRA: VES-2
Disable uuid for "node" mode

Change-Id: I055db4f52eee7a61438a4e7e63740b42dc3f1777
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>

diff --git a/tools/grafana/Dashboard.json b/tools/grafana/Dashboard.json
index 3160296..2ea9eec 100644 (file)
--- a/tools/grafana/Dashboard.json
+++ b/tools/grafana/Dashboard.json
@@ -11,7 +11,7 @@
   "rows": [
     {
       "collapse": false,
-      "height": 273,
+      "height": 401,
       "panels": [
         {
           "aliasColors": {},
@@ -31,13 +31,15 @@
           "id": 3,
           "interval": "30s",
           "legend": {
-            "alignAsTable": false,
+            "alignAsTable": true,
             "avg": true,
             "current": true,
             "max": true,
             "min": true,
             "rightSide": false,
             "show": true,
+            "sort": "current",
+            "sortDesc": true,
             "total": false,
             "values": true
           },
@@ -79,6 +81,12 @@
                       "load-shortterm"
                     ],
                     "type": "field"
+                  },
+                  {
+                    "params": [
+                      "5"
+                    ],
+                    "type": "moving_average"
                   }
                 ]
               ],
@@ -140,15 +148,28 @@
           "dashes": false,
           "datasource": "VESEvents",
           "fill": 1,
-          "id": 5,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null
+          },
+          "id": 6,
+          "interval": "30s",
           "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
             "show": true,
+            "sort": "current",
+            "sortDesc": true,
             "total": false,
-            "values": false
+            "values": true
           },
           "lines": true,
           "linewidth": 1,
@@ -173,20 +194,33 @@
                     "system"
                   ],
                   "type": "tag"
+                },
+                {
+                  "params": [
+                    "cpu"
+                  ],
+                  "type": "tag"
                 }
               ],
-              "measurement": "memoryUsage",
+              "measurement": "cpuUsage",
               "orderByTime": "ASC",
-              "policy": "autogen",
-              "refId": "A",
+              "policy": "default",
+              "query": "SELECT mean(\"cpusystem\") FROM \"cpu\" WHERE $timeFilter GROUP BY time(1m) fill(null)",
+              "refId": "B",
               "resultFormat": "time_series",
               "select": [
                 [
                   {
                     "params": [
-                      "memoryFree"
+                      "cpuUsageUser"
                     ],
                     "type": "field"
+                  },
+                  {
+                    "params": [
+                      "5"
+                    ],
+                    "type": "moving_average"
                   }
                 ]
               ],
@@ -202,13 +236,14 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Memory",
+          "title": "host CPU Usage User",
           "tooltip": {
             "shared": true,
             "sort": 0,
             "value_type": "individual"
           },
           "type": "graph",
+          "x-axis": true,
           "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -216,11 +251,16 @@
             "show": true,
             "values": []
           },
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ],
           "yaxes": [
             {
               "format": "short",
-              "label": null,
-              "logBase": 10,
+              "label": "Percent",
+              "logBase": 1,
               "max": null,
               "min": null,
               "show": true
@@ -245,7 +285,7 @@
     },
     {
       "collapse": false,
-      "height": 433,
+      "height": 442,
       "panels": [
         {
           "aliasColors": {},
@@ -267,11 +307,13 @@
           "legend": {
             "alignAsTable": true,
             "avg": true,
-            "current": false,
-            "max": false,
-            "min": false,
+            "current": true,
+            "max": true,
+            "min": true,
             "rightSide": false,
             "show": true,
+            "sort": "current",
+            "sortDesc": true,
             "total": false,
             "values": true
           },
@@ -319,6 +361,12 @@
                       "receivedTotalPacketsAccumulated"
                     ],
                     "type": "field"
+                  },
+                  {
+                    "params": [
+                      "5"
+                    ],
+                    "type": "moving_average"
                   }
                 ]
               ],
@@ -393,11 +441,13 @@
           "legend": {
             "alignAsTable": true,
             "avg": true,
-            "current": false,
-            "max": false,
-            "min": false,
+            "current": true,
+            "max": true,
+            "min": true,
             "rightSide": false,
             "show": true,
+            "sort": "current",
+            "sortDesc": true,
             "total": false,
             "values": true
           },
@@ -444,6 +494,12 @@
                       "receivedOctetsAccumulated"
                     ],
                     "type": "field"
+                  },
+                  {
+                    "params": [
+                      "5"
+                    ],
+                    "type": "moving_average"
                   }
                 ]
               ],
@@ -505,6 +561,246 @@
       "showTitle": false,
       "title": "Dashboard Row",
       "titleSize": "h6"
+    },
+    {
+      "collapse": false,
+      "height": 362,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "VESEvents",
+          "fill": 1,
+          "id": 5,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "show": true,
+            "sort": "current",
+            "sortDesc": false,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "alias": "",
+              "dsType": "influxdb",
+              "groupBy": [
+                {
+                  "params": [
+                    "system"
+                  ],
+                  "type": "tag"
+                }
+              ],
+              "measurement": "memoryUsage",
+              "orderByTime": "ASC",
+              "policy": "autogen",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "memoryFree"
+                    ],
+                    "type": "field"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "system",
+                  "operator": "=~",
+                  "value": "/^$host$/"
+                }
+              ]
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Memory",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 10,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "VESEvents",
+          "fill": 1,
+          "id": 7,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "alias": "",
+              "dsType": "influxdb",
+              "groupBy": [
+                {
+                  "params": [
+                    "system"
+                  ],
+                  "type": "tag"
+                },
+                {
+                  "params": [
+                    "disk"
+                  ],
+                  "type": "tag"
+                }
+              ],
+              "measurement": "diskUsage",
+              "orderByTime": "ASC",
+              "policy": "autogen",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "diskOpsWriteLast"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [
+                      "5"
+                    ],
+                    "type": "moving_average"
+                  }
+                ]
+              ],
+              "tags": [
+                {
+                  "key": "system",
+                  "operator": "=~",
+                  "value": "/^$host$/"
+                },
+                {
+                  "condition": "AND",
+                  "key": "disk",
+                  "operator": "=",
+                  "value": "sda"
+                }
+              ]
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Disk Usage",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 10,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        }
+      ],
+      "repeat": null,
+      "repeatIteration": null,
+      "repeatRowId": null,
+      "showTitle": false,
+      "title": "Dashboard Row",
+      "titleSize": "h6"
     }
   ],
   "schemaVersion": 14,
@@ -514,48 +810,16 @@
     "list": [
       {
         "allValue": null,
-        "current": {
-          "tags": [],
-          "text": "All",
-          "value": [
-            "$__all"
-          ]
-        },
+        "current": {},
         "datasource": "VESEvents",
         "hide": 0,
         "includeAll": true,
         "label": "host",
         "multi": true,
         "name": "host",
-        "options": [
-          {
-            "selected": true,
-            "text": "All",
-            "value": "$__all"
-          },
-          {
-            "selected": false,
-            "text": "4C4C4544-004C-4E10-8054-B6C04F475831",
-            "value": "4C4C4544-004C-4E10-8054-B6C04F475831"
-          },
-          {
-            "selected": false,
-            "text": "4C4C4544-004C-4E10-8057-B6C04F475831",
-            "value": "4C4C4544-004C-4E10-8057-B6C04F475831"
-          },
-          {
-            "selected": false,
-            "text": "4C4C4544-004C-5010-8053-B6C04F475831",
-            "value": "4C4C4544-004C-5010-8053-B6C04F475831"
-          },
-          {
-            "selected": false,
-            "text": "4C4C4544-004E-3110-8051-C3C04F445631",
-            "value": "4C4C4544-004E-3110-8051-C3C04F445631"
-          }
-        ],
+        "options": [],
         "query": "SHOW TAG VALUES WITH KEY=system",
-        "refresh": 0,
+        "refresh": 1,
         "regex": "",
         "sort": 0,
         "tagValuesQuery": "",
@@ -592,6 +856,6 @@
   },
   "timezone": "browser",
   "title": "VES Demo",
-  "version": 2
+  "version": 3
 }
 }

diff --git a/tools/monitor.py b/tools/monitor.py
index 91c0eae..9405b48 100644 (file)
--- a/tools/monitor.py
+++ b/tools/monitor.py
@@ -291,6 +291,60 @@ def save_event(body):
         pdata = pdata[:i] + ' ' + pdata[i+1:]
         send_to_influxdb("systemLoad", pdata)
 
+#            "cpuUsageArray": [
+#                {
+#                    "cpuIdentifier": "15",
+#                    "cpuIdle": 99.8998998999,
+#                    "cpuUsageInterrupt": 0,
+#                    "cpuUsageNice": 0,
+#                    "cpuUsageSoftIrq": 0,
+#                    "cpuUsageSteal": 0,
+#                    "cpuUsageSystem": 0,
+#                    "cpuUsageUser": 0.1001001001,
+#                    "cpuWait": 0,
+#                    "percentUsage": 0.0
+#                },
+
+    if 'cpuUsageArray' in jobj['event']['measurementsForVfScalingFields']:
+      print('Found diskUsageArray')
+      for disk in jobj['event']['measurementsForVfScalingFields']['cpuUsageArray']:
+        id=disk['cpuIdentifier']
+        pdata = 'cpuUsage,system={},cpu={}'.format(source,id)
+        d = disk.items()
+        for key,val in d:
+          if key != 'cpuIdentifier':
+            pdata = pdata + ',{}={}'.format(key,val)
+        i=pdata.find(',', pdata.find('cpu='))
+        pdata = pdata[:i] + ' ' + pdata[i+1:]
+        send_to_influxdb("cpuUsage", pdata)
+
+#            "diskUsageArray": [
+#                {
+#                    "diskIdentifier": "sda",
+#                    "diskIoTimeLast": 0.3996139893,
+#                    "diskMergedReadLast": 0,
+#                    "diskMergedWriteLast": 26.1747155344,
+#                    "diskOctetsReadLast": 0,
+#                    "diskOctetsWriteLast": 309767.93302,
+#                    "diskOpsReadLast": 0,
+#                    "diskOpsWriteLast": 10.9893839563,
+#                    "diskTimeReadLast": 0,
+#                    "diskTimeWriteLast": 0.699324445683
+#                },
+
+    if 'diskUsageArray' in jobj['event']['measurementsForVfScalingFields']:
+      print('Found diskUsageArray')
+      for disk in jobj['event']['measurementsForVfScalingFields']['diskUsageArray']:
+        id=disk['diskIdentifier']
+        pdata = 'diskUsage,system={},disk={}'.format(source,id)
+        d = disk.items()
+        for key,val in d:
+          if key != 'diskIdentifier':
+            pdata = pdata + ',{}={}'.format(key,val)
+        i=pdata.find(',', pdata.find('disk='))
+        pdata = pdata[:i] + ' ' + pdata[i+1:]
+        send_to_influxdb("diskUsage", pdata)
+
 #            "memoryUsageArray": [
 #                {
 #                    "memoryBuffered": 269056.0,

diff --git a/tools/ves-setup.sh b/tools/ves-setup.sh
index 6e20bfd..7b6bddb 100644 (file)
--- a/tools/ves-setup.sh
+++ b/tools/ves-setup.sh
@@ -242,6 +242,7 @@ LoadPlugin csv
  StoreRates false
 </Plugin>
 
+# TODO: complete the virt plugin install before enabling
 #LoadPlugin virt
 #<Plugin virt>
 #  Connection "qemu:///system"
@@ -275,7 +276,8 @@ LoadPlugin interface
 LoadPlugin memory
 LoadPlugin load
 LoadPlugin disk
-LoadPlugin uuid
+# TODO: how to set this option only to apply to VMs (not nodes)
+#LoadPlugin uuid
 
 LoadPlugin write_kafka
 <Plugin write_kafka>
@@ -337,6 +339,8 @@ LoadPlugin match_regex
 EOF
   fi
 
+#  sudo sed -i -- "s/#Hostname    \"localhost\"/Hostname    \"$HOSTNAME\"/" /opt/collectd/etc/collectd.conf
+
   if [[ $(grep -c $ves_hostname /etc/hosts) -eq 0 ]]; then
     log "add to /etc/hosts: $ves_kafka_host $ves_hostname"
     echo "$ves_kafka_host $ves_hostname" | sudo tee -a /etc/hosts