--- /dev/null
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from datetime import timedelta
+import pprint
+import requests
+import time
+
+PROMETHEUS_URL = "http://127.0.0.1:9090"
+
+
+class Monitoring(object):
+ PROMETHEUS_HEALTH_UP = "up"
+ PROMETHEUS_ISTIO_TARGETS = {"envoy",
+ "istio-mesh",
+ "kubernetes-apiservers",
+ "kubernetes-cadvisor",
+ "kubernetes-nodes",
+ "kubernetes-service-endpoints",
+ "mixer",
+ "pilot"}
+ PROMETHEUS_API_TARGETS = "/api/v1/targets"
+ PROMETHEUS_API_QUERY = "/api/v1/query"
+ PROMETHEUS_API_QUERY_RANGE = "/api/v1/query_range"
+
+ def __init__(self, host):
+ self.host = host
+
+ def get_targets(self):
+ try:
+ # Reference api: https://prometheus.io/docs/prometheus/latest/querying/api/#targets
+ response = requests.get('%s%s' % (self.host, Monitoring.PROMETHEUS_API_TARGETS))
+ if response.status_code != 200:
+ print("ERROR: get targets status code: %r" % response.status_code)
+ return False
+ except Exception as e:
+ print("ERROR: Cannot connect to prometheus\n%s" % e)
+ return False
+
+ return response.json()
+
+ def is_targets_healthy(self):
+ targets = set()
+
+ raw_targets = self.get_targets()
+ if raw_targets == False:
+ return False
+
+ for target in raw_targets["data"]["activeTargets"]:
+ if target["health"] != Monitoring.PROMETHEUS_HEALTH_UP:
+ print("ERROR: target unhealth job: %s, health: %s" % \
+ (target["labels"]["job"], target["health"]))
+ return False
+ targets.add(target["labels"]["job"])
+
+ diff = Monitoring.PROMETHEUS_ISTIO_TARGETS - targets
+ if len(diff):
+ print("ERROR: targets %r not found!" % diff)
+ return False
+
+ return True
+
+ # Reference links:
+ # - https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
+ # - https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
+ # - https://github.com/prometheus/prombench/blob/master/apps/load-generator/main.py
+ def query(self, query_params):
+ try:
+ start = time.time()
+
+ query_type = query_params.get("type", "instant")
+ params = {"query": query_params["query"]}
+ if query_type == "instant":
+ url = "%s%s" % (self.host, Monitoring.PROMETHEUS_API_QUERY)
+ elif query_type == "range":
+ url = "%s%s" % (self.host, Monitoring.PROMETHEUS_API_QUERY_RANGE)
+ params["start"] = start - duration_seconds(query_params.get("start", "0h"))
+ params["end"] = start - duration_seconds(query_params.get("end", "0h"))
+ params["step"] = query_params.get("step", "15s")
+ else:
+ print("ERROR: invalidate query type")
+ return
+
+ resp = requests.get(url, params)
+ dur = time.time() - start
+
+ print("query %s %s, status=%s, size=%d, dur=%.3f" % \
+ (self.host, query_params["query"], resp.status_code, len(resp.text), dur))
+ pp = pprint.PrettyPrinter(indent=2)
+ pp.pprint(resp.json())
+
+ except Exception as e:
+ print("ERROR: Could not query prometheus instance %s. \n %s" % (url, e))
+
+
+def duration_seconds(s):
+ num = int(s[:-1])
+
+ if s.endswith('s'):
+ return timedelta(seconds=num).total_seconds()
+ elif s.endswith('m'):
+ return timedelta(minutes=num).total_seconds()
+ elif s.endswith('h'):
+ return timedelta(hours=num).total_seconds()
+
+ raise "ERROR: unknown duration %s" % s
+
+
+def main():
+ m = Monitoring(PROMETHEUS_URL)
+ if not m.is_targets_healthy():
+ print("ERROR: Prometheus targets is unhealthy!")
+ else:
+ print("Prometheus targets are all healthy!")
+
+ print "\n### query instant"
+ query_params = {
+ "type": "instant",
+ "query": "istio_double_request_count{destination='details.default.svc.cluster.local'}"
+ }
+ m.query(query_params)
+
+ print "\n### query range"
+ query_range_param = {
+ "type": "range",
+ "query": "istio_double_request_count{destination='details.default.svc.cluster.local'}",
+ "start": "5m",
+ "end": "3m",
+ "step": "30s"
+ }
+ m.query(query_range_param)
+
+
+if __name__ == '__main__':
+ main()
+
--- /dev/null
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from monitoring import Monitoring
+from kubernetes import client, config
+
+PROMETHEUS_URL = "http://127.0.0.1:9090"
+PROMETHEUS_DEPLOYMENT = "prometheus"
+PROMETHEUS_LABELS = "app=prometheus"
+ISTIO_NAMESPACE = "istio-system"
+
+
+def validateDeploy():
+ config.load_kube_config()
+ appsv1 = client.AppsV1Api()
+ corev1 = client.CoreV1Api()
+ find_flag = False
+ prom_pod_name = None
+
+ # check prometheus deploytment
+ ret = appsv1.list_deployment_for_all_namespaces(watch=False)
+ for i in ret.items:
+ if PROMETHEUS_DEPLOYMENT == i.metadata.name and \
+ ISTIO_NAMESPACE == i.metadata.namespace:
+ find_flag = True
+ break
+ if find_flag == False:
+ print("ERROR: Deployment: {} doesn't present in {} namespace".format(
+ PROMETHEUS_DEPLOYMENT, ISTIO_NAMESPACE))
+ return False
+
+ # find prometheus pod by label selector
+ ret = corev1.list_namespaced_pod(ISTIO_NAMESPACE, label_selector=PROMETHEUS_LABELS)
+ for i in ret.items:
+ prom_pod_name = i.metadata.name
+ if prom_pod_name == None:
+ print("ERROR: prometheus pod not found")
+ return False
+
+ # check prometheus pod status
+ ret = corev1.read_namespaced_pod_status(prom_pod_name, ISTIO_NAMESPACE)
+ if ret.status.phase != "Running":
+ print("ERROR: prometheus pod %s is under %s state" % (prom_pod_name, ret.status.phase))
+ return False
+
+ return True
+
+
+def validateService():
+ m = Monitoring(PROMETHEUS_URL)
+
+ return m.is_targets_healthy()
+
+
+def main():
+ if validateDeploy() and validateService():
+ print"Prometheus monitoring validation has passed"
+ return True
+ else:
+ print"ERROR: Prometheus monitoring validation has failed"
+ return False
+
+
+if __name__ == '__main__':
+ main()
+
Code Review / clover.git / commitdiff