Add Stats report and Swagger UI

    Add Swagger web ui at /swagger
    Add ability to fetch read/write latency status via ReST ui
    Can now delete where stack was removed from OpenStack but not from the
    storperf DB
    Change to use Floating IPs instead of private IP
    Fix delete bug where there was no dependency on resources in
    the resource group.

    JIRA: STORPERF-19
    JIRA: STORPERF-20

Change-Id: I1d9627d81f3c309b178a9b68cc306a4101c1a231
Signed-off-by: Mark Beierl <mark.beierl@emc.com>

diff --git a/ci/setup.py b/ci/setup.py
index 293fdda..2a02276 100755 (executable)
--- a/ci/setup.py
+++ b/ci/setup.py
@@ -23,6 +23,7 @@ setup(
     url="https://www.opnfv.org",
     install_requires=["flask==0.10",
                       "flask-restful==0.3.5",
+                      "flask-restful-swagger==0.19",
                       "html2text==2016.1.8",
                       "python-cinderclient==1.6.0",
                       "python-glanceclient==1.1.0",

diff --git a/cli.py b/cli.py
index fad275c..85ebcfd 100644 (file)
--- a/cli.py
+++ b/cli.py
@@ -154,7 +154,14 @@ def main(argv=None):
                 raise Usage(content['message'])
 
         if (report is not None):
-            print storperf.fetch_results(report)
+            print "Fetching report for %s..." % (report,)
+            response = requests.get(
+                'http://127.0.0.1:5000/api/v1.0/job?id=%s' % (report,))
+            if (response.status_code == 400):
+                content = json.loads(response.content)
+                raise Usage(content['message'])
+            content = json.loads(response.content)
+            print content
         else:
             print "Calling start..."
             response = requests.post(

diff --git a/docker/Dockerfile b/docker/Dockerfile
index fab768f..fb25bfc 100644 (file)
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -40,6 +40,7 @@ wget \
 puppet \
 build-essential \
 python-dev \
+python-matplotlib \
 python-pip \
 --no-install-recommends
 
@@ -63,9 +64,17 @@ RUN chmod 700 /root/.ssh
 RUN git config --global http.sslVerify false
 RUN git clone https://gerrit.opnfv.org/gerrit/storperf ${repos_dir}/storperf
 RUN git clone https://gerrit.opnfv.org/gerrit/releng ${repos_dir}/releng
+
+# Third party git fetches
+RUN git clone https://github.com/swagger-api/swagger-ui.git ${repos_dir}/swagger-ui
+RUN mkdir -p ${repos_dir}/storperf/storperf/resources/html/swagger
+RUN cp -r ${repos_dir}/swagger-ui/dist/* ${repos_dir}/storperf/storperf/resources/html/swagger
+RUN sed -i 's|url = "http://petstore.swagger.io/v2/swagger.json";|url = window.location.protocol+"//"+window.location.host+"/api/spec.json";|' ${repos_dir}/storperf/storperf/resources/html/swagger/index.html
+
 RUN git clone http://git.kernel.dk/fio.git ${repos_dir}/fio
 RUN cd ${repos_dir}/fio && git checkout tags/fio-2.2.10
 RUN cd ${repos_dir}/fio && make -j 6 install
+
 RUN puppet module install gdsoperations-graphite
 
 RUN chmod 600 ${repos_dir}/storperf/storperf/resources/ssh/storperf_rsa

diff --git a/docker/requirements.pip b/docker/requirements.pip
index 89f19ae..4c9aaae 100644 (file)
--- a/docker/requirements.pip
+++ b/docker/requirements.pip
@@ -9,4 +9,4 @@ flask==0.10
 flask-restful==0.3.5
 flask-restful-swagger==0.19
 flask-swagger==0.2.12
-html2text==2016.1.8
\ No newline at end of file
+html2text==2016.1.8

diff --git a/rest_server.py b/rest_server.py
index c5fe99b..f0a817b 100644 (file)
--- a/rest_server.py
+++ b/rest_server.py
@@ -25,15 +25,40 @@ storperf = StorPerfMaster()
 
 @app.route('/swagger/<path:path>')
 def send_swagger(path):
-    print "called! storperf/resources/html/swagger/" + path
     return send_from_directory('storperf/resources/html/swagger', path)
 
 
+@swagger.model
+class ConfigurationRequestModel:
+    resource_fields = {
+        'agent_count': fields.Integer,
+        'public_network': fields.String,
+        'volume_size': fields.Integer
+    }
+
+
+@swagger.model
+class ConfigurationResponseModel:
+    resource_fields = {
+        'agent_count': fields.Integer,
+        'public_network': fields.String,
+        'stack_created': fields.Boolean,
+        'stack_id': fields.String,
+        'volume_size': fields.Integer
+    }
+
+
 class Configure(Resource):
 
+    """Configuration API"""
+
     def __init__(self):
         self.logger = logging.getLogger(__name__)
 
+    @swagger.operation(
+        notes='Fetch the current agent configuration',
+        type=ConfigurationResponseModel.__name__
+    )
     def get(self):
         return jsonify({'agent_count': storperf.agent_count,
                         'public_network': storperf.public_network,
@@ -41,6 +66,23 @@ class Configure(Resource):
                         'stack_created': storperf.is_stack_created,
                         'stack_id': storperf.stack_id})
 
+    @swagger.operation(
+        notes='''Set the current agent configuration and create a stack in
+        the controller.  Returns once the stack request is submitted.''',
+        parameters=[
+            {
+                "name": "configuration",
+                "description": '''Configuration to be set. All parameters are
+                optional, and will retain their previous value if not
+                specified.  Volume size is in GB.
+                ''',
+                "required": True,
+                "type": "ConfigurationRequestModel",
+                "paramType": "body"
+            }
+        ],
+        type=ConfigurationResponseModel.__name__
+    )
     def post(self):
         if not request.json:
             abort(400, "ERROR: No data specified")
@@ -64,6 +106,9 @@ class Configure(Resource):
         except Exception as e:
             abort(400, str(e))
 
+    @swagger.operation(
+        notes='Deletes the agent configuration and the stack'
+    )
     def delete(self):
         try:
             storperf.delete_stack()
@@ -81,8 +126,17 @@ class WorkloadModel:
     }
 
 
+@swagger.model
+class WorkloadResponseModel:
+    resource_fields = {
+        'job_id': fields.String
+    }
+
+
 class Job(Resource):
 
+    """Job API"""
+
     def __init__(self):
         self.logger = logging.getLogger(__name__)
 
@@ -131,10 +185,11 @@ class Job(Resource):
                 "paramType": "body"
             }
         ],
+        type=WorkloadResponseModel.__name__,
         responseMessages=[
             {
                 "code": 200,
-                "message": "Wordload ID found, response in JSON format"
+                "message": "Job submitted"
             },
             {
                 "code": 400,
@@ -181,16 +236,31 @@ class Job(Resource):
     )
     def delete(self):
         try:
-            storperf.terminate_workloads()
-            return True
+            return jsonify({'Slaves': storperf.terminate_workloads()})
         except Exception as e:
             abort(400, str(e))
 
 
+@swagger.model
+class QuotaModel:
+
+    resource_fields = {
+        'quota': fields.Integer
+    }
+
+
 class Quota(Resource):
+    """Quota API"""
 
+    @swagger.operation(
+        notes='''Fetch the current Cinder volume quota.  This value limits
+        the number of volumes that can be created, and by extension, defines
+        the maximum number of agents that can be created for any given test
+        scenario''',
+        type=QuotaModel.__name__
+    )
     def get(self):
-        quota = storperf.get_volume_quota()
+        quota = storperf.volume_quota
         return jsonify({'quota': quota})
 
 

diff --git a/storperf/db/graphite_db.py b/storperf/db/graphite_db.py
index c62340c..8fef071 100644 (file)
--- a/storperf/db/graphite_db.py
+++ b/storperf/db/graphite_db.py
@@ -1,6 +1,8 @@
 from storperf.db.job_db import JobDB
+import calendar
 import json
 import logging
+import time
 
 import requests
 
@@ -41,18 +43,54 @@ class GraphiteDB(object):
         for io_type in ['read', 'write']:
             for workload_name, times in workload_names.iteritems():
                 workload_pattern = self.make_fullname_pattern(workload_name)
+                short_name = '.'.join(workload_name.split('.')[1:6])
+                start = times[0]
+                end = times[1]
+
+                if end is None:
+                    end = str(calendar.timegm(time.gmtime()))
+                averages[short_name + ".duration"] = \
+                    (int(end) - int(start))
+
+                key = short_name + "." + io_type
+
                 request = ("http://127.0.0.1:8000/render/?target="
                            "averageSeries(%s.jobs.1.%s.lat.mean)"
                            "&format=json"
                            "&from=%s"
                            "&until=%s" %
-                           (workload_pattern, io_type, times[0], times[1]))
+                           (workload_pattern, io_type, start, end))
+                self.logger.debug("Calling %s" % (request))
+
+                response = requests.get(request)
+                if (response.status_code == 200):
+                    averages[key + ".latency"] = \
+                        self._average_results(json.loads(response.content))
+
+                request = ("http://127.0.0.1:8000/render/?target="
+                           "averageSeries(%s.jobs.1.%s.bw)"
+                           "&format=json"
+                           "&from=%s"
+                           "&until=%s" %
+                           (workload_pattern, io_type, start, end))
+                self.logger.debug("Calling %s" % (request))
+
+                response = requests.get(request)
+                if (response.status_code == 200):
+                    averages[key + ".throughput"] = \
+                        self._average_results(json.loads(response.content))
+
+                request = ("http://127.0.0.1:8000/render/?target="
+                           "averageSeries(%s.jobs.1.%s.iops)"
+                           "&format=json"
+                           "&from=%s"
+                           "&until=%s" %
+                           (workload_pattern, io_type, start, end))
                 self.logger.debug("Calling %s" % (request))
 
                 response = requests.get(request)
                 if (response.status_code == 200):
-                    short_name = '.'.join(workload_name.split('.')[1:6])
-                    averages[short_name + "." + io_type] = \
+                    averages[key + ".iops"] = \
                         self._average_results(json.loads(response.content))
 
         return averages

diff --git a/storperf/test_executor.py b/storperf/test_executor.py
index c0ea295..309fbcb 100644 (file)
--- a/storperf/test_executor.py
+++ b/storperf/test_executor.py
@@ -122,8 +122,11 @@ class TestExecutor(object):
 
     def terminate(self):
         self._terminated = True
+        terminated_hosts = []
         for workload in self._workload_executors:
             workload.terminate()
+            terminated_hosts.append(workload.remote_host)
+        return terminated_hosts
 
     def execute_workloads(self):
         self._terminated = False