Support for different agent OS

JIRA: STORPERF-175

Change-Id: I64d13c77c18dd129a79dde3bf9a8bc996019bff5
Signed-off-by: Taseer Ahmed <taseer94@gmail.com>

diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py
index 6da2004..0634b8f 100644 (file)
--- a/docker/storperf-master/rest_server.py
+++ b/docker/storperf-master/rest_server.py
@@ -64,7 +64,9 @@ class ConfigurationRequestModel:
         'agent_image': fields.String,
         'public_network': fields.String,
         'volume_size': fields.Integer,
-        'availability_zone': fields.String
+        'availability_zone': fields.String,
+        'username': fields.String,
+        'password': fields.String
     }
 
 
@@ -137,6 +139,10 @@ class Configure(Resource):
                 storperf.volume_size = request.json['volume_size']
             if ('availability_zone' in request.json):
                 storperf.availabilty_zone = request.json['availability_zone']
+            if ('username' in request.json):
+                storperf.username = request.json['username']
+            if ('password' in request.json):
+                storperf.password = request.json['password']
 
             storperf.create_stack()
             if storperf.stack_id is None:

diff --git a/docker/storperf-master/storperf/db/test_results_db.py b/docker/storperf-master/storperf/db/test_results_db.py
index d6aabee..9c87e32 100644 (file)
--- a/docker/storperf-master/storperf/db/test_results_db.py
+++ b/docker/storperf-master/storperf/db/test_results_db.py
@@ -35,7 +35,6 @@ def push_results_to_db(db_url, details, logger):
             logger.debug(r.content)
         return json.loads(r.content)
     except Exception:
-        if logger:
-            logger.exception("Error [push_results_to_db('%s', '%s', '%s')]:" %
-                             (db_url, params, details['details']))
+        logger.exception("Error [push_results_to_db('%s', '%s')]:" %
+                         (db_url, params))
         return None

diff --git a/docker/storperf-master/storperf/fio/fio_invoker.py b/docker/storperf-master/storperf/fio/fio_invoker.py
index 106696d..0360ea2 100644 (file)
--- a/docker/storperf-master/storperf/fio/fio_invoker.py
+++ b/docker/storperf-master/storperf/fio/fio_invoker.py
@@ -15,13 +15,14 @@ import paramiko
 
 class FIOInvoker(object):
 
-    def __init__(self):
+    def __init__(self, var_dict={}):
         self.logger = logging.getLogger(__name__)
         self.event_listeners = set()
         self.event_callback_ids = set()
         self._remote_host = None
         self.callback_id = None
         self.terminated = False
+        self.metadata = var_dict
 
     @property
     def remote_host(self):
@@ -90,11 +91,7 @@ class FIOInvoker(object):
         self.logger.debug("Finished")
 
     def execute(self, args=[]):
-        ssh = paramiko.SSHClient()
-        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        ssh.connect(self.remote_host, username='storperf',
-                    key_filename='storperf/resources/ssh/storperf_rsa',
-                    timeout=2)
+        ssh = self._ssh_client()
 
         command = "sudo ./fio " + ' '.join(args)
         self.logger.debug("Remote command: %s" % command)
@@ -133,11 +130,7 @@ class FIOInvoker(object):
         self.logger.debug("Terminating fio on " + self.remote_host)
         self.terminated = True
 
-        ssh = paramiko.SSHClient()
-        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        ssh.connect(self.remote_host, username='storperf',
-                    key_filename='storperf/resources/ssh/storperf_rsa',
-                    timeout=2)
+        ssh = self._ssh_client()
 
         command = "sudo killall fio"
 
@@ -151,3 +144,17 @@ class FIOInvoker(object):
 
         stdout.close()
         stderr.close()
+
+    def _ssh_client(self):
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        if 'username' in self.metadata and 'password' in self.metadata:
+            ssh.connect(self.remote_host,
+                        username=self.metadata['username'],
+                        password=self.metadata['password'])
+            return ssh
+        else:
+            ssh.connect(self.remote_host, username='storperf',
+                        key_filename='storperf/resources/ssh/storperf_rsa',
+                        timeout=2)
+            return ssh

diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py
index 4e99e57..8a67048 100644 (file)
--- a/docker/storperf-master/storperf/storperf_master.py
+++ b/docker/storperf-master/storperf/storperf_master.py
@@ -257,6 +257,36 @@ class StorPerfMaster(object):
             'workloads',
             str(self._test_executor.workload_modules))
 
+    @property
+    def username(self):
+        return self.configuration_db.get_configuration_value(
+            'stack',
+            'username'
+        )
+
+    @username.setter
+    def username(self, value):
+        self.configuration_db.set_configuration_value(
+            'stack',
+            'username',
+            value
+        )
+
+    @property
+    def password(self):
+        return self.configuration_db.get_configuration_value(
+            'stack',
+            'password'
+        )
+
+    @password.setter
+    def password(self, value):
+        self.configuration_db.set_configuration_value(
+            'stack',
+            'password',
+            value
+        )
+
     def get_logs(self, lines=None):
         LOG_DIR = './storperf.log'
 
@@ -354,6 +384,9 @@ class StorPerfMaster(object):
         params['agent_count'] = self.agent_count
         params['public_network'] = self.public_network
         params['volume_size'] = self.volume_size
+        if self.username and self.password:
+            params['username'] = self.username
+            params['password'] = self.password
         job_id = self._test_executor.execute(params)
 
         return job_id
@@ -424,14 +457,50 @@ class StorPerfMaster(object):
 
         ssh = paramiko.SSHClient()
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        ssh.connect(slave, username='storperf',
-                    key_filename='storperf/resources/ssh/storperf_rsa',
-                    timeout=2)
+        if self.username and self.password:
+            ssh.connect(slave,
+                        username=self.username,
+                        password=self.password)
+        else:
+            ssh.connect(slave, username='storperf',
+                        key_filename='storperf/resources/ssh/storperf_rsa',
+                        timeout=2)
+
+        available = self._check_root_fs(ssh)
+        logger.debug("Available space on / is %s" % available)
+        if available < 65536:
+            logger.warn("Root filesystem is too small, attemping resize")
+            self._resize_root_fs(ssh, logger)
+
+            available = self._check_root_fs(ssh)
+            logger.debug("Available space on / is now %s" % available)
+            if available < 65536:
+                logger.error("Cannot create enough space on /")
+                raise Exception("Root filesystem has only %s free" %
+                                available)
 
         scp = SCPClient(ssh.get_transport())
         logger.debug("Transferring fio to %s" % slave)
         scp.put('/usr/local/bin/fio', '~/')
 
+    def _check_root_fs(self, ssh):
+        (_, stdout, _) = ssh.exec_command("df /")
+        stdout.readline()
+        lines = stdout.readline().split()
+        if len(lines) > 4:
+            available = lines[3]
+            return int(available)
+
+    def _resize_root_fs(self, ssh, logger):
+        command = "sudo /usr/sbin/resize2fs /dev/vda1"
+        logger.info("Attempting %s" % command)
+        (_, stdout, stderr) = ssh.exec_command(command)
+        stdout.channel.recv_exit_status()
+        for line in iter(stdout.readline, b''):
+            logger.info(line)
+        for line in iter(stderr.readline, b''):
+            logger.error(line)
+
     def _make_parameters(self):
         heat_parameters = {}
         heat_parameters['public_network'] = self.public_network

diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py
index 3d1d9f2..0e3fce0 100644 (file)
--- a/docker/storperf-master/storperf/test_executor.py
+++ b/docker/storperf-master/storperf/test_executor.py
@@ -340,7 +340,7 @@ class TestExecutor(object):
 
     def execute_on_node(self, workload):
 
-        invoker = FIOInvoker()
+        invoker = FIOInvoker(self.metadata)
         invoker.register(self.event)
         workload.invoker = invoker
 

diff --git a/docs/testing/user/test-usage.rst b/docs/testing/user/test-usage.rst
index 8cd7f20..6917e79 100644 (file)
--- a/docs/testing/user/test-usage.rst
+++ b/docs/testing/user/test-usage.rst
@@ -43,6 +43,8 @@ The following pieces of information are required to prepare the environment:
 - The name of the public network that agents will use
 - The size, in gigabytes, of the Cinder volumes to create
 - The availability zone (optional) in which the VM is to be launched. Defaults to **nova**.
+- The username (optional) if we specify a custom image.
+- The password (optional) for the above image.
 
 The ReST API is a POST to http://StorPerf:5000/api/v1.0/configurations and
 takes a JSON payload as follows.
@@ -55,7 +57,9 @@ takes a JSON payload as follows.
      "agent_image": string,
      "public_network": string,
      "volume_size": int,
-     "availability_zone": string
+     "availability_zone": string,
+     "username": string,
+     "password": string
    }
 
 This call will block until the stack is created, at which point it will return