Removes Configuration DB

Removes the configuration DB from storperf_master and replaces
it with direct calls to Heat.  Allows for container to
discover an existing stack if it already exists.

Change-Id: I32e767d5173dbfd58aa1d2127a478a1d36392592
JIRA: STORPERF-236
Signed-off-by: mbeierl <mark.beierl@dell.com>

diff --git a/docker/storperf-master/storperf/db/configuration_db.py b/docker/storperf-master/storperf/db/configuration_db.py
deleted file mode 100644 (file)
index 5b996c7..0000000
--- a/docker/storperf-master/storperf/db/configuration_db.py
+++ /dev/null
@@ -1,120 +0,0 @@
-##############################################################################
-# Copyright (c) 2015 EMC and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from sqlite3 import OperationalError
-from threading import Lock
-import logging
-import sqlite3
-
-db_mutex = Lock()
-
-
-class ConfigurationDB(object):
-
-    db_name = "StorPerfConfig.db"
-
-    def __init__(self):
-        """
-        Creates the StorPerfConfig.db and configuration tables on demand
-        """
-
-        self.logger = logging.getLogger(__name__)
-        self.logger.debug("Connecting to " + ConfigurationDB.db_name)
-        with db_mutex:
-            db = sqlite3.connect(ConfigurationDB.db_name)
-
-            cursor = db.cursor()
-            try:
-                cursor.execute('''CREATE TABLE configuration
-                (configuration_name text,
-                key text,
-                value text)''')
-                self.logger.debug("Created configuration table")
-            except OperationalError:
-                self.logger.debug("Configuration table exists")
-
-            cursor.execute('SELECT * FROM configuration')
-            db.commit()
-            db.close()
-
-    def delete_configuration_value(self, configuration_name, key):
-        """Deletes the value associated with the given key
-        """
-
-        with db_mutex:
-            db = sqlite3.connect(ConfigurationDB.db_name)
-            cursor = db.cursor()
-
-            cursor.execute("delete from configuration where "
-                           "configuration_name=? and key=?",
-                           (configuration_name, key))
-
-            self.logger.debug("Deleted " + configuration_name + ":" + key)
-
-            db.commit()
-            db.close()
-
-    def get_configuration_value(self, configuration_name, key):
-        """Returns a string representation of the value stored
-        with this key under the given configuration name.
-        """
-
-        with db_mutex:
-            db = sqlite3.connect(ConfigurationDB.db_name)
-            cursor = db.cursor()
-
-            cursor.execute(
-                """select value from configuration
-                           where configuration_name = ?
-                           and key = ?""",
-                (configuration_name, key,))
-
-            row = cursor.fetchone()
-
-            return_value = None
-
-            if (row is None):
-                self.logger.debug(
-                    configuration_name + ":" + key + " does not exist")
-            else:
-                self.logger.debug(
-                    configuration_name + ":" + key + " is " + str(row[0]))
-                return_value = str(row[0])
-
-            db.close()
-
-            return return_value
-
-    def set_configuration_value(self, configuration_name, key, value):
-        """Updates or creates the key under the given configuration
-        name so that it holds the value specified.
-        """
-
-        if (value is None):
-            return self.delete_configuration_value(configuration_name, key)
-
-        with db_mutex:
-            value = str(value)
-
-            db = sqlite3.connect(ConfigurationDB.db_name)
-            cursor = db.cursor()
-
-            cursor.execute("delete from configuration where "
-                           "configuration_name=? and key=?",
-                           (configuration_name, key))
-
-            cursor.execute(
-                """insert into configuration(configuration_name, key, value)
-                 values (?,?,?)""", (configuration_name, key, value))
-
-            self.logger.debug(
-                configuration_name + ":" + key + " set to " + value)
-
-            db.commit()
-            db.close()

diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py
index 45d5d89..da8a05e 100644 (file)
--- a/docker/storperf-master/storperf/storperf_master.py
+++ b/docker/storperf-master/storperf/storperf_master.py
@@ -20,10 +20,9 @@ from snaps.config.stack import StackConfig
 from snaps.openstack.create_stack import OpenStackHeatStack
 from snaps.openstack.os_credentials import OSCreds
 
-from storperf.db.configuration_db import ConfigurationDB
 from storperf.db.job_db import JobDB
 from storperf.test_executor import TestExecutor
-from snaps.openstack.utils import heat_utils
+from snaps.openstack.utils import heat_utils, cinder_utils, glance_utils
 
 
 class ParameterError(Exception):
@@ -35,11 +34,10 @@ class StorPerfMaster(object):
     def __init__(self):
         self.logger = logging.getLogger(__name__)
 
-        self.configuration_db = ConfigurationDB()
         self.job_db = JobDB()
 
         self.stack_settings = StackConfig(
-            name='StorPerfAgent',
+            name='StorPerfAgentGroup',
             template_path='storperf/resources/hot/agent-group.yaml')
 
         self.os_creds = OSCreds(username=os.environ.get('OS_USERNAME'),
@@ -49,129 +47,137 @@ class StorPerfMaster(object):
 
         self.heat_stack = OpenStackHeatStack(self.os_creds,
                                              self.stack_settings)
+        self.username = None
+        self.password = None
         self._test_executor = TestExecutor()
-        self._last_openstack_auth = datetime.now()
+        self._agent_count = 1
+        self._agent_image = "Ubuntu 14.04"
+        self._agent_flavor = "storperf"
+        self._availability_zone = None
+        self._public_network = None
+        self._volume_size = 1
+        self._cached_stack_id = None
+        self._last_snaps_check_time = None
 
     @property
     def volume_size(self):
-        value = self.configuration_db.get_configuration_value(
-            'stack',
-            'volume_size')
-        if (value is None):
-            self.volume_size = 1
-            value = 1
-        return int(value)
+        self._get_stack_info()
+        return self._volume_size
 
     @volume_size.setter
     def volume_size(self, value):
         if (self.stack_id is not None):
             raise ParameterError(
                 "ERROR: Cannot change volume size after stack is created")
-
-        self.configuration_db.set_configuration_value(
-            'stack',
-            'volume_size',
-            value)
+        self._volume_size = value
 
     @property
     def agent_count(self):
-        value = self.configuration_db.get_configuration_value(
-            'stack',
-            'agent_count')
-
-        if (value is None):
-            self.agent_count = 1
-            value = 1
-        return int(value)
+        self._get_stack_info()
+        return self._agent_count
 
     @agent_count.setter
     def agent_count(self, value):
         if (self.stack_id is not None):
             raise ParameterError(
                 "ERROR: Cannot change agent count after stack is created")
-
-        self.configuration_db.set_configuration_value(
-            'stack',
-            'agent_count',
-            value)
+        self._agent_count = value
 
     @property
     def agent_image(self):
-        value = self.configuration_db.get_configuration_value(
-            'stack',
-            'agent_image')
-
-        if (value is None):
-            value = 'Ubuntu 14.04'
-            self.agent_image = value
-
-        return value
+        self._get_stack_info()
+        return self._agent_image
 
     @agent_image.setter
     def agent_image(self, value):
         if (self.stack_id is not None):
             raise ParameterError(
                 "ERROR: Cannot change agent image after stack is created")
-
-        self.configuration_db.set_configuration_value(
-            'stack',
-            'agent_image',
-            value)
+        self._agent_image = value
 
     @property
     def public_network(self):
-        return self.configuration_db.get_configuration_value(
-            'stack',
-            'public_network')
+        self._get_stack_info()
+        return self._public_network
 
     @public_network.setter
     def public_network(self, value):
         if (self.stack_id is not None):
             raise ParameterError(
                 "ERROR: Cannot change public network after stack is created")
-
-        self.configuration_db.set_configuration_value(
-            'stack',
-            'public_network',
-            value)
+        self._public_network = value
 
     @property
     def agent_flavor(self):
-        return self.configuration_db.get_configuration_value(
-            'stack',
-            'agent_flavor')
+        self._get_stack_info()
+        return self._agent_flavor
 
     @agent_flavor.setter
     def agent_flavor(self, value):
         if (self.stack_id is not None):
             raise ParameterError(
                 "ERROR: Cannot change flavor after stack is created")
-
-        self.configuration_db.set_configuration_value(
-            'stack',
-            'agent_flavor',
-            value)
+        self._agent_flavor = value
 
     @property
     def stack_id(self):
+        self._get_stack_info()
+        return self._cached_stack_id
+
+    def _get_stack_info(self):
+        if self._last_snaps_check_time is not None:
+            time_since_check = datetime.now() - self._last_snaps_check_time
+            if time_since_check.total_seconds() < 30:
+                return self._cached_stack_id
+
         self.heat_stack.initialize()
         if self.heat_stack.get_stack() is not None:
-            return self.heat_stack.get_stack().id
+            self._last_snaps_check_time = datetime.now()
+            if self._cached_stack_id == self.heat_stack.get_stack().id:
+                return self._cached_stack_id
+            self._cached_stack_id = self.heat_stack.get_stack().id
+            cinder_cli = cinder_utils.cinder_client(self.os_creds)
+            glance_cli = glance_utils.glance_client(self.os_creds)
+
+            vm_inst_creators = self.heat_stack.get_vm_inst_creators()
+
+            self._agent_count = len(vm_inst_creators)
+            vm1 = vm_inst_creators[0]
+            self._availability_zone = \
+                vm1.instance_settings.availability_zone
+            self._agent_flavor = vm1.instance_settings.flavor.name
+
+            server = vm1.get_vm_inst()
+
+            image_id = server.image_id
+            image = glance_utils.get_image_by_id(glance_cli, image_id)
+            self._agent_image = image.name
+
+            volume_id = server.volume_ids[0]['id']
+            volume = cinder_utils.get_volume_by_id(
+                cinder_cli, volume_id)
+            self._volume_size = volume.size
+            router_creators = self.heat_stack.get_router_creators()
+            router1 = router_creators[0]
+
+            self._public_network = \
+                router1.router_settings.external_gateway
         else:
-            return None
+            self._cached_stack_id = None
+
+        return self._cached_stack_id
 
     @property
     def availability_zone(self):
-        return self.configuration_db.get_configuration_value(
-            'stack',
-            'availability_zone')
+        self._get_stack_info()
+        return self._availability_zone
 
     @availability_zone.setter
     def availability_zone(self, value):
-        self.configuration_db.set_configuration_value(
-            'stack',
-            'availability_zone',
-            value)
+        if (self.stack_id is not None):
+            raise ParameterError(
+                "ERROR: Cannot change zone after stack is created")
+        self._availability_zone = value
 
     @property
     def volume_quota(self):
@@ -225,49 +231,12 @@ class StorPerfMaster(object):
 
     @property
     def workloads(self):
-        return self.configuration_db.get_configuration_value(
-            'workload',
-            'workloads')
+        return str(self._test_executor.workload_modules)
 
     @workloads.setter
     def workloads(self, value):
         self._test_executor.register_workloads(value)
 
-        self.configuration_db.set_configuration_value(
-            'workload',
-            'workloads',
-            str(self._test_executor.workload_modules))
-
-    @property
-    def username(self):
-        return self.configuration_db.get_configuration_value(
-            'stack',
-            'username'
-        )
-
-    @username.setter
-    def username(self, value):
-        self.configuration_db.set_configuration_value(
-            'stack',
-            'username',
-            value
-        )
-
-    @property
-    def password(self):
-        return self.configuration_db.get_configuration_value(
-            'stack',
-            'password'
-        )
-
-    @password.setter
-    def password(self, value):
-        self.configuration_db.set_configuration_value(
-            'stack',
-            'password',
-            value
-        )
-
     def get_logs(self, lines=None):
         LOG_DIR = './storperf.log'
 

diff --git a/docker/storperf-master/tests/db_tests/configuration_db_test.py b/docker/storperf-master/tests/db_tests/configuration_db_test.py
deleted file mode 100644 (file)
index d8b021a..0000000
--- a/docker/storperf-master/tests/db_tests/configuration_db_test.py
+++ /dev/null
@@ -1,71 +0,0 @@
-##############################################################################
-# Copyright (c) 2015 EMC and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from storperf.db.configuration_db import ConfigurationDB
-import os
-import unittest
-
-
-class ConfigurationDBTest(unittest.TestCase):
-
-    def setUp(self):
-        ConfigurationDB.db_name = __name__ + '.db'
-        try:
-            os.remove(ConfigurationDB.db_name)
-        except OSError:
-            pass
-        self.config_db = ConfigurationDB()
-
-    def tearDown(self):
-        try:
-            os.remove(ConfigurationDB.db_name)
-        except OSError:
-            pass
-
-    def test_create_key(self):
-        expected = "ABCDE-12345"
-
-        self.config_db.set_configuration_value(
-            "test", "key", expected)
-
-        actual = self.config_db.get_configuration_value(
-            "test", "key")
-
-        self.assertEqual(
-            expected, actual, "Did not expect: " + str(actual))
-
-    def test_update_key(self):
-        expected = "ABCDE-12345"
-
-        self.config_db.set_configuration_value(
-            "test", "key", "initial_value")
-
-        self.config_db.set_configuration_value(
-            "test", "key", expected)
-
-        actual = self.config_db.get_configuration_value(
-            "test", "key")
-
-        self.assertEqual(
-            expected, actual, "Did not expect: " + str(actual))
-
-    def test_deleted_key(self):
-        expected = None
-
-        self.config_db.set_configuration_value(
-            "test", "key", "initial_value")
-
-        self.config_db.delete_configuration_value(
-            "test", "key")
-
-        actual = self.config_db.get_configuration_value(
-            "test", "key")
-
-        self.assertEqual(
-            expected, actual, "Did not expect: " + str(actual))

diff --git a/docker/storperf-master/tests/storperf_master_test.py b/docker/storperf-master/tests/storperf_master_test.py
index f328982..cd0a2f3 100644 (file)
--- a/docker/storperf-master/tests/storperf_master_test.py
+++ b/docker/storperf-master/tests/storperf_master_test.py
@@ -7,32 +7,16 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-import os
 import unittest
 
 import mock
 
-from storperf.db.configuration_db import ConfigurationDB
 from storperf.storperf_master import StorPerfMaster
 
 
-class MockStack(object):
-
-    def __init__(self):
-        pass
-
-    def get_stack(self):
-        return None
-
-
 class StorPerfMasterTest(unittest.TestCase):
 
     def setUp(self):
-        ConfigurationDB.db_name = __name__ + '.db'
-        try:
-            os.remove(ConfigurationDB.db_name)
-        except OSError:
-            pass
         with mock.patch("storperf.storperf_master.OSCreds"), \
                 mock.patch(
                     "storperf.storperf_master.OpenStackHeatStack") as oshs:
@@ -40,12 +24,6 @@ class StorPerfMasterTest(unittest.TestCase):
 
             self.storperf = StorPerfMaster()
 
-    def tearDown(self):
-        try:
-            os.remove(ConfigurationDB.db_name)
-        except OSError:
-            pass
-
     def test_agent_count(self):
         expected = 10