Change OpenStack clean behaviour

JIRA: FUNCTEST-236

The openstack snapshot generation is now triggered before running
a test case and removed from prepare_env

Change-Id: I4d1bc95dedd7f59d4b1d5866f288e1c1a70ec69e
Signed-off-by: jose.lausuch <jose.lausuch@ericsson.com>

diff --git a/ci/check_os.sh b/ci/check_os.sh
index c9e0b51..38fe32f 100755 (executable)
--- a/ci/check_os.sh
+++ b/ci/check_os.sh
@@ -26,7 +26,7 @@ fi
 echo "Checking OpenStack endpoints:"
 publicURL=$OS_AUTH_URL
 publicIP=$(echo $publicURL|sed 's/^.*http\:\/\///'|sed 's/.[^:]*$//')
 echo "Checking OpenStack endpoints:"
 publicURL=$OS_AUTH_URL
 publicIP=$(echo $publicURL|sed 's/^.*http\:\/\///'|sed 's/.[^:]*$//')

-publicPort=$(echo $publicURL|sed 's/^.*://'|sed 's/.[^\/]*$//')
+publicPort=$(echo $publicURL|sed 's/^.*://'|sed 's/\/.*$//')

 echo ">>Verifying connectivity to the public endpoint $publicIP:$publicPort..."
 verify_connectivity $publicIP $publicPort
 RETVAL=$?
 echo ">>Verifying connectivity to the public endpoint $publicIP:$publicPort..."
 verify_connectivity $publicIP $publicPort
 RETVAL=$?

diff --git a/ci/config_functest.yaml b/ci/config_functest.yaml
index dc0919b..ea502da 100644 (file)
--- a/ci/config_functest.yaml
+++ b/ci/config_functest.yaml
@@ -29,6 +29,8 @@ general:
         dir_rally_inst:     /home/opnfv/.rally
 
     openstack:
         dir_rally_inst:     /home/opnfv/.rally
 
     openstack:

+        snapshot_file: /home/opnfv/functest/conf/openstack_snapshot.yaml
+

         image_name: Cirros-0.3.4
         image_file_name:  cirros-0.3.4-x86_64-disk.img
         image_disk_format:  qcow2
         image_name: Cirros-0.3.4
         image_file_name:  cirros-0.3.4-x86_64-disk.img
         image_disk_format:  qcow2

diff --git a/ci/prepare_env.py b/ci/prepare_env.py
index 5754a62..4f29260 100644 (file)
--- a/ci/prepare_env.py
+++ b/ci/prepare_env.py
@@ -12,20 +12,19 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 

+

 import argparse
 import os
 import re
 import subprocess
 import sys
 import argparse
 import os
 import re
 import subprocess
 import sys

-import yaml

 
 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils
 
 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils

-import functest.utils.generate_defaults as gen_def

 import functest.utils.openstack_utils as os_utils
 import functest.utils.openstack_utils as os_utils

+import yaml

 
 
 
 

-""" arguments """

 actions = ['start', 'check']
 parser = argparse.ArgumentParser()
 parser.add_argument("action", help="Possible actions are: "
 actions = ['start', 'check']
 parser = argparse.ArgumentParser()
 parser.add_argument("action", help="Possible actions are: "


@@ -241,12 +240,6 @@ def install_rally():
                                         "OpenStack flavors."))
 
 
                                         "OpenStack flavors."))
 
 

-def generate_os_defaults():
-    print_separator()
-    logger.info("Generating OpenStack defaults...")
-    gen_def.main()
-
-

 def check_environment():
     msg_not_active = "The Functest environment is not installed."
     if not os.path.isfile(ENV_FILE):
 def check_environment():
     msg_not_active = "The Functest environment is not installed."
     if not os.path.isfile(ENV_FILE):


@@ -274,7 +267,6 @@ def main():
         source_rc_file()
         verify_deployment()
         install_rally()
         source_rc_file()
         verify_deployment()
         install_rally()

-        generate_os_defaults()

 
         with open(ENV_FILE, "w") as env_file:
             env_file.write("1")
 
         with open(ENV_FILE, "w") as env_file:
             env_file.write("1")

diff --git a/ci/run_tests.py b/ci/run_tests.py
index 5dba181..29b01d7 100644 (file)
--- a/ci/run_tests.py
+++ b/ci/run_tests.py
@@ -14,13 +14,13 @@ import re
 import sys
 
 import functest.ci.tier_builder as tb
 import sys
 
 import functest.ci.tier_builder as tb

-import functest.utils.clean_openstack as clean_os

 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils
 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils

+import functest.utils.openstack_clean as os_clean
+import functest.utils.openstack_snapshot as os_snapshot

 import functest.utils.openstack_utils as os_utils
 
 
 import functest.utils.openstack_utils as os_utils
 
 

-""" arguments """

 parser = argparse.ArgumentParser()
 parser.add_argument("-t", "--test", dest="test", action='store',
                     help="Test case or tier (group of tests) to be executed. "
 parser = argparse.ArgumentParser()
 parser.add_argument("-t", "--test", dest="test", action='store',
                     help="Test case or tier (group of tests) to be executed. "


@@ -61,8 +61,13 @@ def source_rc_file():
     os_utils.source_credentials(rc_file)
 
 
     os_utils.source_credentials(rc_file)
 
 

+def generate_os_snapshot():
+    logger.debug("Generating OpenStack snapshot...")
+    os_snapshot.main()
+
+

 def cleanup():
 def cleanup():

-    clean_os.main()
+    os_clean.main()

 
 
 def run_test(test):
 
 
 def run_test(test):


@@ -71,6 +76,9 @@ def run_test(test):
     logger.info("Running test case '%s'..." % test_name)
     print_separator("=")
     logger.debug("\n%s" % test)
     logger.info("Running test case '%s'..." % test_name)
     print_separator("=")
     logger.debug("\n%s" % test)

+
+    generate_os_snapshot()
+

     flags = (" -t %s" % (test_name))
     if REPORT_FLAG:
         flags += " -r"
     flags = (" -t %s" % (test_name))
     if REPORT_FLAG:
         flags += " -r"

diff --git a/cli/commands/cli_os.py b/cli/commands/cli_os.py
index 23b7bea..b007842 100644 (file)
--- a/cli/commands/cli_os.py
+++ b/cli/commands/cli_os.py
@@ -7,15 +7,15 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 

-import click
+

 import os
 import os

+import click
+import functest.utils.functest_utils as ft_utils
+import functest.utils.openstack_clean as os_clean
+import functest.utils.openstack_snapshot as os_snapshot

 import yaml
 
 import yaml
 

-import functest.utils.clean_openstack as clean_os
-import functest.utils.functest_utils as ft_utils
-import functest.utils.generate_defaults as gen_def

 
 

-""" global variables """

 with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
     functest_yaml = yaml.safe_load(f)
 
 with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
     functest_yaml = yaml.safe_load(f)
 


@@ -24,10 +24,12 @@ FUNCTEST_REPO = ("%s/functest/" % REPOS_DIR)
 FUNCTEST_CONF_DIR = functest_yaml.get("general").get(
     "directories").get("dir_functest_conf")
 RC_FILE = os.getenv('creds')
 FUNCTEST_CONF_DIR = functest_yaml.get("general").get(
     "directories").get("dir_functest_conf")
 RC_FILE = os.getenv('creds')

-OS_DEFAULTS_FILE = FUNCTEST_CONF_DIR + '/os_defaults.yaml'
+OS_SNAPSHOT_FILE = ft_utils.get_parameter_from_yaml(
+    "general.openstack.snapshot_file")

 
 
 class CliOpenStack:
 
 
 class CliOpenStack:

+

     def __init__(self):
         self.os_auth_url = os.getenv('OS_AUTH_URL')
         self.endpoint_ip = None
     def __init__(self):
         self.os_auth_url = os.getenv('OS_AUTH_URL')
         self.endpoint_ip = None


@@ -86,7 +88,7 @@ class CliOpenStack:
 
     def snapshot_create(self):
         self.ping_endpoint()
 
     def snapshot_create(self):
         self.ping_endpoint()

-        if os.path.isfile(OS_DEFAULTS_FILE):
+        if os.path.isfile(OS_SNAPSHOT_FILE):

             answer = raw_input("It seems there is already an OpenStack "
                                "snapshot. Do you want to overwrite it with "
                                "the current OpenStack status? [y|n]\n")
             answer = raw_input("It seems there is already an OpenStack "
                                "snapshot. Do you want to overwrite it with "
                                "the current OpenStack status? [y|n]\n")


@@ -99,22 +101,22 @@ class CliOpenStack:
                     answer = raw_input("Invalid answer. Please type [y|n]\n")
 
         click.echo("Generating Openstack snapshot...")
                     answer = raw_input("Invalid answer. Please type [y|n]\n")
 
         click.echo("Generating Openstack snapshot...")

-        gen_def.main()
+        os_snapshot.main()

 
     def snapshot_show(self):
 
     def snapshot_show(self):

-        if not os.path.isfile(OS_DEFAULTS_FILE):
+        if not os.path.isfile(OS_SNAPSHOT_FILE):

             click.echo("There is no OpenStack snapshot created. To create "
                        "one run the command 'functest env os-create-snapshot'")
             return
             click.echo("There is no OpenStack snapshot created. To create "
                        "one run the command 'functest env os-create-snapshot'")
             return

-        with open(OS_DEFAULTS_FILE, 'r') as yaml_file:
+        with open(OS_SNAPSHOT_FILE, 'r') as yaml_file:

             click.echo("\n%s"
                        % yaml_file.read())
 
     def clean(self):
         self.ping_endpoint()
             click.echo("\n%s"
                        % yaml_file.read())
 
     def clean(self):
         self.ping_endpoint()

-        if not os.path.isfile(OS_DEFAULTS_FILE):
+        if not os.path.isfile(OS_SNAPSHOT_FILE):

             click.echo("Not possible to clean OpenStack without a snapshot. "
                        "This could cause problems. "
                        "Run first the command 'os-create-shapshot'.")
             return
             click.echo("Not possible to clean OpenStack without a snapshot. "
                        "This could cause problems. "
                        "Run first the command 'os-create-shapshot'.")
             return

-        clean_os.main()
+        os_clean.main()

diff --git a/docs/configguide/configguide.rst b/docs/configguide/configguide.rst
index 7c8ce8b..9f66939 100644 (file)
--- a/docs/configguide/configguide.rst
+++ b/docs/configguide/configguide.rst
@@ -327,10 +327,10 @@ The structure under the **functest** repository can be described as follows::
     |   `-- vIMS
     `-- utils
         |-- __init__.py
     |   `-- vIMS
     `-- utils
         |-- __init__.py

-        |-- clean_openstack.py

         |-- functest_logger.py
         |-- functest_utils.py
         |-- functest_logger.py
         |-- functest_utils.py

-        |-- generate_defaults.py
+        |-- openstack_clean.py
+        |-- openstack_snapshot.py

         `-- openstack_utils.py
 
     (Note: All *.pyc files removed from above list for brevity...)
         `-- openstack_utils.py
 
     (Note: All *.pyc files removed from above list for brevity...)

diff --git a/utils/functest_utils.py b/utils/functest_utils.py
index 2e7a122..285f887 100644 (file)
--- a/utils/functest_utils.py
+++ b/utils/functest_utils.py
@@ -8,23 +8,25 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 

+""" global variables """
+

 from datetime import datetime as dt
 import json
 import os
 import os.path
 import re
 from datetime import datetime as dt
 import json
 import os
 import os.path
 import re

-import requests

 import shutil
 import socket
 import subprocess
 import sys
 import urllib2
 import shutil
 import socket
 import subprocess
 import sys
 import urllib2

-import yaml
-from git import Repo
+

 import functest.ci.tier_builder as tb
 import functest.ci.tier_builder as tb

+from git import Repo
+import requests
+import yaml

 
 
 
 

-""" global variables """

 REPOS_DIR = os.getenv('repos_dir')
 FUNCTEST_REPO = ("%s/functest/" % REPOS_DIR)
 
 REPOS_DIR = os.getenv('repos_dir')
 FUNCTEST_REPO = ("%s/functest/" % REPOS_DIR)
 


@@ -300,3 +302,26 @@ def get_criteria_by_test(testname):
                 criteria = test.get_criteria()
 
     return criteria
                 criteria = test.get_criteria()
 
     return criteria

+
+
+# ----------------------------------------------------------
+#
+#               YAML UTILS
+#
+# -----------------------------------------------------------
+def get_parameter_from_yaml(parameter):
+    """
+    Returns the value of a given parameter in config_functest.yaml
+    parameter must be given in string format with dots
+    Example: general.openstack.image_name
+    """
+    with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
+        functest_yaml = yaml.safe_load(f)
+    f.close()
+    value = functest_yaml
+    for element in parameter.split("."):
+        value = value.get(element)
+        if value is None:
+            raise ValueError("The parameter %s is not defined in"
+                             " config_functest.yaml" % parameter)
+    return value

diff --git a/utils/clean_openstack.py b/utils/openstack_clean.py
similarity index 94%
rename from utils/clean_openstack.py
rename to utils/openstack_clean.py
index 70eced6..3eb39a2 100644 (file)
--- a/utils/clean_openstack.py
+++ b/utils/openstack_clean.py
@@ -20,24 +20,24 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 

-import os
+""" logging configuration """
+

 import time
 import time

-import yaml

 
 

-from novaclient import client as novaclient
-from neutronclient.v2_0 import client as neutronclient
-from keystoneclient.v2_0 import client as keystoneclient

 from cinderclient import client as cinderclient
 from cinderclient import client as cinderclient

-

 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_logger as ft_logger

+import functest.utils.functest_utils as ft_utils

 import functest.utils.openstack_utils as os_utils
 import functest.utils.openstack_utils as os_utils

+from keystoneclient.v2_0 import client as keystoneclient
+from neutronclient.v2_0 import client as neutronclient
+from novaclient import client as novaclient
+import yaml

 
 
 
 

-""" logging configuration """
-logger = ft_logger.Logger("clean_openstack").getLogger()
+logger = ft_logger.Logger("openstack_clean").getLogger()

 
 

-REPO_PATH = os.environ['repos_dir'] + '/functest/'
-DEFAULTS_FILE = '/home/opnfv/functest/conf/os_defaults.yaml'
+OS_SNAPSHOT_FILE = ft_utils.get_parameter_from_yaml(
+    "general.openstack.snapshot_file")

 
 
 def separator():
 
 
 def separator():


@@ -365,22 +365,22 @@ def main():
     logger.info("+++++++++++++++++++++++++++++++")
 
     try:
     logger.info("+++++++++++++++++++++++++++++++")
 
     try:

-        with open(DEFAULTS_FILE) as f:
-            defaults_yaml = yaml.safe_load(f)
+        with open(OS_SNAPSHOT_FILE) as f:
+            snapshot_yaml = yaml.safe_load(f)

     except Exception:
         logger.info("The file %s does not exist. The OpenStack snapshot must"
     except Exception:
         logger.info("The file %s does not exist. The OpenStack snapshot must"

-                    " be created first. Aborting cleanup." % DEFAULTS_FILE)
+                    " be created first. Aborting cleanup." % OS_SNAPSHOT_FILE)

         exit(0)
 
         exit(0)
 

-    default_images = defaults_yaml.get('images')
-    default_instances = defaults_yaml.get('instances')
-    default_volumes = defaults_yaml.get('volumes')
-    default_networks = defaults_yaml.get('networks')
-    default_routers = defaults_yaml.get('routers')
-    default_security_groups = defaults_yaml.get('secgroups')
-    default_floatingips = defaults_yaml.get('floatingips')
-    default_users = defaults_yaml.get('users')
-    default_tenants = defaults_yaml.get('tenants')
+    default_images = snapshot_yaml.get('images')
+    default_instances = snapshot_yaml.get('instances')
+    default_volumes = snapshot_yaml.get('volumes')
+    default_networks = snapshot_yaml.get('networks')
+    default_routers = snapshot_yaml.get('routers')
+    default_security_groups = snapshot_yaml.get('secgroups')
+    default_floatingips = snapshot_yaml.get('floatingips')
+    default_users = snapshot_yaml.get('users')
+    default_tenants = snapshot_yaml.get('tenants')

 
     creds_nova = os_utils.get_credentials("nova")
     nova_client = novaclient.Client('2', **creds_nova)
 
     creds_nova = os_utils.get_credentials("nova")
     nova_client = novaclient.Client('2', **creds_nova)

diff --git a/utils/generate_defaults.py b/utils/openstack_snapshot.py
similarity index 84%
rename from utils/generate_defaults.py
rename to utils/openstack_snapshot.py
index 35b9ed3..704ef50 100644 (file)
--- a/utils/generate_defaults.py
+++ b/utils/openstack_snapshot.py
@@ -20,19 +20,21 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 

+""" logging configuration """
+

 import os
 import os

-import yaml

 
 

-from novaclient import client as novaclient
-from neutronclient.v2_0 import client as neutronclient
-from keystoneclient.v2_0 import client as keystoneclient

 from cinderclient import client as cinderclient
 from cinderclient import client as cinderclient

-
-import functest.utils.openstack_utils as os_utils

 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_logger as ft_logger

+import functest.utils.functest_utils as ft_utils
+import functest.utils.openstack_utils as os_utils
+from keystoneclient.v2_0 import client as keystoneclient
+from neutronclient.v2_0 import client as neutronclient
+from novaclient import client as novaclient
+import yaml

 
 

-""" logging configuration """
-logger = ft_logger.Logger("generate_defaults").getLogger()
+
+logger = ft_logger.Logger("openstack_snapshot").getLogger()

 
 REPO_PATH = os.environ['repos_dir'] + '/functest/'
 if not os.path.exists(REPO_PATH):
 
 REPO_PATH = os.environ['repos_dir'] + '/functest/'
 if not os.path.exists(REPO_PATH):


@@ -40,7 +42,8 @@ if not os.path.exists(REPO_PATH):
     exit(-1)
 
 
     exit(-1)
 
 

-DEFAULTS_FILE = '/home/opnfv/functest/conf/os_defaults.yaml'
+OS_SNAPSHOT_FILE = ft_utils.get_parameter_from_yaml(
+    "general.openstack.snapshot_file")

 
 
 def separator():
 
 
 def separator():


@@ -129,7 +132,7 @@ def get_users(keystone_client):
 
 
 def get_tenants(keystone_client):
 
 
 def get_tenants(keystone_client):

-    logger.debug("Getting users...")
+    logger.debug("Getting tenants...")

     dic_tenants = {}
     tenants = os_utils.get_tenants(keystone_client)
     if not (tenants is None or len(tenants) == 0):
     dic_tenants = {}
     tenants = os_utils.get_tenants(keystone_client)
     if not (tenants is None or len(tenants) == 0):


@@ -161,24 +164,24 @@ def main():
                      "script again.")
         exit(-1)
 
                      "script again.")
         exit(-1)
 

-    defaults = {}
-    defaults.update(get_instances(nova_client))
-    defaults.update(get_images(nova_client))
-    defaults.update(get_volumes(cinder_client))
-    defaults.update(get_networks(neutron_client))
-    defaults.update(get_routers(neutron_client))
-    defaults.update(get_security_groups(neutron_client))
-    defaults.update(get_floatinips(nova_client))
-    defaults.update(get_users(keystone_client))
-    defaults.update(get_tenants(keystone_client))
-
-    with open(DEFAULTS_FILE, 'w+') as yaml_file:
-        yaml_file.write(yaml.safe_dump(defaults, default_flow_style=False))
+    snapshot = {}
+    snapshot.update(get_instances(nova_client))
+    snapshot.update(get_images(nova_client))
+    snapshot.update(get_volumes(cinder_client))
+    snapshot.update(get_networks(neutron_client))
+    snapshot.update(get_routers(neutron_client))
+    snapshot.update(get_security_groups(neutron_client))
+    snapshot.update(get_floatinips(nova_client))
+    snapshot.update(get_users(keystone_client))
+    snapshot.update(get_tenants(keystone_client))
+
+    with open(OS_SNAPSHOT_FILE, 'w+') as yaml_file:
+        yaml_file.write(yaml.safe_dump(snapshot, default_flow_style=False))

         yaml_file.seek(0)
         yaml_file.seek(0)

-        logger.info("Openstack Defaults found in the deployment:\n%s"
-                    % yaml_file.read())
+        logger.debug("Openstack Snapshot found in the deployment:\n%s"
+                     % yaml_file.read())

         logger.debug("NOTE: These objects will NOT be deleted after " +
         logger.debug("NOTE: These objects will NOT be deleted after " +

-                     "running the tests.")
+                     "running the test.")

 
 
 if __name__ == '__main__':
 
 
 if __name__ == '__main__':