"focal_point_type": True,
"link_types": True,
"environment": True,
- "name": True
+ "name": True,
+ "distribution": True,
+ "distribution_version": True,
+ "mechanism_drivers": True,
+ "type_drivers": True,
+ "use_implicit_links": True
}
RESERVED_NAMES = ["ANY"]
+ def __init__(self):
+ super().__init__()
+ self.focal_point_types = self.get_constants_by_name("object_types")
+ self.link_types = self.get_constants_by_name("link_types")
+ self.mechanism_drivers = self.get_constants_by_name("mechanism_drivers")
+ self.type_drivers = self.get_constants_by_name("type_drivers")
+
def on_get(self, req, resp):
self.log.debug("Getting clique types")
filters = self.parse_query_params(req)
- focal_point_types = self.get_constants_by_name("object_types")
- link_types = self.get_constants_by_name("link_types")
filters_requirements = {
- 'env_name': self.require(str, mandatory=True),
+ 'env_name': self.require(str),
'id': self.require(ObjectId, convert_to_type=True),
+ 'distribution': self.require(str),
+ 'distribution_version': self.require(str),
+ 'mechanism_drivers': self.require(str,
+ validate=DataValidate.LIST,
+ requirement=self.mechanism_drivers),
+ 'type_drivers': self.require(str,
+ validate=DataValidate.LIST,
+ requirement=self.type_drivers),
'focal_point_type': self.require(str,
validate=DataValidate.LIST,
- requirement=focal_point_types),
+ requirement=self.focal_point_types),
'link_type': self.require([list, str],
validate=DataValidate.LIST,
- requirement=link_types),
+ requirement=self.link_types),
'name': self.require(str),
'page': self.require(int, convert_to_type=True),
'page_size': self.require(int, convert_to_type=True)
}
self.validate_query_data(filters, filters_requirements)
+ if 'distribution_version' in filters and 'distribution' not in filters:
+ self.bad_request("Distribution version without distribution "
+ "is not allowed")
+
page, page_size = self.get_pagination(filters)
query = self.build_query(filters)
if self.ID in query:
error, clique_type = self.get_content_from_request(req)
if error:
self.bad_request(error)
- focal_point_types = self.get_constants_by_name("object_types")
- link_types = self.get_constants_by_name("link_types")
+
clique_type_requirements = {
- 'environment': self.require(str, mandatory=True),
+ 'environment': self.require(str),
'focal_point_type': self.require(str,
mandatory=True,
validate=DataValidate.LIST,
- requirement=focal_point_types),
+ requirement=self.focal_point_types),
'link_types': self.require(list,
mandatory=True,
validate=DataValidate.LIST,
- requirement=link_types),
- 'name': self.require(str, mandatory=True)
+ requirement=self.link_types),
+ 'name': self.require(str, mandatory=True),
+ 'distribution': self.require(str),
+ 'distribution_version': self.require(str),
+ 'mechanism_drivers': self.require(str,
+ validate=DataValidate.LIST,
+ requirement=self.mechanism_drivers),
+ 'type_drivers': self.require(str,
+ validate=DataValidate.LIST,
+ requirement=self.type_drivers),
+ 'use_implicit_links': self.require(bool)
}
self.validate_query_data(clique_type, clique_type_requirements)
-
- env_name = clique_type['environment']
- if not self.check_environment_name(env_name):
- self.bad_request("Unknown environment: {}".format(env_name))
- elif env_name.upper() in self.RESERVED_NAMES:
- self.bad_request("Environment name '{}' is reserved".format(env_name))
+ self.validate_required_fields(clique_type)
+ self.validate_focal_point_type(clique_type)
+ self.validate_duplicate_configuration(clique_type)
self.write(clique_type, self.COLLECTION)
self.set_successful_response(resp,
- {"message": "created a new clique_type "
- "for environment {0}"
- .format(env_name)},
+ {"message": "created a new clique_type"},
"201")
def build_query(self, filters):
query = {}
- filters_keys = ['name', 'focal_point_type']
+ filters_keys = ['name', 'focal_point_type',
+ 'distribution', 'distribution_version',
+ 'mechanism_drivers', 'type_drivers']
self.update_query_with_filters(filters, filters_keys, query)
+
link_types = filters.get('link_type')
if link_types:
if type(link_types) != list:
if _id:
query[self.ID] = _id
- query['environment'] = filters['env_name']
+ env_name = filters.get('env_name')
+ if env_name:
+ query['environment'] = filters['env_name']
return query
+
+ def validate_required_fields(self, clique_type):
+ env_name = clique_type.get('environment')
+ distribution = clique_type.get('distribution')
+ distribution_version = clique_type.get('distribution_version')
+ if distribution_version and not distribution:
+ self.bad_request("Distribution version without distribution "
+ "is not allowed")
+
+ configuration_specified = ((distribution and distribution_version)
+ or clique_type.get('mechanism_drivers')
+ or clique_type.get('type_drivers'))
+ if env_name:
+ if configuration_specified:
+ self.bad_request("Either environment or configuration "
+ "should be specified (not both).")
+
+ if not self.check_environment_name(env_name):
+ self.bad_request("Unknown environment: {}".format(env_name))
+ elif env_name.upper() in self.RESERVED_NAMES:
+ self.bad_request(
+ "Environment name '{}' is reserved".format(env_name))
+ elif not configuration_specified:
+ self.bad_request("Either environment or configuration "
+ "should be specified.")
+
+ def validate_focal_point_type(self, clique_type):
+ focal_point_type = clique_type['focal_point_type']
+ environment = clique_type.get('environment')
+ if environment:
+ env_match = self.read(
+ matches={"environment": environment,
+ "focal_point_type": focal_point_type},
+ collection="clique_types"
+ )
+ if env_match:
+ self.bad_request("Clique type with focal point {} "
+ "is already registered for environment {}"
+ .format(focal_point_type, environment))
+ else:
+ pass
+
+ def validate_duplicate_configuration(self, clique_type):
+ if clique_type.get('environment'):
+ return
+
+ search = {'focal_point_type': clique_type['focal_point_type']}
+ for field in ['distribution', 'mechanism_drivers', 'type_drivers']:
+ value = clique_type.get(field)
+ if value:
+ search[field] = value
+ if field == 'distribution':
+ dv = clique_type.get('distribution_version')
+ if dv:
+ search['distribution_version'] = dv
+ # Got a match with higher score, no need to look further
+ break
+
+ env_match = self.read(matches=search,
+ collection="clique_types")
+ if env_match:
+ self.bad_request("Clique type with configuration '{}' "
+ "is already registered"
+ .format(search))
from bson.objectid import ObjectId
from datetime import datetime
from utils.constants import EnvironmentFeatures
-from utils.inventory_mgr import InventoryMgr
class EnvironmentConfigs(ResponderBase):
"distribution": True
}
CONFIGURATIONS_NAMES = ["mysql", "OpenStack", "CLI", "AMQP",
- "Monitoring", "NFV_provider", "ACI"]
- OPTIONAL_CONFIGURATIONS_NAMES = ["AMQP", "Monitoring",
- "NFV_provider", "ACI"]
+ "Monitoring", "NFV_provider", "ACI",
+ "Kubernetes", "VMware", "Bare-metal"]
+ REQUIRED_CONFIGURATIONS_NAMES = {
+ "OpenStack": ["OpenStack", "mysql", "CLI"],
+ "Kubernetes": ["Kubernetes", "CLI"],
+ }
+ DEFAULT_ENV_TYPE = "OpenStack"
def __init__(self):
super().__init__()
get_constants_by_name("environment_operational_status")
self.type_drivers = self.\
get_constants_by_name("type_drivers")
+ self.environment_types = self.\
+ get_constants_by_name("environment_types")
self.CONFIGURATIONS_REQUIREMENTS = {
"mysql": {
},
"Monitoring": {
"name": self.require(str, mandatory=True),
+ "install_monitoring_client": self.require(bool),
"config_folder": self.require(str,
mandatory=True,
validate=DataValidate.REGEX,
requirement=[regex.IP, regex.HOSTNAME]),
"user": self.require(str, mandatory=True),
"pwd": self.require(str, mandatory=True)
+ },
+ "Kubernetes": {
+ "name": self.require(str, mandatory=True),
+ "host": self.require(str,
+ mandatory=True,
+ validate=DataValidate.REGEX,
+ requirement=[regex.IP, regex.HOSTNAME]),
+ "port": self.require(int,
+ mandatory=True,
+ convert_to_type=True,
+ validate=DataValidate.REGEX,
+ requirement=regex.PORT),
+ "user": self.require(str, mandatory=True),
+ "token": self.require(str, mandatory=True)
}
}
self.AUTH_REQUIREMENTS = {
"operational": self.require(str,
validate=DataValidate.LIST,
requirement=self.operational_values),
+ "environment_type": self.require(str,
+ validate=DataValidate.LIST,
+ requirement=self.environment_types),
"page": self.require(int, convert_to_type=True),
"page_size": self.require(int, convert_to_type=True)
}
query = {}
filters_keys = ["name", "distribution", "distribution_version",
"type_drivers", "user", "listen",
- "monitoring_setup_done", "scanned", "operational"]
+ "monitoring_setup_done", "scanned", "operational",
+ "environment_type"]
self.update_query_with_filters(filters, filters_keys, query)
mechanism_drivers = filters.get("mechanism_drivers")
if mechanism_drivers:
"enable_monitoring": self.require(bool, convert_to_type=True),
"monitoring_setup_done": self.require(bool, convert_to_type=True),
"auth": self.require(dict),
- "aci_enabled": self.require(bool, convert_to_type=True)
+ "aci_enabled": self.require(bool, convert_to_type=True),
+ "environment_type": self.require(str,
+ validate=DataValidate.LIST,
+ requirement=self.environment_types),
}
self.validate_query_data(env_config,
environment_config_requirement,
- can_be_empty_keys=["last_scanned"]
- )
+ can_be_empty_keys=["last_scanned",
+ "environment_type"])
self.check_and_convert_datetime("last_scanned", env_config)
+
# validate the configurations
+ environment_type = env_config.get("environment_type")
+ if not environment_type:
+ environment_type = self.DEFAULT_ENV_TYPE
configurations = env_config['configuration']
- config_validation = self.validate_environment_config(configurations)
+ config_validation = (
+ self.validate_environment_config(configurations=configurations,
+ environment_type=environment_type)
+ )
if not config_validation['passed']:
self.bad_request(config_validation['error_message'])
.format(env_config["name"])},
"201")
- def validate_environment_config(self, configurations,
+ def validate_environment_config(self, configurations, environment_type=None,
require_mandatory=True):
configurations_of_names = {}
validation = {"passed": True}
- if [config for config in configurations
- if 'name' not in config]:
+ if any('name' not in config for config in configurations):
validation['passed'] = False
validation['error_message'] = "configuration must have name"
return validation
"configuration for {0}".format(name)
return validation
configurations_of_names[name] = configs[0]
- elif require_mandatory:
- if name not in self.OPTIONAL_CONFIGURATIONS_NAMES:
- validation["passed"] = False
- validation['error_message'] = "configuration for {0} " \
- "is mandatory".format(name)
- return validation
+
+ if require_mandatory:
+ required_list = (
+ self.REQUIRED_CONFIGURATIONS_NAMES.get(environment_type, [])
+ )
+ if any(required_conf not in configurations_of_names
+ for required_conf
+ in required_list):
+ validation["passed"] = False
+ validation['error_message'] = ("configurations for ({})"
+ "are mandatory for "
+ "this environment type"
+ .format(", ".join(required_list)))
for name, config in configurations_of_names.items():
error_message = self.validate_configuration(name, config)
def validate_query_data(self, data, data_requirements,
additional_key_reg=None,
- can_be_empty_keys=[]):
+ can_be_empty_keys=None):
error_message = self.validate_data(data, data_requirements,
additional_key_reg,
can_be_empty_keys)
': no "value" key for data: ' + str(d))
return consts
- def read(self, collection, matches={}, projection=None, skip=0, limit=1000):
+ def read(self, collection, matches=None, projection=None, skip=0, limit=1000):
+ if matches is None:
+ matches = {}
collection = self.get_collection_by_name(collection)
skip *= limit
query = collection.find(matches, projection).skip(skip).limit(limit)
def validate_data(self, data, requirements,
additional_key_re=None,
- can_be_empty_keys=[]):
+ can_be_empty_keys=None):
+ if can_be_empty_keys is None:
+ can_be_empty_keys = []
illegal_keys = [key for key in data.keys()
if key not in requirements.keys()]
"FindLinksForVconnectors",
"FindLinksForVedges",
"FindLinksForVserviceVnics",
- "FindLinksForPnics"
+ "FindLinksForPnics",
+ "FindImplicitLinks"
]
}
\ No newline at end of file
"types_name": "regions",
"parent_type": "environment"
},
- "children_scanner": "ScanRegionsRoot"
+ "children_scanner": "ScanRegionsRoot",
+ "environment_condition": {"environment_type": "OpenStack"}
},
{
"type": "projects_folder",
"types_name": "projects",
"parent_type": "environment"
},
- "children_scanner": "ScanProjectsRoot"
+ "children_scanner": "ScanProjectsRoot",
+ "environment_condition": {"environment_type": "OpenStack"}
+ },
+ {
+ "type": "namespaces_folder",
+ "fetcher": {
+ "folder": true,
+ "types_name": "namespaces",
+ "parent_type": "environment"
+ },
+ "children_scanner": "ScanNamespacesRoot",
+ "environment_condition": {
+ "environment_type": "Kubernetes"
+ }
}
],
"ScanHostNetworkAgentsRoot": [
"type": "vservice",
"fetcher": "CliFetchHostVservices"
}
+ ],
+ "ScanNamespacesRoot": [
+ {
+ "type": "namespace",
+ "fetcher": "KubeFetchNamespaces",
+ "environment_condition": {"environment_type": "Kubernetes"}
+ }
]
}
}
+++ /dev/null
-###############################################################################
-# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
-# and others #
-# #
-# All rights reserved. This program and the accompanying materials #
-# are made available under the terms of the Apache License, Version 2.0 #
-# which accompanies this distribution, and is available at #
-# http://www.apache.org/licenses/LICENSE-2.0 #
-###############################################################################
-import argparse
-import datetime
-from kombu import Connection
-
-import time
-
-import pymongo
-from functools import partial
-
-from discover.fetchers.api.api_access import ApiAccess
-from discover.fetchers.db.db_access import DbAccess
-from discover.manager import Manager
-from utils.constants import ConnectionTestStatus, ConnectionTestType
-from utils.logging.file_logger import FileLogger
-from utils.mongo_access import MongoAccess
-from utils.ssh_connection import *
-
-
-def test_openstack(config, test_request):
- try:
- api = ApiAccess(config)
- ConnectionTest.report_success(test_request,
- ConnectionTestType.OPENSTACK.value)
- if api:
- pass
- except ValueError:
- pass
-
-
-def test_mysql(config, test_request):
- db_access = DbAccess(config)
- ConnectionTest.report_success(test_request, ConnectionTestType.MYSQL.value)
- if db_access:
- pass
-
-
-def test_ssh_connect(config) -> bool:
- ssh = SshConnection(config.get('host', ''),
- config.get('user', ''),
- _pwd=config.get('pwd'),
- _key=config.get('key'),
- _port=int(config.get('port',
- SshConnection.DEFAULT_PORT)))
- ret = ssh.connect()
- return ret
-
-
-def test_cli(config, test_request):
- ret = test_ssh_connect(config)
- ConnectionTest.set_test_result(test_request,
- ConnectionTestType.CLI.value,
- ret)
-
-
-def test_amqp_connect(config):
- connect_url = 'amqp://{user}:{pwd}@{host}:{port}//' \
- .format(user=config.get("user", ''),
- pwd=config.get('pwd', ''),
- host=config.get('host', ''),
- port=int(config.get('port', 5671)))
- conn = Connection(connect_url)
- conn.connect()
-
-
-def test_amqp(config, test_request):
- test_amqp_connect(config)
- ConnectionTest.report_success(test_request, ConnectionTestType.AMQP.value)
-
-
-def test_monitoring(config, test_request):
- # for monitoring configuration test, need to test:
- # 1. SSH access
- # 2. RabbitMQ access
- ssh_config = {
- 'host': config.get('server_ip'),
- 'user': config.get('ssh_user'),
- 'pwd': config.get('ssh_password'),
- 'port': int(config.get('ssh_port', 0))
- }
- if not test_ssh_connect(ssh_config):
- return
- amqp_connect_config = {
- 'user': config.get('rabbitmq_user', ''),
- 'pwd': config.get('rabbitmq_pass', ''),
- 'host': config.get('server_ip'),
- 'port': int(config.get('rabbitmq_port', 5672)),
- }
- test_amqp_connect(amqp_connect_config)
- ConnectionTest.report_success(test_request, ConnectionTestType.AMQP.value)
-
-
-def test_aci(config, test_request):
- pass
-
-
-TEST_HANDLERS = {
- ConnectionTestType.OPENSTACK.value: test_openstack,
- ConnectionTestType.MYSQL.value: test_mysql,
- ConnectionTestType.CLI.value: test_cli,
- ConnectionTestType.AMQP.value: test_amqp,
- ConnectionTestType.ACI.value: test_aci,
- ConnectionTestType.MONITORING.value: test_monitoring
-}
-
-
-class ConnectionTest(Manager):
-
- DEFAULTS = {
- 'mongo_config': '',
- 'connection_tests': 'connection_tests',
- 'environments': 'environments_config',
- 'interval': 1,
- 'loglevel': 'WARNING'
- }
-
- def __init__(self):
- self.args = self.get_args()
- super().__init__(log_directory=self.args.log_directory,
- mongo_config_file=self.args.mongo_config)
- self.db_client = None
- self.connection_tests_collection = None
- self.environments_collection = None
-
- @staticmethod
- def get_args():
- parser = argparse.ArgumentParser()
- parser.add_argument('-m', '--mongo_config', nargs='?', type=str,
- default=ConnectionTest.DEFAULTS['mongo_config'],
- help='Name of config file ' +
- 'with MongoDB server access details')
- parser.add_argument('-c', '--connection_tests_collection', nargs='?',
- type=str,
- default=ConnectionTest.DEFAULTS['connection_tests'],
- help='connection_tests collection to read from')
- parser.add_argument('-e', '--environments_collection', nargs='?',
- type=str,
- default=ConnectionTest.DEFAULTS['environments'],
- help='Environments collection to update '
- 'after tests')
- parser.add_argument('-i', '--interval', nargs='?', type=float,
- default=ConnectionTest.DEFAULTS['interval'],
- help='Interval between collection polls'
- '(must be more than {} seconds)'
- .format(ConnectionTest.MIN_INTERVAL))
- parser.add_argument('-l', '--loglevel', nargs='?', type=str,
- default=ConnectionTest.DEFAULTS['loglevel'],
- help='Logging level \n(default: {})'
- .format(ConnectionTest.DEFAULTS['loglevel']))
- parser.add_argument('-d', '--log_directory', nargs='?', type=str,
- default=FileLogger.LOG_DIRECTORY,
- help='File logger directory \n(default: {})'
- .format(FileLogger.LOG_DIRECTORY))
- args = parser.parse_args()
- return args
-
- def configure(self):
- self.db_client = MongoAccess()
- self.connection_tests_collection = \
- self.db_client.db[self.args.connection_tests_collection]
- self.environments_collection = \
- self.db_client.db[self.args.environments_collection]
- self._update_document = \
- partial(MongoAccess.update_document,
- self.connection_tests_collection)
- self.interval = max(self.MIN_INTERVAL, self.args.interval)
- self.log.set_loglevel(self.args.loglevel)
-
- self.log.info('Started ConnectionTest with following configuration:\n'
- 'Mongo config file path: {0.args.mongo_config}\n'
- 'connection_tests collection: '
- '{0.connection_tests_collection.name}\n'
- 'Polling interval: {0.interval} second(s)'
- .format(self))
-
- def _build_test_args(self, test_request: dict):
- args = {
- 'mongo_config': self.args.mongo_config
- }
-
- def set_arg(name_from: str, name_to: str = None):
- if name_to is None:
- name_to = name_from
- val = test_request.get(name_from)
- if val:
- args[name_to] = val
-
- set_arg('object_id', 'id')
- set_arg('log_level', 'loglevel')
- set_arg('environment', 'env')
- set_arg('scan_only_inventory', 'inventory_only')
- set_arg('scan_only_links', 'links_only')
- set_arg('scan_only_cliques', 'cliques_only')
- set_arg('inventory')
- set_arg('clear')
- set_arg('clear_all')
-
- return args
-
- def _finalize_test(self, test_request: dict):
- # update the status and timestamps.
- self.log.info('Request {} has been tested.'
- .format(test_request['_id']))
- start_time = test_request['submit_timestamp']
- end_time = datetime.datetime.utcnow()
- test_request['response_timestamp'] = end_time
- test_request['response_time'] = \
- str(end_time - start_time.replace(tzinfo=None))
- test_request['status'] = ConnectionTestStatus.RESPONSE.value
- self._update_document(test_request)
-
- @staticmethod
- def set_test_result(test_request, target, result):
- test_request.get('test_results', {})[target] = result
-
- @staticmethod
- def report_success(test_request, target):
- ConnectionTest.set_test_result(test_request, target, True)
-
- @staticmethod
- def handle_test_target(target, test_request):
- targets_config = test_request.get('targets_configuration', [])
- try:
- config = next(t for t in targets_config if t['name'] == target)
- except StopIteration:
- raise ValueError('failed to find {} in targets_configuration'
- .format(target))
- handler = TEST_HANDLERS.get(target)
- if not handler:
- raise ValueError('unknown test target: {}'.format(target))
- handler(config, test_request)
-
- def do_test(self, test_request):
- targets = [t for t in test_request.get('test_targets', [])]
- test_request['test_results'] = {t: False for t in targets}
- for test_target in test_request.get('test_targets', []):
- self.log.info('testing connection to: {}'.format(test_target))
- try:
- self.handle_test_target(test_target, test_request)
- except Exception as e:
- self.log.exception(e)
- if 'errors' not in test_request:
- test_request['errors'] = {}
- test_request['errors'][test_target] = str(e)
- self.log.error('Test of target {} failed (id: {}):\n{}'
- .format(test_target,
- test_request['_id'],
- str(e)))
- self._finalize_test(test_request)
- self._set_env_operational(test_request['environment'])
-
- # if environment_config document for this specific environment exists,
- # update the value of the 'operational' field to 'running'
- def _set_env_operational(self, env):
- self.environments_collection. \
- update_one({'name': env}, {'$set': {'operational': 'running'}})
-
- def do_action(self):
- while True:
- # Find a pending request that is waiting the longest time
- results = self.connection_tests_collection \
- .find({'status': ConnectionTestStatus.REQUEST.value,
- 'submit_timestamp': {'$ne': None}}) \
- .sort('submit_timestamp', pymongo.ASCENDING) \
- .limit(1)
-
- # If no connection tests are pending, sleep for some time
- if results.count() == 0:
- time.sleep(self.interval)
- else:
- self.do_test(results[0])
-
-
-if __name__ == '__main__':
- ConnectionTest().run()
self.find_cliques_for_type(clique_type)
self.log.info("finished scanning for cliques")
- # Calculate priority score
+ # Calculate priority score for clique type per environment and configuration
def _get_priority_score(self, clique_type):
+ # environment-specific clique type takes precedence
if self.env == clique_type['environment']:
+ return 16
+ if (self.env_config['distribution'] == clique_type.get('distribution')
+ and
+ self.env_config['distribution_version'] ==
+ clique_type.get('distribution_version')):
+ return 8
+ if clique_type.get('mechanism_drivers') \
+ in self.env_config['mechanism_drivers']:
return 4
- if (self.env_config['distribution'] == clique_type.get('distribution') and
- self.env_config['distribution_version'] == clique_type.get('distribution_version')):
- return 3
- if clique_type.get('mechanism_drivers') in self.env_config['mechanism_drivers']:
- return 2
if self.env_config['type_drivers'] == clique_type.get('type_drivers'):
+ return 2
+ if clique_type.get('environment', '') == 'ANY':
+ # environment=ANY serves as fallback option, but it's not mandatory
return 1
else:
return 0
# Get clique type with max priority
- # for given environment configuration and focal point type
- def _get_clique_type(self, focal_point, clique_types):
- # If there's no configuration match for the specified environment,
- # we use the default clique type definition with environment='ANY'
- fallback_type = next(
- filter(lambda t: t['environment'] == 'ANY', clique_types),
- None
- )
- if not fallback_type:
- raise ValueError("No fallback clique type (ANY) "
- "defined for focal point type '{}'"
- .format(focal_point))
-
- clique_types.remove(fallback_type)
-
- priority_scores = [self._get_priority_score(clique_type)
- for clique_type
- in clique_types]
- max_score = max(priority_scores) if priority_scores else 0
-
- return (fallback_type
- if max_score == 0
- else clique_types[priority_scores.index(max_score)])
+ # for given focal point type
+ def _get_clique_type(self, clique_types):
+ scored_clique_types = [{'score': self._get_priority_score(clique_type),
+ 'clique_type': clique_type}
+ for clique_type in clique_types]
+ max_score = max(scored_clique_types, key=lambda t: t['score'])
+ if max_score['score'] == 0:
+ self.log.warn('No matching clique types for focal point type: {}'
+ .format(clique_types[0].get('focal_point_type')))
+ return None
+ return max_score.get('clique_type')
def get_clique_types(self):
if not self.clique_types_by_type:
- clique_types_by_focal_point = self.clique_types.aggregate([{
- "$group": {
- "_id": "$focal_point_type",
- "types": {"$push": "$$ROOT"}
- }
- }])
-
- self.clique_types_by_type = {
- cliques['_id']: self._get_clique_type(cliques['_id'],
- cliques['types'])
- for cliques in
- clique_types_by_focal_point
- }
-
+ clique_types_candidates = {}
+ for clique in self.clique_types.find({}):
+ fp_type = clique.get('focal_point_type', '')
+ if not clique_types_candidates.get(fp_type):
+ clique_types_candidates[fp_type] = []
+ clique_types_candidates[fp_type].append(clique)
+ for t in clique_types_candidates.keys():
+ selected = self._get_clique_type(clique_types_candidates[t])
+ if not selected:
+ continue
+ self.clique_types_by_type[t] = selected
return self.clique_types_by_type
def find_cliques_for_type(self, clique_type):
.find_one({"focal_point_type": o['type']})
constraints = [] if not constraint else constraint["constraints"]
clique_types = self.get_clique_types()
- clique_type = clique_types[o['type']]
- new_clique = self.construct_clique_for_focal_point(o, clique_type,
- constraints)
- if not new_clique:
+ clique_type = clique_types.get(o['type'])
+ if not clique_type:
self.cliques.delete({'_id': clique['_id']})
+ else:
+ new_clique = self.construct_clique_for_focal_point(o, clique_type,
+ constraints)
+ if not new_clique:
+ self.cliques.delete({'_id': clique['_id']})
def construct_clique_for_focal_point(self, o, clique_type, constraints):
# keep a hash of nodes in clique that were visited for each type
for c in constraints:
val = o[c] if c in o else None
clique["constraints"][c] = val
+ allow_implicit = clique_type.get('use_implicit_links', False)
for link_type in clique_type["link_types"]:
- self.check_link_type(clique, link_type, nodes_of_type)
+ self.check_link_type(clique, link_type, nodes_of_type,
+ allow_implicit=allow_implicit)
# after adding the links to the clique, create/update the clique
if not clique["links"]:
return None
+ clique["clique_type"] = clique_type["_id"]
focal_point_obj = self.inventory.find({"_id": clique["focal_point"]})
if not focal_point_obj:
return None
'-'.join(link_type_parts)
return CliqueFinder.link_type_reversed.get(link_type)
- def check_link_type(self, clique, link_type, nodes_of_type):
+ def check_link_type(self, clique, link_type, nodes_of_type,
+ allow_implicit=False):
# check if it's backwards
link_type_reversed = self.get_link_type_reversed(link_type)
# handle case of links like T<-->T
self_linked = link_type == link_type_reversed
use_reversed = False
if not self_linked:
- matches = self.links.find_one({
+ link_search_condition = {
"environment": self.env,
"link_type": link_type_reversed
- })
+ }
+ if not allow_implicit:
+ link_search_condition['implicit'] = False
+ matches = self.links.find_one(link_search_condition)
use_reversed = True if matches else False
if self_linked or not use_reversed:
- self.check_link_type_forward(clique, link_type, nodes_of_type)
+ self.check_link_type_forward(clique, link_type, nodes_of_type,
+ allow_implicit=allow_implicit)
if self_linked or use_reversed:
- self.check_link_type_back(clique, link_type, nodes_of_type)
+ self.check_link_type_back(clique, link_type, nodes_of_type,
+ allow_implicit=allow_implicit)
def check_link_type_for_direction(self, clique, link_type, nodes_of_type,
- is_reversed=False):
+ is_reversed=False,
+ allow_implicit=False):
if is_reversed:
link_type = self.get_link_type_reversed(link_type)
from_type = link_type[:link_type.index("-")]
clique,
link_type,
side_to_match,
- other_side)
+ other_side,
+ allow_implicit=allow_implicit)
nodes_to_add = nodes_to_add | matches
if other_side_type not in nodes_of_type:
nodes_of_type[other_side_type] = set()
nodes_of_type[other_side_type] | nodes_to_add
def find_matches_for_point(self, match_point, clique, link_type,
- side_to_match, other_side) -> set:
+ side_to_match, other_side,
+ allow_implicit=False) -> set:
nodes_to_add = set()
- matches = self.links.find({
+ link_search_condition = {
"environment": self.env,
"link_type": link_type,
side_to_match: ObjectId(match_point)
- })
+ }
+ if not allow_implicit:
+ link_search_condition['implicit'] = False
+ matches = self.links.find(link_search_condition)
for link in matches:
link_id = link["_id"]
if link_id in clique["links"]:
nodes_to_add.add(other_side_point)
return nodes_to_add
- def check_link_type_forward(self, clique, link_type, nodes_of_type):
+ def check_link_type_forward(self, clique, link_type, nodes_of_type,
+ allow_implicit=False):
self.check_link_type_for_direction(clique, link_type, nodes_of_type,
- is_reversed=False)
+ is_reversed=False,
+ allow_implicit=allow_implicit)
- def check_link_type_back(self, clique, link_type, nodes_of_type):
+ def check_link_type_back(self, clique, link_type, nodes_of_type,
+ allow_implicit=False):
self.check_link_type_for_direction(clique, link_type, nodes_of_type,
- is_reversed=True)
+ is_reversed=True,
+ allow_implicit=allow_implicit)
def get_listener(self, env: str):
env_config = self.inv.get_env_config(env)
return (self.LISTENERS.get(env_config.get('distribution'), {})
- .get(env_config.get('distribution_version',
- DefaultListener)))
+ .get(env_config.get('distribution_version'),
+ DefaultListener))
def listen_to_events(self, listener: ListenerBase, env_name: str, process_vars: dict):
listener.listen({
import requests\r
import time\r
\r
-from discover.configuration import Configuration\r
-from discover.fetcher import Fetcher\r
+from utils.api_access_base import ApiAccessBase\r
from utils.string_utils import jsonify\r
\r
\r
-class ApiAccess(Fetcher):\r
+class ApiAccess(ApiAccessBase):\r
+\r
+ ADMIN_PORT = "35357"\r
+\r
subject_token = None\r
initialized = False\r
regions = {}\r
- config = None\r
- api_config = None\r
\r
- host = ""\r
- base_url = ""\r
- admin_token = ""\r
tokens = {}\r
admin_endpoint = ""\r
admin_project = None\r
\r
# identity API v2 version with admin token\r
def __init__(self, config=None):\r
- super(ApiAccess, self).__init__()\r
- if ApiAccess.initialized:\r
+ super().__init__('OpenStack', config)\r
+ self.base_url = "http://" + self.host + ":" + self.port\r
+ if self.initialized:\r
return\r
- ApiAccess.config = {'OpenStack': config} if config else Configuration()\r
- ApiAccess.api_config = ApiAccess.config.get("OpenStack")\r
- host = ApiAccess.api_config.get("host", "")\r
- ApiAccess.host = host\r
- port = ApiAccess.api_config.get("port", "")\r
- if not (host and port):\r
- raise ValueError('Missing definition of host or port ' +\r
- 'for OpenStack API access')\r
- ApiAccess.base_url = "http://" + host + ":" + port\r
- ApiAccess.admin_token = ApiAccess.api_config.get("admin_token", "")\r
- ApiAccess.admin_project = ApiAccess.api_config.get("admin_project",\r
- "admin")\r
- ApiAccess.admin_endpoint = "http://" + host + ":" + "35357"\r
+ ApiAccess.admin_project = self.api_config.get("admin_project", "admin")\r
+ ApiAccess.admin_endpoint = "http://" + self.host + ":" + self.ADMIN_PORT\r
\r
token = self.v2_auth_pwd(ApiAccess.admin_project)\r
if not token:\r
raise ValueError("Authentication failed. Failed to obtain token")\r
else:\r
self.subject_token = token\r
+ self.initialized = True\r
\r
@staticmethod\r
def parse_time(time_str):\r
subject_token = self.get_existing_token(project_id)\r
if subject_token:\r
return subject_token\r
- req_url = ApiAccess.base_url + "/v2.0/tokens"\r
+ req_url = self.base_url + "/v2.0/tokens"\r
response = requests.post(req_url, json=post_body, headers=headers,\r
- timeout=5)\r
+ timeout=self.CONNECT_TIMEOUT)\r
response = response.json()\r
ApiAccess.auth_response[project_id] = response\r
if 'error' in response:\r
return token_details\r
\r
def v2_auth_pwd(self, project):\r
- user = ApiAccess.api_config["user"]\r
- pwd = ApiAccess.api_config["pwd"]\r
+ user = self.api_config["user"]\r
+ pwd = self.api_config["pwd"]\r
post_body = {\r
"auth": {\r
"passwordCredentials": {\r
auth_response = ApiAccess.auth_response.get('admin', {})\r
return auth_response\r
\r
- def get_rel_url(self, relative_url, headers):\r
- req_url = ApiAccess.base_url + relative_url\r
- return self.get_url(req_url, headers)\r
-\r
- def get_url(self, req_url, headers):\r
- response = requests.get(req_url, headers=headers)\r
- if response.status_code != requests.codes.ok:\r
- # some error happened\r
- if "reason" in response:\r
- msg = ", reason: {}".format(response.reason)\r
- else:\r
- msg = ", response: {}".format(response.text)\r
- self.log.error("req_url: {} {}".format(req_url, msg))\r
- return None\r
- ret = response.json()\r
- return ret\r
-\r
def get_region_url(self, region_name, service):\r
if region_name not in self.regions:\r
return None\r
return None\r
orig_url = s["adminURL"]\r
# replace host name with the host found in config\r
- url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + ApiAccess.host, orig_url)\r
+ url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + self.host, orig_url)\r
return url\r
\r
# like get_region_url(), but remove everything starting from the "/v2"\r
def __init__(self):
super(ApiFetchHostInstances, self).__init__()
self.inv = InventoryMgr()
- self.endpoint = ApiAccess.base_url.replace(":5000", ":8774")
+ self.endpoint = self.base_url.replace(":5000", ":8774")
self.projects = None
self.db_fetcher = DbFetchInstances()
from discover.fetchers.api.api_access import ApiAccess
from discover.fetchers.db.db_access import DbAccess
+from discover.fetchers.cli.cli_access import CliAccess
+from utils.ssh_connection import SshError
-class ApiFetchProjectHosts(ApiAccess, DbAccess):
+class ApiFetchProjectHosts(ApiAccess, DbAccess, CliAccess):
def __init__(self):
super(ApiFetchProjectHosts, self).__init__()
s = services["nova-compute"]
if s["available"] and s["active"]:
self.add_host_type(doc, "Compute", az['zoneName'])
+ self.fetch_host_os_details(doc)
return doc
# fetch more details of network nodes from neutron DB agents table
""".format(self.neutron_db)
results = self.get_objects_list(query, "")
for r in results:
- host = hosts[r["host"]]
+ host = r["host"]
+ if host not in hosts:
+ self.log.error("host from agents table not in hosts list: {}"
+ .format(host))
+ continue
+ host = hosts[host]
host["config"] = json.loads(r["configurations"])
self.add_host_type(host, "Network", '')
for db_row in results:
doc.update(db_row)
- def add_host_type(self, doc, type, zone):
- if not type in doc["host_type"]:
- doc["host_type"].append(type)
- if type == 'Compute':
+ @staticmethod
+ def add_host_type(doc, host_type, zone):
+ if host_type not in doc["host_type"]:
+ doc["host_type"].append(host_type)
+ if host_type == 'Compute':
doc['zone'] = zone
doc['parent_id'] = zone
+
+ def fetch_host_os_details(self, doc):
+ cmd = 'cat /etc/os-release && echo "ARCHITECURE=`arch`"'
+ try:
+ lines = self.run_fetch_lines(cmd, ssh_to_host=doc['host'])
+ except SshError as e:
+ self.log.error('{}: {}', cmd, str(e))
+ os_attributes = {}
+ attributes_to_fetch = {
+ 'NAME': 'name',
+ 'VERSION': 'version',
+ 'ID': 'ID',
+ 'ID_LIKE': 'ID_LIKE',
+ 'ARCHITECURE': 'architecure'
+ }
+ for attr in attributes_to_fetch:
+ matches = [l for l in lines if l.startswith(attr + '=')]
+ if matches:
+ line = matches[0]
+ attr_name = attributes_to_fetch[attr]
+ os_attributes[attr_name] = line[line.index('=')+1:].strip('"')
+ if os_attributes:
+ doc['OS'] = os_attributes
class ApiFetchRegions(ApiAccess):\r
def __init__(self):\r
super(ApiFetchRegions, self).__init__()\r
- self.endpoint = ApiAccess.base_url\r
+ self.endpoint = self.base_url\r
\r
def get(self, regions_folder_id):\r
token = self.v2_auth_pwd(self.admin_project)\r
conn = None
query_count_per_con = 0
- # connection timeout set to 30 seconds,
- # due to problems over long connections
+ # connection timeout set to 5 seconds
TIMEOUT = 5
def __init__(self, mysql_config=None):
self.config = {'mysql': mysql_config} if mysql_config \
else Configuration()
self.conf = self.config.get("mysql")
+ self.connect_timeout = int(self.conf['connect_timeout']) \
+ if 'connect_timeout' in self.conf \
+ else self.TIMEOUT
self.connect_to_db()
self.neutron_db = self.get_neutron_db_name()
return
try:
connector = mysql.connector
- DbAccess.conn = connector.connect(host=_host, port=_port,
- connection_timeout=self.TIMEOUT,
- user=_user,
- password=_pwd,
- database=_database,
- raise_on_warnings=True)
+ conn = connector.connect(host=_host, port=_port,
+ connection_timeout=self.connect_timeout,
+ user=_user,
+ password=_pwd,
+ database=_database,
+ raise_on_warnings=True)
+ DbAccess.conn = conn
DbAccess.conn.ping(True) # auto-reconnect if necessary
except Exception as e:
- self.log.critical("failed to connect to MySQL DB: {}"
- .format(str(e)))
+ msg = "failed to connect to MySQL DB: {}".format(str(e))
+ self.log.critical(msg)
+ raise ScanError(msg)
return
DbAccess.query_count_per_con = 0
DbAccess.conn = None
self.conf = self.config.get("mysql")
cnf = self.conf
+ pwd = cnf.get('pwd', '')
+ if not pwd:
+ raise ScanError('db_access: attribute pwd is missing')
self.db_connect(cnf.get('host', ''), cnf.get('port', ''),
- cnf.get('user', ''), cnf.get('pwd', ''),
+ cnf.get('user', ''), pwd,
cnf.get('schema', 'nova'))
@with_cursor
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from kubernetes.client import Configuration as KubConf, CoreV1Api
+
+from utils.api_access_base import ApiAccessBase
+
+
+class KubeAccess(ApiAccessBase):
+
+ def __init__(self, config=None):
+ super().__init__('Kubernetes', config)
+ self.base_url = 'https://{}:{}'.format(self.host, self.port)
+ self.bearer_token = self.api_config.get('token', '')
+ conf = KubConf()
+ conf.host = self.base_url
+ conf.user = self.api_config.get('user')
+ conf.api_key_prefix['authorization'] = 'Bearer'
+ conf.api_key['authorization'] = self.bearer_token
+ conf.verify_ssl = False
+ self.api = CoreV1Api()
+
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from discover.fetchers.kube.kube_access import KubeAccess
+
+
+class KubeFetchNamespaces(KubeAccess):
+
+ def __init__(self, config=None):
+ super().__init__(config)
+
+ def get(self, object_id):
+ namespaces = self.api.list_namespace()
+ return [self.get_namespace(i) for i in namespaces.items]
+
+ @staticmethod
+ def get_namespace(namespace):
+ attrs = ['creation_timestamp', 'self_link', 'uid']
+ namespace_details = {
+ 'name': namespace.metadata.name,
+ 'status': namespace.status.phase
+ }
+ namespace_details.update({x: getattr(namespace.metadata, x, '')
+ for x in attrs})
+ namespace_details['id'] = namespace_details['uid']
+ return namespace_details
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from discover.link_finders.find_links import FindLinks
+
+
+class FindImplicitLinks(FindLinks):
+
+ def __init__(self):
+ super().__init__()
+ self.links = []
+ self.constraint_attributes = self.get_constraint_attributes()
+
+ def add_links(self):
+ self.log.info('adding implicit links')
+ self.get_existing_links()
+ self.get_transitive_closure()
+
+ def get_constraint_attributes(self) -> list:
+ attributes = set()
+ for c in self.inv.find({'environment': self.get_env()},
+ collection='clique_constraints'):
+ for a in c['constraints']:
+ attributes.add(a)
+ return list(attributes)
+
+ def get_existing_links(self):
+ self.log.info('fetching existing links')
+ existing_links = self.inv.find({'environment': self.get_env()},
+ collection='links')
+ for l in existing_links:
+ self.links.append({'pass': 0, 'link': l})
+
+ def constraints_match(self, link1, link2):
+ if 'attributes' not in link1 or 'attributes' not in link2:
+ return True
+ attr1 = link1['attributes']
+ attr2 = link2['attributes']
+ for a in self.constraint_attributes:
+ if a in attr1 and a in attr2 and attr1[a] != attr2[a]:
+ return False
+ return True
+
+ def links_match(self, start, dest):
+ if start['link_type'] == dest['link_type']:
+ return False # obviously we cannot make an implicit link of this
+ if start['source_id'] == dest['target_id']:
+ return False # avoid cyclic links
+ if not self.constraints_match(start, dest):
+ return False
+ return start['target_id'] == dest['source_id']
+
+ def add_matching_links(self, link, pass_no):
+ self.log.debug('looking for matches for link: {};{}'
+ .format(link['source_id'], link['target_id']))
+ matches = [l for l in self.links
+ if l['pass'] == 0 # take only original links
+ and self.links_match(link, l['link'])]
+ for l in matches:
+ implicit = self.add_implicit_link(link, l['link'])
+ self.links.append({'pass': pass_no, 'link': implicit})
+ return len(matches)
+
+ def get_link_constraint_attributes(self, link1, link2) -> dict:
+ attributes = {}
+ for a in self.constraint_attributes:
+ # constraints_match() verified the attribute values don't conflict
+ if a in link1.get('attributes', {}):
+ attributes[a] = link1['attributes'][a]
+ elif a in link2.get('attributes', {}):
+ attributes[a] = link2['attributes'][a]
+ return attributes
+
+ @staticmethod
+ def get_attr(attr, link1, link2):
+ if attr not in link1 and attr not in link2:
+ return None
+ if attr not in link1:
+ return link2[attr]
+ if attr not in link2 or link1[attr] == link2[attr]:
+ return link1[attr]
+ return None
+
+ def add_implicit_link(self, link1, link2):
+ link_type_from = link1['link_type'].split('-')[0]
+ link_type_to = link2['link_type'].split('-')[1]
+ link_type = '{}-{}'.format(link_type_from, link_type_to)
+ link_name = ''
+ state = 'down' \
+ if link1['state'] == 'down' or link2['state'] == 'down' \
+ else 'up'
+ link_weight = 0 # TBD
+ host = self.get_attr('host', link1, link2)
+ switch = self.get_attr('switch', link1, link2)
+ extra_attributes = self.get_link_constraint_attributes(link1, link2)
+ self.log.debug('adding implicit link: link type: {}, from: {}, to: {}'
+ .format(link_type,
+ link1['source_id'],
+ link2['target_id']))
+ implicit = self.create_link(self.get_env(),
+ link1['source'], link1['source_id'],
+ link2['target'], link2['target_id'],
+ link_type, link_name, state, link_weight,
+ host=host, switch=switch,
+ implicit=True,
+ extra_attributes=extra_attributes)
+ return implicit
+
+ def get_transitive_closure(self):
+ pass_no = 1
+ while True:
+ match_count = 0
+ last_pass_links = [l for l in self.links if l['pass'] == pass_no-1]
+ for l in last_pass_links:
+ match_count += self.add_matching_links(l['link'], pass_no)
+ self.log.info('Transitive closure pass #{}: '
+ 'found {} implicit links'
+ .format(pass_no, match_count))
+ if match_count == 0:
+ break
+ pass_no += 1
+ self.log.info('done adding implicit links')
def create_link(self, env, source, source_id, target, target_id,
link_type, link_name, state, link_weight,
host=None, switch=None,
+ implicit=False,
extra_attributes=None):
if extra_attributes is None:
extra_attributes = {}
link = self.inv.create_link(env,
source, source_id, target, target_id,
link_type, link_name, state, link_weight,
+ implicit=implicit,
source_label=source_label,
target_label=target_label,
host=host, switch=switch,
extra_attributes=extra_attributes)
if self.inv.monitoring_setup_manager:
self.inv.monitoring_setup_manager.create_setup(link)
+ return link
network_id = net['network']['id']
v['network'] = network_id
self.inv.set(v)
+ if self.inv.monitoring_setup_manager:
+ self.inv.monitoring_setup_manager.create_setup(instance)
break
state = "up" # TBD
link_weight = 0 # TBD
for interval in self.INTERVALS.keys():
self._prepare_scheduled_requests_for_interval(interval)
+ def handle_scans(self):
+ self._prepare_scheduled_requests()
+
+ # Find a pending request that is waiting the longest time
+ results = self.scans_collection \
+ .find({'status': ScanStatus.PENDING.value,
+ 'submit_timestamp': {'$ne': None}}) \
+ .sort("submit_timestamp", pymongo.ASCENDING) \
+ .limit(1)
+
+ # If no scans are pending, sleep for some time
+ if results.count() == 0:
+ time.sleep(self.interval)
+ else:
+ scan_request = results[0]
+ env = scan_request.get('environment')
+ scan_feature = EnvironmentFeatures.SCANNING
+ if not self.inv.is_feature_supported(env, scan_feature):
+ self.log.error("Scanning is not supported for env '{}'"
+ .format(scan_request.get('environment')))
+ self._fail_scan(scan_request)
+ return
+
+ scan_request['start_timestamp'] = datetime.datetime.utcnow()
+ scan_request['status'] = ScanStatus.RUNNING.value
+ self._update_document(scan_request)
+
+ # Prepare scan arguments and run the scan with them
+ try:
+ scan_args = self._build_scan_args(scan_request)
+
+ self.log.info("Starting scan for '{}' environment"
+ .format(scan_args.get('env')))
+ self.log.debug("Scan arguments: {}".format(scan_args))
+ result, message = ScanController().run(scan_args)
+ except ScanArgumentsError as e:
+ self.log.error("Scan request '{id}' "
+ "has invalid arguments. "
+ "Errors:\n{errors}"
+ .format(id=scan_request['_id'],
+ errors=e))
+ self._fail_scan(scan_request)
+ except Exception as e:
+ self.log.exception(e)
+ self.log.error("Scan request '{}' has failed."
+ .format(scan_request['_id']))
+ self._fail_scan(scan_request)
+ else:
+ # Check is scan returned success
+ if not result:
+ self.log.error(message)
+ self.log.error("Scan request '{}' has failed."
+ .format(scan_request['_id']))
+ self._fail_scan(scan_request)
+ return
+
+ # update the status and timestamps.
+ self.log.info("Request '{}' has been scanned. ({})"
+ .format(scan_request['_id'], message))
+ end_time = datetime.datetime.utcnow()
+ scan_request['end_timestamp'] = end_time
+ self._complete_scan(scan_request, message)
+
def do_action(self):
self._clean_up()
try:
while True:
- self._prepare_scheduled_requests()
-
- # Find a pending request that is waiting the longest time
- results = self.scans_collection \
- .find({'status': ScanStatus.PENDING.value,
- 'submit_timestamp': {'$ne': None}}) \
- .sort("submit_timestamp", pymongo.ASCENDING) \
- .limit(1)
-
- # If no scans are pending, sleep for some time
- if results.count() == 0:
- time.sleep(self.interval)
- else:
- scan_request = results[0]
- env = scan_request.get('environment')
- scan_feature = EnvironmentFeatures.SCANNING
- if not self.inv.is_feature_supported(env, scan_feature):
- self.log.error("Scanning is not supported for env '{}'"
- .format(scan_request.get('environment')))
- self._fail_scan(scan_request)
- continue
-
- scan_request['start_timestamp'] = datetime.datetime.utcnow()
- scan_request['status'] = ScanStatus.RUNNING.value
- self._update_document(scan_request)
-
- # Prepare scan arguments and run the scan with them
- try:
- scan_args = self._build_scan_args(scan_request)
-
- self.log.info("Starting scan for '{}' environment"
- .format(scan_args.get('env')))
- self.log.debug("Scan arguments: {}".format(scan_args))
- result, message = ScanController().run(scan_args)
- except ScanArgumentsError as e:
- self.log.error("Scan request '{id}' "
- "has invalid arguments. "
- "Errors:\n{errors}"
- .format(id=scan_request['_id'],
- errors=e))
- self._fail_scan(scan_request)
- except Exception as e:
- self.log.exception(e)
- self.log.error("Scan request '{}' has failed."
- .format(scan_request['_id']))
- self._fail_scan(scan_request)
- else:
- # Check is scan returned success
- if not result:
- self.log.error(message)
- self.log.error("Scan request '{}' has failed."
- .format(scan_request['_id']))
- self._fail_scan(scan_request)
- continue
-
- # update the status and timestamps.
- self.log.info("Request '{}' has been scanned. ({})"
- .format(scan_request['_id'], message))
- end_time = datetime.datetime.utcnow()
- scan_request['end_timestamp'] = end_time
- self._complete_scan(scan_request, message)
+ self.handle_scans()
finally:
self._clean_up()
self.add_error('missing or empty fetcher in scanner {} type #{}'
.format(scanner_name, str(type_index)))
elif isinstance(fetcher, str):
+ error_str = None
try:
- module_name = ClassResolver.get_module_file_by_class_name(fetcher)
+ get_module = ClassResolver.get_module_file_by_class_name
+ module_name = get_module(fetcher)
fetcher_package = module_name.split("_")[0]
if package:
fetcher_package = ".".join((package, fetcher_package))
- instance = ClassResolver.get_instance_of_class(package_name=fetcher_package,
- module_name=module_name,
- class_name=fetcher)
- except ValueError:
- instance = None
- if not instance:
+ # get the fetcher qualified class but not a class instance
+ # instances will be created just-in-time (before fetching):
+ # this avoids init of access classes not needed in some envs
+ get_class = ClassResolver.get_fully_qualified_class
+ class_qualified = get_class(fetcher, fetcher_package,
+ module_name)
+ except ValueError as e:
+ class_qualified = None
+ error_str = str(e)
+ if not class_qualified:
self.add_error('failed to find fetcher class {} in scanner {}'
- ' type #{}'
- .format(fetcher, scanner_name, type_index))
- scan_type[self.FETCHER] = instance
+ ' type #{} ({})'
+ .format(fetcher, scanner_name, type_index,
+ error_str))
+ scan_type[self.FETCHER] = class_qualified
elif isinstance(fetcher, dict):
is_folder = fetcher.get('folder', False)
if not is_folder:
def validate_children_scanner(self, scanner_name: str, type_index: int,
scanners: dict, scan_type: dict):
- scanner = scanners[scanner_name]
if 'children_scanner' in scan_type:
children_scanner = scan_type.get('children_scanner')
if not isinstance(children_scanner, str):
class Scanner(Fetcher):
+
+ ENV_TYPE_OPENSTACK = 'OpenStack'
+ ENV_TYPE_KUBERNETES = 'Kubernetes'
+
config = None
environment = None
env = None
def check_type_env(self, type_to_fetch):
# check if type is to be run in this environment
- if "environment_condition" not in type_to_fetch:
- return True
- env_cond = type_to_fetch.get("environment_condition", {})
+ basic_cond = {'environment_type': self.ENV_TYPE_OPENSTACK}
+ env_cond = type_to_fetch.get("environment_condition", {}) \
+ if "environment_condition" in type_to_fetch \
+ else basic_cond
if not env_cond:
- return True
+ env_cond = basic_cond
+ if 'environment_type' not in env_cond:
+ env_cond.update(basic_cond)
if not isinstance(env_cond, dict):
self.log.warn('illegal environment_condition given '
'for type {}'.format(type_to_fetch['type']))
return True
conf = self.config.get_env_config()
+ if 'environment_type' not in conf:
+ conf.update(basic_cond)
for attr, required_val in env_cond.items():
if attr == "mechanism_drivers":
if "mechanism_drivers" not in conf:
# get Fetcher instance
fetcher = type_to_fetch["fetcher"]
+ if not isinstance(fetcher, Fetcher):
+ type_to_fetch['fetcher'] = fetcher() # make it an instance
+ fetcher = type_to_fetch["fetcher"]
fetcher.set_env(self.get_env())
# get children_scanner instance
def load_link_finders_metadata(self):
parser = FindLinksMetadataParser()
- conf = self.config.get_env_config()
finders_file = os.path.join(self.get_run_app_path(),
'config',
FindLinksMetadataParser.FINDERS_FILE)
environment=[root_url, mongo_url, LDAP_CONFIG])
+def start_test():
+ name = "calipso-test"
+ if container_started(name):
+ return
+ print("\nstarting container {}...\n".format(name))
+ image_name = "korenlev/calipso:test"
+ download_image(image_name)
+ ports = {'22/tcp': 10022}
+ DockerClient.containers.run(image_name,
+ detach=True,
+ name=name,
+ ports=ports,
+ restart_policy=RESTART_POLICY,
+ environment=[PYTHON_PATH, MONGO_CONFIG],
+ volumes=calipso_volume)
+
# check and stop a calipso container by given name
def container_stop(container_name):
if not container_started(container_name, print_message=False):
container = ""
action = ""
-container_names = ["calipso-ui", "calipso-scan", "calipso-listen",
+container_names = ["calipso-ui", "calipso-scan", "calipso-test", "calipso-listen",
"calipso-ldap", "calipso-api", "calipso-sensu", "calipso-mongo"]
container_actions = ["stop", "start"]
while action not in container_actions:
if container == "calipso-scan" or container == "all":
start_scan()
time.sleep(1)
+ if container == "calipso-test" or container == "all":
+ start_test()
+ time.sleep(1)
if container == "calipso-sensu" or container == "all":
start_sensu(args.uchiwaport, args.sensuport, args.rabbitport, args.rabbitmport)
time.sleep(1)
],
"name" : "vservice"
},
+{
+ "environment" : "config_based_example",
+ "focal_point_type" : "vservice",
+ "link_types" : [
+ "vservice-vnic",
+ "vnic-vedge",
+ "vedge-otep",
+ "otep-vconnector",
+ "vconnector-host_pnic",
+ "host_pnic-network"
+ ],
+ "name" : "vservice_config_based",
+ "distribution" : "Mirantis",
+ "distribution_version" : "6.0",
+ "mechanism_drivers" : "OVS",
+ "type_drivers" : "vxlan"
+},
{
"environment" : "ANY",
"focal_point_type" : "network",
"vnic-vservice"
],
"name" : "network"
+},
+{
+ "name" : "instance",
+ "use_implicit_links" : true,
+ "link_types" : [
+ "instance-network"
+ ],
+ "environment" : "implicit-links-ex",
+ "focal_point_type" : "instance"
}
]
],
"name" : "log_levels"
},
+{
+ "data" : [
+ {
+ "label" : "OpenStack",
+ "value" : "OpenStack"
+ },
+ {
+ "label" : "Kubernetes",
+ "value" : "Kubernetes"
+ },
+ {
+ "label" : "VMware",
+ "value" : "VMware"
+ },
+ {
+ "label" : "Bare-metal",
+ "value" : "Bare-metal"
+ }
+ ],
+ "name" : "environment_types"
+},
{
"data" : [
{
"value" : "10239",
"label" : "10239"
},
+ {
+ "label" : "10307",
+ "value" : "10307"
+ },
{
"value" : "10918",
"label" : "10918"
{
"label" : "switch",
"value" : "switch"
+ },
+ {
+ "value" : "namespace",
+ "label" : "namespace"
+ },
+ {
+ "value" : "namespaces_folder",
+ "label" : "namespaces_folder"
}
]
},
"server_name" : "sensu_server",
"env_type" : "production",
"provision" : "None",
- "name" : "Monitoring",
+ "name" : "Monitoring",
+ "install_monitoring_client": false,
"ssh_port" : "20022",
"rabbitmq_pass" : "dummy_pwd",
"ssh_password" : "dummy_pwd",
}
],
"enable_monitoring" : true,
- "name" : "DEMO-ENVIRONMENT-SCHEME",
+ "name" : "DEMO-OpenStack",
"distribution" : "Mirantis",
"distribution_version" : "8.0",
"last_scanned" : "filled-by-scanning",
"wNLeBJxNDyw8G7Ssg"
]
},
- "type" : "environment"
+ "type" : "environment",
+ "environment_type" : "OpenStack"
+},
+{
+ "user" : "wNLeBJxNDyw8G7Ssg",
+ "name" : "DEMO-Kubernetes",
+ "last_scanned" : "filled-by-scanning",
+ "auth" : {
+ "view-env" : [
+ "wNLeBJxNDyw8G7Ssg"
+ ],
+ "edit-env" : [
+ "wNLeBJxNDyw8G7Ssg"
+ ]
+ },
+ "type_drivers" : "vxlan",
+ "distribution_version" : "8.0",
+ "enable_monitoring" : true,
+ "operational" : "stopped",
+ "mechanism_drivers" : [
+ "OVS"
+ ],
+ "type" : "environment",
+ "distribution" : "Mirantis",
+ "listen" : true,
+ "configuration" : [
+ {
+ "user" : "adminuser",
+ "name" : "OpenStack",
+ "pwd" : "dummy_pwd",
+ "host" : "10.0.0.1",
+ "admin_token" : "dummy_token",
+ "port" : "5000"
+ },
+ {
+ "host" : "10.56.20.78",
+ "name" : "Kubernetes",
+ "user" : "koren",
+ "token" : "baba-token-xyz",
+ "port" : "6443"
+ },
+ {
+ "user" : "mysqluser",
+ "name" : "mysql",
+ "pwd" : "dummy_pwd",
+ "port" : "3307",
+ "host" : "10.0.0.1"
+ },
+ {
+ "user" : "sshuser",
+ "name" : "CLI",
+ "pwd" : "dummy_pwd",
+ "host" : "10.0.0.1"
+ },
+ {
+ "user" : "rabbitmquser",
+ "name" : "AMQP",
+ "pwd" : "dummy_pwd",
+ "port" : "5673",
+ "host" : "10.0.0.1"
+ },
+ {
+ "name" : "Monitoring",
+ "install_monitoring_client": false,
+ "api_port" : 4567,
+ "ssh_port" : "20022",
+ "rabbitmq_pass" : "dummy_pwd",
+ "env_type" : "production",
+ "rabbitmq_port" : "5671",
+ "server_ip" : "10.0.0.1",
+ "config_folder" : "/local_dir/sensu_config",
+ "type" : "Sensu",
+ "provision" : "None",
+ "ssh_user" : "root",
+ "ssh_password" : "dummy_pwd",
+ "rabbitmq_user" : "sensu",
+ "server_name" : "sensu_server"
+ },
+ {
+ "user" : "admin",
+ "name" : "ACI",
+ "pwd" : "dummy_pwd",
+ "host" : "10.1.1.104"
+ }
+ ],
+ "app_path" : "/home/scan/calipso_prod/app",
+ "scanned" : false,
+ "environment_type" : "Kubernetes"
}
]
"monitoring_system" : "sensu", \r
"type" : "client_check_link_vnic-vconnector.json"\r
},\r
+{\r
+ "side" : "client",\r
+ "order" : "1",\r
+ "condition" : {\r
+ "mechanism_drivers" : [\r
+ "OVS"\r
+ ]\r
+ },\r
+ "config" : {\r
+ "checks" : {\r
+ "{objtype}_{objid}" : {\r
+ "interval" : 15,\r
+ "command" : "check_vconnector_ovs.py {name}",\r
+ "standalone" : true,\r
+ "type": "metric",\r
+ "subscribers" : [\r
+ "base"\r
+ ],\r
+ "handlers" : [\r
+ "file",\r
+ "osdna-monitor"\r
+ ]\r
+ }\r
+ }\r
+ },\r
+ "monitoring_system" : "sensu",\r
+ "type" : "client_check_vconnector_ovs.json"\r
+},\r
{\r
"side" : "client", \r
"order" : "1", \r
}, \r
"monitoring_system" : "sensu", \r
"type" : "client_check_vservice.json"\r
+},\r
+{\r
+ "side" : "client",\r
+ "order" : "1",\r
+ "config" : {\r
+ "checks" : {\r
+ "{objtype}_{objid}" : {\r
+ "standalone" : true,\r
+ "interval" : 15,\r
+ "command" : "PYTHONPATH=/etc/sensu/plugins check_instance_communications.py {services_and_vnics}",\r
+ "handlers" : [\r
+ "file",\r
+ "osdna-monitor"\r
+ ],\r
+ "type" : "metric",\r
+ "subscribers" : [\r
+ "base"\r
+ ]\r
+ }\r
+ }\r
+ },\r
+ "monitoring_system" : "sensu",\r
+ "type" : "client_check_instance.json"\r
}\r
]\r
"monitoring" : true
}
},
+ {
+ "environment" : {
+ "distribution_version" : [
+ "10307"
+ ],
+ "distribution" : "Mercury",
+ "type_drivers" : "vlan",
+ "mechanism_drivers" : "OVS"
+ },
+ "features" : {
+ "scanning" : true,
+ "monitoring" : false,
+ "listening" : true
+ }
+ },
{
"environment" : {
"distribution" : "Devstack",
--- /dev/null
+#!/usr/bin/env python3
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+
+# find status of instance network
+# For each instance vNIC - take the MAC address
+# For each vService in the same network as the instance,
+# use local_service_id attribute in the following command in the network node:
+# "ip netns exec <local_service_id> arp -n"
+# look for the instance vNIC's mac_address to appear in the response
+# for each mac_address:
+# - if Flag 'C' = 'Complete' - mark result OK for that instance,
+# - 'I' = 'Incomplete' - mark as 'warn',
+# - no mac_address mark as 'error'
+
+import sys
+import subprocess
+
+from binary_converter import binary2str
+
+
+arp_headers = ['Address', 'HWtype', 'HWaddress', 'Flags', 'Mask', 'Iface']
+arp_mac_pos = arp_headers.index('HWaddress')
+arp_flags_pos = arp_headers.index('Flags')
+
+
+def check_vnic_tuple(vnic_and_service: str):
+ tuple_parts = vnic_and_service.split(',')
+ local_service_id = tuple_parts[0]
+ mac_address = tuple_parts[1]
+ check_output = None
+ try:
+ netns_cmd = 'ip netns exec {} arp -n'.format(local_service_id)
+ check_output = 'MAC={}, local_service_id={}\n'\
+ .format(mac_address, local_service_id)
+ netns_out = subprocess.check_output([netns_cmd],
+ stderr=subprocess.STDOUT,
+ shell=True)
+ netns_out = binary2str(netns_out)
+ check_output += '{}\n'.format(netns_out)
+ netns_lines = netns_out.splitlines()
+ if not netns_lines or \
+ netns_lines[0].endswith('No such file or directory'):
+ check_rc = 2
+ else:
+ mac_found = False
+ flags = None
+ for l in netns_lines:
+ line_parts = l.split()
+ line_mac = line_parts[arp_mac_pos]
+ if len(line_parts) > arp_mac_pos and line_mac == mac_address:
+ mac_found = True
+ flags = line_parts[arp_flags_pos]
+ break
+ if mac_found:
+ check_rc = 1 if flags == 'I' else 0
+ else:
+ check_rc = 2
+ except subprocess.CalledProcessError as e:
+ check_output = str(e)
+ check_rc = 2
+ return check_rc, check_output
+
+
+if len(sys.argv) < 2:
+ print('usage: ' + sys.argv[0] +
+ ' <vService local_service_id>,<MAC>[;<>,<>]...')
+ exit(1)
+
+rc = 0
+output = ''
+vnics = str(sys.argv[1]).split(';')
+for vnic_tuple in vnics:
+ tuple_ret, out = check_vnic_tuple(vnic_tuple)
+ rc = min(rc, tuple_ret)
+ output += out
+print(output)
+exit(rc)
--- /dev/null
+#!/usr/bin/env python3
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+
+# find status of vconnector
+# vconnector object name defines name of bridge
+# use "brctl showmacs <bridge>", return ERROR if 'No such device' is returned
+
+import sys
+import subprocess
+
+from binary_converter import binary2str
+
+
+if len(sys.argv) < 2:
+ print('usage: ' + sys.argv[0] + ' <bridge>')
+ exit(1)
+bridge_name = str(sys.argv[1])
+
+rc = 0
+
+cmd = None
+out = ''
+try:
+ cmd = "brctl showmacs {}".format(bridge_name)
+ out = subprocess.check_output([cmd],
+ stderr=subprocess.STDOUT,
+ shell=True)
+ out = binary2str(out)
+ lines = out.splitlines()
+ if not lines or lines[0].endswith('No such device'):
+ rc = 2
+ else:
+ print(out)
+except subprocess.CalledProcessError as e:
+ rc = 2
+ out = str(e)
+
+if rc != 0:
+ print('Failed to find vConnector {}:\n{}\n'
+ .format(bridge_name, out))
+
+exit(rc)
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+# handle monitoring event for pNIC objects
+
+from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler
+from utils.special_char_converter import SpecialCharConverter
+
+
+class HandleVconnector(MonitoringCheckHandler):
+
+ def handle(self, obj_id, check_result):
+ object_id = obj_id[:obj_id.index('-')]
+ mac = obj_id[obj_id.index('-')+1:]
+ converter = SpecialCharConverter()
+ mac_address = converter.decode_special_characters(mac)
+ object_id += '-' + mac_address
+ doc = self.doc_by_id(object_id)
+ if not doc:
+ return 1
+ self.keep_result(doc, check_result)
+ return check_result['status']
# handle monitoring events
import argparse
+import datetime
import json
import sys
+from discover.configuration import Configuration
+from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler
from utils.inventory_mgr import InventoryMgr
from utils.mongo_access import MongoAccess
+from utils.special_char_converter import SpecialCharConverter
from utils.util import ClassResolver
MongoAccess.set_config_file(self.args.mongo_config)
self.inv = InventoryMgr()
self.inv.set_collections(self.args.inventory)
+ self.configuration = Configuration()
self.input_text = None
+ self.converter = SpecialCharConverter()
def get_args(self):
parser = argparse.ArgumentParser()
return handler
def get_handler(self, check_type, obj_type):
- basic_handling_types = ['vedge', 'vservice']
+ basic_handling_types = ['instance', 'vedge', 'vservice', 'vconnector']
if obj_type not in basic_handling_types:
return self.get_handler_by_type(check_type, obj_type)
from monitoring.handlers.basic_check_handler \
import BasicCheckHandler
return BasicCheckHandler(self.args)
+ def check_link_interdependency_for(self,
+ object_id: str,
+ from_type: str=None,
+ to_type: str=None):
+ if from_type is not None and to_type is not None or \
+ from_type is None and to_type is None:
+ raise ValueError('check_link_interdependency: '
+ 'supply one of from_type/to_type')
+ obj_id = self.converter.decode_special_characters(object_id)
+ obj = self.inv.get_by_id(environment=self.args.env, item_id=obj_id)
+ if not obj:
+ self.inv.log.error('check_link_interdependency: '
+ 'failed to find object with ID: {}'
+ .format(object_id))
+ return
+ if 'status' not in obj:
+ return
+ id_attr = 'source_id' if from_type is None else 'target_id'
+ link_type = '{}-{}'.format(
+ from_type if from_type is not None else obj['type'],
+ to_type if to_type is not None else obj['type'])
+ condition = {
+ 'environment': self.args.env,
+ 'link_type': link_type,
+ id_attr: obj_id
+ }
+ link = self.inv.find_one(search=condition, collection='links')
+ if not link:
+ self.inv.log.error('check_link_interdependency: '
+ 'failed to find {} link with {}: {}'
+ .format(link_type, id_attr, obj_id))
+ return
+ other_id_attr = '{}_id' \
+ .format('source' if from_type is not None else 'target')
+ other_obj = self.inv.get_by_id(environment=self.args.env,
+ item_id=link[other_id_attr])
+ if not other_obj:
+ self.inv.log.error('check_link_interdependency: '
+ 'failed to find {} with ID: {} (link type: {})'
+ .format(other_id_attr, link[other_id_attr],
+ link_type))
+ return
+ if 'status' not in other_obj:
+ return
+ status = 'Warning'
+ if obj['status'] == 'OK' and other_obj['status'] == 'OK':
+ status = 'OK'
+ elif obj['status'] == 'OK' and other_obj['status'] == 'OK':
+ status = 'OK'
+ link['status'] = status
+ time_format = MonitoringCheckHandler.TIME_FORMAT
+ timestamp1 = obj['status_timestamp']
+ t1 = datetime.datetime.strptime(timestamp1, time_format)
+ timestamp2 = other_obj['status_timestamp']
+ t2 = datetime.datetime.strptime(timestamp2, time_format)
+ timestamp = max(t1, t2)
+ link['status_timestamp'] = datetime.datetime.strftime(timestamp,
+ time_format)
+ self.inv.set(link, self.inv.collections['links'])
+
+ def check_link_interdependency(self, object_id: str, object_type: str):
+ conf = self.configuration.get_env_config()
+ if 'OVS' in conf['mechanism_drivers']:
+ if object_type == 'vedge':
+ self.check_link_interdependency_for(object_id,
+ to_type='host_pnic')
+ if object_type == 'host_pnic':
+ self.check_link_interdependency_for(object_id,
+ from_type='vedge')
+
def process_input(self):
check_result_full = json.loads(self.input_text)
check_client = check_result_full['client']
monitor.find_object_type_and_id(name)
if 'environment' in check_client:
self.args.env = check_client['environment']
+ else:
+ raise ValueError('Check client should contain environment name')
+ self.configuration.use_env(self.args.env)
check_handler = self.get_handler(check_type, object_type)
if check_handler:
check_handler.handle(object_id, check_result)
+ self.check_link_interdependency(object_id, object_type)
def process_check_result(self):
self.read_input()
self.process_input()
+
monitor = Monitor()
monitor.process_check_result()
from utils.special_char_converter import SpecialCharConverter
from utils.string_utils import stringify_datetime
-TIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
SOURCE_SYSTEM = 'Sensu'
ERROR_LEVEL = ['info', 'warn', 'error']
class MonitoringCheckHandler(SpecialCharConverter):
STATUS_LABEL = ['OK', 'Warning', 'Error']
+ TIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
def __init__(self, args):
super().__init__()
else status
if status_text:
doc['status_text'] = status_text
- doc['status_timestamp'] = strftime(TIME_FORMAT, timestamp)
+ doc['status_timestamp'] = strftime(self.TIME_FORMAT, timestamp)
if 'link_type' in doc:
self.inv.write_link(doc)
else:
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
from monitoring.setup.monitoring_handler import MonitoringHandler
-from utils.inventory_mgr import InventoryMgr
from utils.special_char_converter import SpecialCharConverter
type_str = values['check_type'] if 'check_type' in values else \
(o['type'] if 'type' in o else 'link_' + o['link_type'])
file_type = 'client_check_' + type_str + '.json'
- host = o['host']
+ host = values['host'] if 'host' in values else o['host']
sub_dir = '/host/' + host
content = self.prepare_config_file(
file_type,
{'side': 'client', 'type': file_type})
# need to put this content inside client.json file
client_file = 'client.json'
- host = o['host']
client_file_content = self.get_config_from_db(host, client_file)
# merge checks attribute from current content into client.json
checks = client_file_content['config']['checks'] \
}
content = client_file_content
self.write_config_file(client_file, sub_dir, host, content)
+
+ def get_check_from_db(self, o, postfix=''):
+ client_config = self.get_config_from_db(o. get('host', ''),
+ 'client.json')
+ if not client_config:
+ return {}
+ checks = client_config.get('config', {}).get('checks', {})
+ objid = self.encode_special_characters(o.get('id', ''))
+ object_check_id = '{}_{}{}'.format(o.get('type'), objid, postfix)
+ check = checks.get(object_check_id, {})
+ return check
from os.path import join, sep
from monitoring.setup.monitoring_handler import MonitoringHandler
+from monitoring.setup.sensu_client_installer import SensuClientInstaller
RABBITMQ_CONFIG_FILE = 'rabbitmq.json'
RABBITMQ_CONFIG_ATTR = 'rabbitmq'
# add monitoring setup for remote host
def create_setup(self, o):
+ host_id = o.get('host', '')
+ self.install_sensu_on_host(host_id)
sensu_host_files = [
'transport.json',
'rabbitmq.json',
'client.json'
]
server_ip = self.env_monitoring_config['server_ip']
- host_id = o['host']
sub_dir = join('/host', host_id)
config = copy.copy(self.env_monitoring_config)
env_name = self.configuration.env_name
# this configuration requires SSL
# keep the path of the files for later use
self.fetch_ssl_files.append(path)
+
+ def install_sensu_on_host(self, host_id):
+ auto_install = self.env_monitoring_config \
+ .get('install_monitoring_client', False)
+ if auto_install:
+ installer = SensuClientInstaller(self.env, host_id)
+ installer.install()
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from monitoring.setup.monitoring_simple_object import MonitoringSimpleObject
+
+
+class MonitoringInstance(MonitoringSimpleObject):
+
+ def __init__(self, env):
+ super().__init__(env)
+
+ # monitoring setup for instance can only be done after vNIC is found
+ # and network for vNIC is set, so the first call will not do anything
+ def create_setup(self, instance: dict):
+ vnics = self.inv.find_items({
+ 'environment': self.get_env(),
+ 'type': 'vnic',
+ 'vnic_type': 'instance_vnic',
+ 'id_path': {'$regex': '^{}/'.format(instance['id_path'])}
+ })
+ for vnic in vnics:
+ self.add_instance_communication_monitoring(instance, vnic)
+
+ # for instance we keep list of instance vNICs and services to use in call
+ # to check_instance_communications.py
+ # add this vNIC to the list with the corresponding
+ def add_instance_communication_monitoring(self, instance: dict, vnic: dict):
+ service = self.get_service_for_vnic(vnic)
+ if not service:
+ return
+ check = self.get_check_from_db(instance)
+ services_and_vnics = check.get('command', '')
+ if services_and_vnics:
+ services_and_vnics = \
+ services_and_vnics[services_and_vnics.index('.py')+4:]
+ services_and_vnics_list = \
+ services_and_vnics.split(';') if services_and_vnics \
+ else []
+ service_and_vnic = '{},{}'.format(service.get('local_service_id', ''),
+ vnic.get('id'))
+ if service_and_vnic in services_and_vnics_list:
+ return # we already have this tuple define
+ services_and_vnics_list.append(service_and_vnic)
+ values = {
+ 'objtype': 'instance',
+ 'objid': self.encode_special_characters(instance['id']),
+ 'host': service['host'],
+ 'services_and_vnics': ';'.join(services_and_vnics_list)
+ }
+ self.create_monitoring_for_object(instance, values)
+
+ def get_service_for_vnic(self, vnic: dict) -> dict:
+ services = self.inv.find_items({'environment': self.get_env(),
+ 'type': 'vservice',
+ 'network': vnic.get('network', '')})
+ if not services:
+ return {}
+ dhcp = next(s for s in services if s.get('service_type') == 'dhcp')
+ if dhcp:
+ return dhcp # If we have both DHCP and router, return the DHCP
+ return services[0] # currently only DHCP and router services
from monitoring.setup.monitoring_handler import MonitoringHandler
from monitoring.setup.monitoring_host import MonitoringHost
+from monitoring.setup.monitoring_instance import MonitoringInstance
from monitoring.setup.monitoring_link_vnic_vconnector \
import MonitoringLinkVnicVconnector
from monitoring.setup.monitoring_pnic import MonitoringPnic
from monitoring.setup.monitoring_otep import MonitoringOtep
from monitoring.setup.monitoring_vedge import MonitoringVedge
from monitoring.setup.monitoring_vnic import MonitoringVnic
+from monitoring.setup.monitoring_vconnector import MonitoringVconnector
from monitoring.setup.monitoring_vservice import MonitoringVservice
"otep": MonitoringOtep(env),
"vedge": MonitoringVedge(env),
"host_pnic": MonitoringPnic(env),
+ "instance": MonitoringInstance(env),
"vnic": MonitoringVnic(env),
+ "vconnector": MonitoringVconnector(env),
"vservice": MonitoringVservice(env),
"vnic-vconnector": MonitoringLinkVnicVconnector(env)}
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from monitoring.setup.monitoring_simple_object import MonitoringSimpleObject
+
+
+class MonitoringVconnector(MonitoringSimpleObject):
+
+ # add monitoring setup for remote host
+ def create_setup(self, o):
+ type = 'vconnector'
+ env_config = self.configuration.get_env_config()
+ vpp_or_ovs = 'vpp' if 'VPP' in env_config['mechanism_drivers'] \
+ else 'ovs'
+ type_str = '{}_{}'.format(type, vpp_or_ovs)
+ self.setup(type, o, values={'check_type': type_str,
+ 'name': o['name']})
+
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+import os.path
+from pkg_resources import parse_version
+
+from monitoring.setup.monitoring_handler import MonitoringHandler
+from utils.inventory_mgr import InventoryMgr
+
+
+class SensuClientInstaller(MonitoringHandler):
+
+ UBUNTU = 'ubuntu'
+ CENTOS = 'centos'
+
+ INSTALL_CMD = {
+ UBUNTU: 'dpkg -i {}',
+ CENTOS: 'rpm -i {}'
+ }
+ PERMISSIONS_CMD = {
+ UBUNTU: '',
+ CENTOS: 'usermod -aG wheel sensu'
+ }
+ SUDOERS_FILE = '/etc/sudoers'
+
+ available_downloads = {}
+
+ def __init__(self, env: str, host_id: str):
+ super().__init__(env)
+ self.cli_ssh = self.get_ssh(host_id)
+ self.inv = InventoryMgr()
+ self.host = self.inv.get_by_id(env, host_id)
+ self.server = self.env_monitoring_config.get('server_ip')
+ self.server_cli_ssh = self.get_ssh(self.server)
+ self.ubuntu_dist = None
+ self.required_package = None
+
+ def install(self):
+ pkg_to_install = self.get_pkg_to_install()
+ if not pkg_to_install:
+ return
+ try:
+ self.fetch_package(pkg_to_install)
+ self.install_package(pkg_to_install)
+ self.set_permissions()
+ except SystemError as e:
+ self.log.error('Sensu install on host {} failed: {}'
+ .format(self.host, str(e)))
+ return
+
+ @staticmethod
+ def get_attr_from_output(output_lines: list, attr: str) -> str:
+ matches = [l for l in output_lines if l.startswith(attr)]
+ if not matches:
+ return ''
+ line = matches[0]
+ return SensuClientInstaller.get_attr_from_output_line(line)
+
+ @staticmethod
+ def get_attr_from_output_line(output_line: str):
+ val = output_line[output_line.index(':')+1:].strip()
+ return val
+
+ INSTALLED = 'Installed: '
+ CANDIDATE = 'Candidate: '
+ SENSU_DIR = '/opt/sensu'
+ SENSU_PKG_DIR = '/etc/sensu/pkg'
+ SENSU_PKG_DIR_LOCAL = '/tmp/sensu_pkg'
+ SENSU_VERSION_FILE = '/opt/sensu/version-manifest.txt'
+
+ def find_available_downloads(self):
+ ls_output = self.server_cli_ssh.exec('ls -R {}'
+ .format(self.SENSU_PKG_DIR))
+ ls_lines = ls_output.splitlines()
+ last_target_dir = None
+ for line in ls_lines:
+ if line[-4:] in ['/32:', '/64:']:
+ last_target_dir = line.replace(self.SENSU_PKG_DIR, '')
+ continue
+ elif last_target_dir:
+ target_dir = last_target_dir.strip(os.path.sep).strip(':')
+ self.available_downloads[target_dir] = line
+ last_target_dir = None
+ else:
+ last_target_dir = None
+
+ def find_available_package(self, os_details: dict):
+ if not self.available_downloads:
+ self.find_available_downloads()
+ distribution = os_details['ID']
+ version = os_details['version'].split()[-2].lower()
+ arch = os_details['architecure'][-2:]
+ download_dir = os.path.join(distribution, version, arch)
+ download_file = self.available_downloads.get(download_dir)
+ full_path = '' if not download_file \
+ else os.path.join(self.SENSU_PKG_DIR, download_dir, download_file)
+ return download_file, full_path
+
+ @staticmethod
+ def find_available_version(download_file: str) -> str:
+ ver = download_file.replace('sensu', '').strip('-_')
+ ver = ver[:ver.index('-')]
+ return ver
+
+ def get_pkg_to_install(self) -> str:
+ if self.provision == self.provision_levels['none']:
+ return ''
+ if not self.host:
+ return ''
+ supported_os = [self.UBUNTU, self.CENTOS]
+ distribution = self.host['OS']['ID']
+ if distribution not in [self.UBUNTU, self.CENTOS]:
+ self.log.error('Sensu client auto-install only supported for: {}'
+ .format(', '.join(supported_os)))
+ return ''
+ cmd = 'if [ -d {} ]; then head -1 {} | sed "s/sensu //"; fi' \
+ .format(self.SENSU_DIR, self.SENSU_VERSION_FILE)
+ installed_version = self.cli_ssh.exec(cmd).strip()
+ os_details = self.host['OS']
+ available_pkg, pkg_path = self.find_available_package(os_details)
+ available_version = self.find_available_version(available_pkg)
+ if parse_version(available_version) <= parse_version(installed_version):
+ return ''
+ return pkg_path
+
+ def get_local_path(self, pkg_to_install: str):
+ return os.path.join(self.SENSU_PKG_DIR_LOCAL,
+ os.path.basename(pkg_to_install))
+
+ def fetch_package(self, pkg_to_install: str):
+ self.make_directory(self.SENSU_PKG_DIR_LOCAL)
+ self.get_file(self.server, pkg_to_install,
+ self.get_local_path(pkg_to_install))
+ local_path = self.get_local_path(pkg_to_install)
+ self.copy_to_remote_host(self.host['host'],
+ local_path=local_path,
+ remote_path=local_path)
+
+ def install_package(self, pkg_to_install):
+ local_path = self.get_local_path(pkg_to_install)
+ install_cmd = self.INSTALL_CMD[self.host['OS']['ID']]
+ self.cli_ssh.exec(install_cmd.format(local_path))
+
+ def set_permissions(self):
+ cmd = self.PERMISSIONS_CMD[self.host['OS']['ID']]
+ if cmd:
+ self.cli_ssh.exec(cmd)
+ # add to sudoers file
+ sudoer_permission = 'sensu ALL=(ALL) NOPASSWD: ALL'
+ sudoer_cmd = 'grep --silent -w sensu {} || echo "{}" >> {}'\
+ .format(self.SUDOERS_FILE, sudoer_permission, self.SUDOERS_FILE)
+ self.cli_ssh.exec(sudoer_cmd)
class TestCliqueTypes(TestBase):
- def test_get_clique_types_list_without_env_name(self):
- self.validate_get_request(clique_types.URL,
- params={},
- expected_code=base.BAD_REQUEST_CODE)
+ @patch(base.RESPONDER_BASE_READ)
+ def test_get_all_clique_types_list(self, read):
+ self.validate_get_request(
+ clique_types.URL,
+ params={},
+ mocks={
+ read: clique_types.CLIQUE_TYPES
+ },
+ expected_code=base.SUCCESSFUL_CODE,
+ expected_response=clique_types.CLIQUE_TYPES_RESPONSE
+ )
def test_get_clique_types_with_invalid_filter(self):
self.validate_get_request(clique_types.URL,
expected_code=base.SUCCESSFUL_CODE
)
+ def test_get_clique_type_with_insufficient_configuration(self):
+ self.validate_get_request(
+ clique_types.URL,
+ params={
+ "distribution_version": base.CORRECT_DIST_VER,
+ },
+ expected_code=base.BAD_REQUEST_CODE
+ )
+
+ @patch(base.RESPONDER_BASE_READ)
+ def test_get_clique_type_with_correct_configuration(self, read):
+ self.validate_get_request(
+ clique_types.URL,
+ params=clique_types.TEST_CONFIGURATION,
+ mocks={
+ read: clique_types.CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION
+ },
+ expected_response=clique_types.
+ CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION_RESPONSE,
+ expected_code=base.SUCCESSFUL_CODE
+ )
+
def test_get_clique_types_list_with_wrong_focal_point_type(self):
self.validate_get_request(clique_types.URL,
params={
body=json.dumps(clique_types.NON_DICT_CLIQUE_TYPE),
expected_code=base.BAD_REQUEST_CODE)
- def test_post_clique_type_without_env_name(self):
+ @patch(base.RESPONDER_BASE_CHECK_ENVIRONMENT_NAME)
+ def test_post_clique_type_with_reserved_env_name(self, check_env_name):
+ self.validate_post_request(
+ clique_types.URL,
+ mocks={
+ check_env_name: True
+ },
+ body=json.dumps(clique_types.CLIQUE_TYPE_WITH_RESERVED_NAME),
+ expected_code=base.BAD_REQUEST_CODE
+ )
+
+ def test_post_clique_type_without_env_name_and_configuration(self):
+ self.validate_post_request(
+ clique_types.URL,
+ body=json.dumps(clique_types.CLIQUE_TYPE_WITHOUT_ENV_NAME_AND_CONF),
+ expected_code=base.BAD_REQUEST_CODE
+ )
+
+ def test_post_clique_type_with_both_env_name_and_configuration(self):
+ self.validate_post_request(
+ clique_types.URL,
+ body=json.dumps(
+ clique_types.CLIQUE_TYPE_WITH_BOTH_ENV_AND_CONF),
+ expected_code=base.BAD_REQUEST_CODE
+ )
+
+ @patch(base.RESPONDER_BASE_CHECK_ENVIRONMENT_NAME)
+ def test_post_clique_type_with_insufficient_configuration(self, check_env_name):
+ self.validate_post_request(
+ clique_types.URL,
+ mocks={
+ check_env_name: True
+ },
+ body=json.dumps(clique_types.CLIQUE_TYPE_WITH_INSUFFICIENT_CONF),
+ expected_code=base.BAD_REQUEST_CODE
+ )
+
+ @patch(base.RESPONDER_BASE_READ)
+ def test_post_clique_type_with_duplicate_configuration(self, read):
+ data = clique_types.CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION[0]
+ resp = clique_types.CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION_RESPONSE
+ test_data = self.get_updated_data(data, deleted_keys=['id'])
self.validate_post_request(clique_types.URL,
- body=json.dumps(clique_types.CLIQUE_TYPE_WITHOUT_ENVIRONMENT),
+ body=json.dumps(test_data),
+ mocks={
+ read: resp,
+ },
expected_code=base.BAD_REQUEST_CODE)
@patch(base.RESPONDER_BASE_CHECK_ENVIRONMENT_NAME)
CLIQUE_TYPE_WITH_WRONG_FOCAL_POINT_TYPE),
expected_code=base.BAD_REQUEST_CODE)
+ @patch(base.RESPONDER_BASE_READ)
+ def test_post_clique_type_with_duplicate_focal_point_type(self, read):
+ test_data = self.get_updated_data(clique_types.CLIQUE_TYPE,
+ updates={'name': 'test-name'})
+ self.validate_post_request(clique_types.URL,
+ body=json.dumps(test_data),
+ mocks={
+ read: [clique_types.CLIQUE_TYPE],
+ },
+ expected_code=base.BAD_REQUEST_CODE)
+
def test_post_clique_type_without_link_types(self):
self.validate_post_request(clique_types.URL,
body=json.dumps(
body=json.dumps(clique_types.CLIQUE_TYPE_WITHOUT_NAME),
expected_code=base.BAD_REQUEST_CODE)
+ def test_post_clique_type_with_wrong_mechanism_drivers(self):
+ self.validate_post_request(clique_types.URL,
+ body=json.dumps(clique_types.
+ CLIQUE_TYPE_WITH_WRONG_MECH_DRIVERS),
+ expected_code=base.BAD_REQUEST_CODE)
+
+ def test_post_clique_type_with_wrong_type_drivers(self):
+ self.validate_post_request(clique_types.URL,
+ body=json.dumps(clique_types.
+ CLIQUE_TYPE_WITH_WRONG_TYPE_DRIVERS),
+ expected_code=base.BAD_REQUEST_CODE)
+
@patch(base.RESPONDER_BASE_CHECK_ENVIRONMENT_NAME)
@patch(base.RESPONDER_BASE_WRITE)
def test_post_clique_type(self, write, check_environment_name):
###############################################################################
import json
+from api.responders.resource.environment_configs import EnvironmentConfigs
from test.api.responders_test.test_data import base
+from test.api.responders_test.test_data.base import CONSTANTS_BY_NAMES
from test.api.test_base import TestBase
from test.api.responders_test.test_data import environment_configs
from utils.constants import EnvironmentFeatures
def test_get_environment_configs_list(self, read):
self.validate_get_request(environment_configs.URL,
params={},
- mocks={
- read: environment_configs.ENV_CONFIGS
- },
+ mocks={read: environment_configs.ENV_CONFIGS},
expected_code=base.SUCCESSFUL_CODE,
expected_response=environment_configs.
- ENV_CONFIGS_RESPONSE
- )
+ ENV_CONFIGS_RESPONSE)
def test_get_environment_configs_list_with_invalid_filters(self):
self.validate_get_request(environment_configs.URL,
- params={
- "unknown": "unknown"
- },
+ params={"unknown": "unknown"},
expected_code=base.BAD_REQUEST_CODE)
@patch(base.RESPONDER_BASE_READ)
def test_get_environment_configs_list_with_name(self, read):
+ mocks = {read: environment_configs.ENV_CONFIGS_WITH_SPECIFIC_NAME}
self.validate_get_request(environment_configs.URL,
- params={
- "name": environment_configs.NAME
- },
- mocks={
- read: environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_NAME
- },
+ params={"name": environment_configs.NAME},
+ mocks=mocks,
expected_code=base.SUCCESSFUL_CODE,
expected_response=environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_NAME[0]
- )
+ ENV_CONFIGS_WITH_SPECIFIC_NAME[0])
@patch(base.RESPONDER_BASE_READ)
def test_get_environment_configs_list_with_unknown_name(self, read):
@patch(base.RESPONDER_BASE_READ)
def test_get_environment_configs_list_with_distribution(self, read):
+ config = environment_configs.ENV_CONFIGS_WITH_SPECIFIC_DISTRIBUTION
+ config_response = \
+ environment_configs.ENV_CONFIGS_WITH_SPECIFIC_DISTRIBUTION_RESPONSE
self.validate_get_request(environment_configs.URL,
params={
"distribution":
environment_configs.
CORRECT_DISTRIBUTION
},
- mocks={
- read: environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_DISTRIBUTION
- },
+ mocks={read: config},
expected_code=base.SUCCESSFUL_CODE,
- expected_response=environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_DISTRIBUTION_RESPONSE)
+ expected_response=config_response)
def test_get_environment_configs_list_with_wrong_mechanism_driver(self):
+ config = environment_configs.WRONG_MECHANISM_DRIVER
self.validate_get_request(environment_configs.URL,
- params={
- "mechanism_drivers":
- environment_configs.WRONG_MECHANISM_DRIVER
- },
+ params={"mechanism_drivers": config},
expected_code=base.BAD_REQUEST_CODE)
@patch(base.RESPONDER_BASE_READ)
def test_get_environment_configs_list_with_mechanism_driver(self, read):
+ mechanism = environment_configs.CORRECT_MECHANISM_DRIVER
+ config = environment_configs.ENV_CONFIGS_WITH_SPECIFIC_MECHANISM_DRIVER
+ config_response = environment_configs.\
+ ENV_CONFIGS_WITH_SPECIFIC_MECHANISM_DRIVER_RESPONSE
self.validate_get_request(environment_configs.URL,
- params={
- "mechanism_drivers":
- environment_configs.CORRECT_MECHANISM_DRIVER
- },
- mocks={
- read: environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_MECHANISM_DRIVER
- },
+ params={"mechanism_drivers": mechanism},
+ mocks={read: config},
expected_code=base.SUCCESSFUL_CODE,
- expected_response=environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_MECHANISM_DRIVER_RESPONSE
- )
+ expected_response=config_response)
def test_get_environment_configs_list_with_wrong_type_driver(self):
+ driver = environment_configs.WRONG_TYPE_DRIVER
self.validate_get_request(environment_configs.URL,
- params={
- "type_drivers":
- environment_configs.WRONG_TYPE_DRIVER
- },
+ params={"type_drivers": driver},
expected_code=base.BAD_REQUEST_CODE)
@patch(base.RESPONDER_BASE_READ)
def test_get_environment_configs_list_with_type_driver(self, read):
+ driver = environment_configs.CORRECT_TYPE_DRIVER
+ config = environment_configs.ENV_CONFIGS_WITH_SPECIFIC_TYPE_DRIVER
+ config_response = environment_configs.\
+ ENV_CONFIGS_WITH_SPECIFIC_TYPE_DRIVER_RESPONSE
self.validate_get_request(environment_configs.URL,
- params={
- "type_drivers":
- environment_configs.CORRECT_TYPE_DRIVER
- },
- mocks={
- read: environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_TYPE_DRIVER
- },
+ params={"type_drivers": driver},
+ mocks={read: config},
expected_code=base.SUCCESSFUL_CODE,
- expected_response=environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_TYPE_DRIVER_RESPONSE
+ expected_response=config_response
)
@patch(base.RESPONDER_BASE_READ)
def test_get_environment_configs_list_with_user(self, read):
+ config = environment_configs.ENV_CONFIGS_WITH_SPECIFIC_USER
+ config_response = \
+ environment_configs.ENV_CONFIGS_WITH_SPECIFIC_USER_RESPONSE
self.validate_get_request(environment_configs.URL,
- params={
- "user": environment_configs.USER
- },
- mocks={
- read: environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_USER
- },
+ params={"user": environment_configs.USER},
+ mocks={read: config},
expected_code=base.SUCCESSFUL_CODE,
- expected_response=environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_USER_RESPONSE
- )
+ expected_response=config_response)
def test_get_environment_configs_list_with_non_bool_listen(self):
self.validate_get_request(environment_configs.URL,
- params={
- "listen": environment_configs.NON_BOOL_LISTEN
- },
+ params={"listen": environment_configs.
+ NON_BOOL_LISTEN},
expected_code=base.BAD_REQUEST_CODE)
@patch(base.RESPONDER_BASE_READ)
def test_get_environment_configs_list_with_bool_listen(self, read):
+ config = environment_configs.ENV_CONFIGS_WITH_SPECIFIC_LISTEN
+ config_response = \
+ environment_configs.ENV_CONFIGS_WITH_SPECIFIC_LISTEN_RESPONSE
self.validate_get_request(environment_configs.URL,
- params={
- "listen": environment_configs.BOOL_LISTEN
- },
- mocks={
- read: environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_LISTEN
- },
+ params={"listen": environment_configs.
+ BOOL_LISTEN},
+ mocks={read: config},
expected_code=base.SUCCESSFUL_CODE,
- expected_response=environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_LISTEN_RESPONSE
- )
+ expected_response=config_response)
def test_get_environment_configs_list_with_non_bool_scanned(self):
self.validate_get_request(environment_configs.URL,
- params={
- "scanned": environment_configs.
- NON_BOOL_SCANNED
- },
+ params={"scanned": environment_configs.
+ NON_BOOL_SCANNED},
expected_code=base.BAD_REQUEST_CODE)
@patch(base.RESPONDER_BASE_READ)
def test_get_environment_configs_list_with_bool_scanned(self, read):
+ config = environment_configs.ENV_CONFIGS_WITH_SPECIFIC_SCANNED
+ config_response = \
+ environment_configs.ENV_CONFIGS_WITH_SPECIFIC_SCANNED_RESPONSE
self.validate_get_request(environment_configs.URL,
- params={
- "scanned": environment_configs.BOOL_SCANNED
- },
- mocks={
- read: environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_SCANNED
- },
+ params={"scanned": environment_configs.
+ BOOL_SCANNED},
+ mocks={read: config},
expected_code=base.SUCCESSFUL_CODE,
- expected_response=environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_SCANNED_RESPONSE
+ expected_response=config_response
)
- def test_get_environment_configs_list_with_non_bool_monitoring_setup_done(self):
+ def test_get_env_configs_list_with_non_bool_monitoring_setup_done(self):
self.validate_get_request(environment_configs.URL,
- params={
- "listen": environment_configs.
- NON_BOOL_MONITORING_SETUP_DONE
- },
+ params={"listen": environment_configs.
+ NON_BOOL_MONITORING_SETUP_DONE},
expected_code=base.BAD_REQUEST_CODE)
@patch(base.RESPONDER_BASE_READ)
- def test_get_environment_configs_list_with_bool_monitoring_setup_done(self, read):
+ def test_get_environment_configs_list_with_bool_monitoring_setup_done(self,
+ read):
+ config = environment_configs.\
+ ENV_CONFIGS_WITH_SPECIFIC_MONITORING_SETUP_DONE
+ config_response = environment_configs.\
+ ENV_CONFIGS_WITH_SPECIFIC_MONITORING_SETUP_DONE_RESPONSE
self.validate_get_request(environment_configs.URL,
- params={
- "scanned": environment_configs.
- BOOL_MONITORING_SETUP_DONE
- },
- mocks={
- read: environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_MONITORING_SETUP_DONE
- },
+ params={"scanned": environment_configs.
+ BOOL_MONITORING_SETUP_DONE},
+ mocks={read: config},
expected_code=base.SUCCESSFUL_CODE,
- expected_response=environment_configs.
- ENV_CONFIGS_WITH_SPECIFIC_MONITORING_SETUP_DONE_RESPONSE
- )
+ expected_response=config_response)
def test_get_environment_configs_list_with_non_int_page(self):
self.validate_get_request(environment_configs.URL,
- params={
- "page": base.NON_INT_PAGE
- },
+ params={"page": base.NON_INT_PAGE},
expected_code=base.BAD_REQUEST_CODE)
@patch(base.RESPONDER_BASE_READ)
def test_get_environment_configs_list_with_int_page(self, read):
+ config_response = environment_configs.ENV_CONFIGS_RESPONSE
self.validate_get_request(environment_configs.URL,
- params={
- "page": base.INT_PAGE
- },
- mocks={
- read: environment_configs.ENV_CONFIGS
- },
+ params={"page": base.INT_PAGE},
+ mocks={read: environment_configs.ENV_CONFIGS},
expected_code=base.SUCCESSFUL_CODE,
- expected_response=environment_configs.
- ENV_CONFIGS_RESPONSE
- )
+ expected_response=config_response)
def test_get_environment_configs_list_with_non_int_page_size(self):
self.validate_get_request(environment_configs.URL,
- params={
- "page_size": base.NON_INT_PAGESIZE
- },
+ params={"page_size": base.NON_INT_PAGESIZE},
expected_code=base.BAD_REQUEST_CODE)
@patch(base.RESPONDER_BASE_READ)
def test_get_environment_configs_list_with_int_page_size(self, read):
+ config_response = environment_configs.ENV_CONFIGS_RESPONSE
self.validate_get_request(environment_configs.URL,
- params={
- "page_size": base.INT_PAGESIZE
- },
- mocks={
- read: environment_configs.ENV_CONFIGS
- },
+ params={"page_size": base.INT_PAGESIZE},
+ mocks={read: environment_configs.ENV_CONFIGS},
expected_code=base.SUCCESSFUL_CODE,
- expected_response=environment_configs.
- ENV_CONFIGS_RESPONSE
- )
+ expected_response=config_response)
def test_post_environment_config_without_app_path(self):
test_data = self.get_updated_data(environment_configs.ENV_CONFIG,
expected_code=base.BAD_REQUEST_CODE)
def test_post_environment_config_with_wrong_distribution(self):
+ dist = environment_configs.WRONG_DISTRIBUTION
test_data = self.get_updated_data(environment_configs.ENV_CONFIG,
- updates={"distribution": environment_configs.WRONG_DISTRIBUTION})
+ updates={"distribution": dist})
self.validate_post_request(environment_configs.URL,
body=json.dumps(test_data),
expected_code=base.BAD_REQUEST_CODE)
expected_code=base.BAD_REQUEST_CODE)
def test_post_environment_config_with_wrong_listen(self):
+ listen_val = environment_configs.NON_BOOL_LISTEN
test_data = self.get_updated_data(environment_configs.ENV_CONFIG,
- updates={"listen": environment_configs.NON_BOOL_LISTEN})
+ updates={"listen": listen_val})
self.validate_post_request(environment_configs.URL,
body=json.dumps(test_data),
expected_code=base.BAD_REQUEST_CODE)
expected_code=base.BAD_REQUEST_CODE)
def test_post_environment_config_with_wrong_mechanism_driver(self):
+ mechanism = environment_configs.WRONG_MECHANISM_DRIVER
test_data = self.get_updated_data(environment_configs.ENV_CONFIG,
updates={
- "mechanism_drivers":
- [environment_configs.WRONG_MECHANISM_DRIVER]
+ "mechanism_drivers": [mechanism]
})
self.validate_post_request(environment_configs.URL,
body=json.dumps(test_data),
expected_code=base.BAD_REQUEST_CODE)
def test_post_environment_config_with_wrong_scanned(self):
+ scanned_val = environment_configs.NON_BOOL_SCANNED
test_data = self.get_updated_data(environment_configs.ENV_CONFIG,
- updates={
- "scanned": environment_configs.NON_BOOL_SCANNED
- })
+ updates={"scanned": scanned_val})
self.validate_post_request(environment_configs.URL,
body=json.dumps(test_data),
expected_code=base.BAD_REQUEST_CODE)
def test_post_environment_config_with_wrong_last_scanned(self):
+ scanned_val = base.WRONG_FORMAT_TIME
test_data = self.get_updated_data(environment_configs.ENV_CONFIG,
- updates={
- "last_scanned": base.WRONG_FORMAT_TIME
- })
+ updates={"last_scanned": scanned_val})
self.validate_post_request(environment_configs.URL,
body=json.dumps(test_data),
expected_code=base.BAD_REQUEST_CODE)
expected_code=base.BAD_REQUEST_CODE)
def test_post_environment_config_with_wrong_type_drivers(self):
+ driver = environment_configs.WRONG_TYPE_DRIVER
+ test_data = self.get_updated_data(environment_configs.ENV_CONFIG,
+ updates={"type_drivers": [driver]})
+ self.validate_post_request(environment_configs.URL,
+ body=json.dumps(test_data),
+ expected_code=base.BAD_REQUEST_CODE)
+
+ def test_post_environment_config_with_duplicate_configurations(self):
+ test_data = self.get_updated_data(environment_configs.ENV_CONFIG)
+ test_data["configuration"].append({
+ "name": "OpenStack"
+ })
+ self.validate_post_request(environment_configs.URL,
+ body=json.dumps(test_data),
+ expected_code=base.BAD_REQUEST_CODE)
+
+ def test_post_environment_config_with_empty_configuration(self):
+ test_data = self.get_updated_data(environment_configs.ENV_CONFIG)
+ test_data["configuration"].append({})
+ self.validate_post_request(environment_configs.URL,
+ body=json.dumps(test_data),
+ expected_code=base.BAD_REQUEST_CODE)
+
+ def test_post_environment_config_with_unknown_configuration(self):
+ test_data = self.get_updated_data(environment_configs.ENV_CONFIG)
+ test_data["configuration"].append({
+ "name": "Unknown configuration",
+ })
+ self.validate_post_request(environment_configs.URL,
+ body=json.dumps(test_data),
+ expected_code=base.BAD_REQUEST_CODE)
+
+ def test_post_environment_config_without_required_configurations(self):
+ for env_type in CONSTANTS_BY_NAMES["environment_types"]:
+ required_conf_list = (
+ EnvironmentConfigs.REQUIRED_CONFIGURATIONS_NAMES.get(env_type,
+ [])
+ )
+ if required_conf_list:
+ test_data = \
+ self.get_updated_data(environment_configs.ENV_CONFIG)
+ test_data['environment_type'] = env_type
+ test_data['configuration'] = [
+ c
+ for c in test_data['configuration']
+ if c['name'] != required_conf_list[0]
+ ]
+
+ self.validate_post_request(environment_configs.URL,
+ body=json.dumps(test_data),
+ expected_code=base.BAD_REQUEST_CODE)
+
+ def test_post_environment_config_with_incomplete_configuration(self):
test_data = self.get_updated_data(environment_configs.ENV_CONFIG,
updates={
- "type_drivers": [environment_configs.WRONG_TYPE_DRIVER]
+ "configuration": [{
+ "host": "10.56.20.239",
+ "name": "mysql",
+ "user": "root"
+ }, {
+ "name": "OpenStack",
+ "host": "10.56.20.239",
+ }, {
+ "host": "10.56.20.239",
+ "name": "CLI",
+ "user": "root"
+ }]
})
self.validate_post_request(environment_configs.URL,
body=json.dumps(test_data),
expected_code=base.BAD_REQUEST_CODE)
- def mock_validate_env_config_with_supported_envs(self, scanning,
- monitoring, listening):
+ @staticmethod
+ def mock_validate_env_config_with_supported_envs(scanning, monitoring,
+ listening):
InventoryMgr.is_feature_supported_in_env = \
lambda self, matches, feature: {
EnvironmentFeatures.SCANNING: scanning,
@patch(base.RESPONDER_BASE_WRITE)
def test_post_environment_config(self, write):
self.mock_validate_env_config_with_supported_envs(True, True, True)
+ post_body = json.dumps(environment_configs.ENV_CONFIG)
self.validate_post_request(environment_configs.URL,
mocks={
write: None
},
- body=json.dumps(environment_configs.ENV_CONFIG),
+ body=post_body,
expected_code=base.CREATED_CODE)
def test_post_unsupported_environment_config(self):
"listening": False
}
]
+ mock_validate = self.mock_validate_env_config_with_supported_envs
+ config = environment_configs.ENV_CONFIG
for test_case in test_cases:
- self.mock_validate_env_config_with_supported_envs(test_case["scanning"],
- test_case["monitoring"],
- test_case["listening"])
+ mock_validate(test_case["scanning"], test_case["monitoring"],
+ test_case["listening"])
self.validate_post_request(environment_configs.URL,
- body=json.dumps(environment_configs.ENV_CONFIG),
+ body=json.dumps(config),
expected_code=base.BAD_REQUEST_CODE)
CREATED_CODE = "201"
ENV_NAME = "Mirantis-Liberty-API"
-UNKNOWN_ENV = "Unkown-Environment"
+UNKNOWN_ENV = "Unknown-Environment"
NON_INT_PAGE = 1.4
INT_PAGE = 1
NON_INT_PAGESIZE = 2.4
INT_PAGESIZE = 2
WRONG_LINK_TYPE = "instance-host"
-CORRECT_LINK_TYPE= "instance-vnic"
+CORRECT_LINK_TYPE = "instance-vnic"
WRONG_LINK_STATE = "wrong"
CORRECT_LINK_STATE = "up"
CORRECT_TYPE_DRIVER = "local"
WRONG_MECHANISM_DRIVER = "wrong-mechanism-dirver"
-CORRECT_MECHANISM_DRIVER = "ovs"
+CORRECT_MECHANISM_DRIVER = "OVS"
WRONG_LOG_LEVEL = "wrong-log-level"
CORRECT_LOG_LEVEL = "critical"
CONSTANTS_BY_NAMES = {
"link_types": [
"instance-vnic",
- "otep-vconnector",
- "otep-host_pnic",
+ "vnic-instance",
+ "vnic-vconnector",
+ "vconnector-vnic",
+ "vconnector-vedge",
+ "vedge-vconnector",
+ "vedge-host_pnic",
+ "host_pnic-vedge",
"host_pnic-network",
+ "network-host_pnic",
"vedge-otep",
- "vnic-vconnector",
+ "otep-vedge",
+ "otep-vconnector",
+ "vconnector-otep",
+ "otep-host_pnic",
+ "host_pnic-otep",
"vconnector-host_pnic",
- "vconnector-vedge",
+ "host_pnic-vconnector",
"vnic-vedge",
- "vedge-host_pnic",
- "vservice-vnic"
+ "vedge-vnic",
+ "vservice-vnic",
+ "vnic-vservice",
+ "switch_pnic-host_pnic",
+ "host_pnic-switch_pnic",
+ "switch_pnic-switch_pnic",
+ "switch_pnic-switch",
+ "switch-switch_pnic"
],
"link_states": [
"up",
"flat"
],
"mechanism_drivers": [
- "ovs",
- "vpp",
- "LinuxBridge",
+ "OVS",
+ "VPP",
+ "LXB",
"Arista",
"Nexus"
],
"Mirantis",
"RDO"
],
+ "distribution_versions": [
+ "8.0",
+ "9.0"
+ ],
"environment_operational_status": [
"stopped",
"running",
],
"environment_monitoring_types": [
"Sensu"
+ ],
+ "scans_statuses": [
+ "draft",
+ "pending",
+ "running",
+ "completed",
+ "completed_with_errors",
+ "failed",
+ "aborted"
+ ],
+ "configuration_targets": [
+ "AMQP",
+ "CLI",
+ "ACI",
+ "mysql",
+ "OpenStack",
+ "Monitoring",
+ "Kubernetes"
+ ],
+ "environment_types": [
+ "OpenStack",
+ "Kubernetes",
+ "VMware",
+ "Bare-metal"
]
}
RESPONDER_BASE_PATH = "api.responders.responder_base.ResponderBase"
RESPONDER_BASE_GET_OBJECTS_LIST = RESPONDER_BASE_PATH + ".get_objects_list"
RESPONDER_BASE_GET_OBJECT_BY_ID = RESPONDER_BASE_PATH + ".get_object_by_id"
-RESPONDER_BASE_CHECK_ENVIRONMENT_NAME = RESPONDER_BASE_PATH + ".check_environment_name"
+RESPONDER_BASE_CHECK_ENVIRONMENT_NAME = \
+ RESPONDER_BASE_PATH + ".check_environment_name"
RESPONDER_BASE_READ = RESPONDER_BASE_PATH + ".read"
RESPONDER_BASE_WRITE = RESPONDER_BASE_PATH + ".write"
RESPONDER_BASE_AGGREGATE = RESPONDER_BASE_PATH + ".aggregate"
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
from test.api.responders_test.test_data import base
-
+from test.api.responders_test.test_data.base import WRONG_MECHANISM_DRIVER, \
+ CORRECT_MECHANISM_DRIVER, CORRECT_TYPE_DRIVER, WRONG_TYPE_DRIVER, \
+ CORRECT_DISTRIBUTION, CORRECT_DIST_VER
URL = "/clique_types"
WRONG_ID = base.WRONG_OBJECT_ID
NONEXISTENT_ID = "58ca73ae3a8a836d10ff3b44"
CORRECT_ID = base.CORRECT_OBJECT_ID
+SAMPLE_IDS = ['58ca73ae3a8a836d10ff3b80', '58ca73ae3a8a836d10ff3b81']
+
+RESERVED_ENV_NAME = 'ANY'
WRONG_FOCAL_POINT_TYPE = base.WRONG_OBJECT_TYPE
CORRECT_FOCAL_POINT_POINT_TYPE = base.CORRECT_OBJECT_TYPE
NONEXISTENT_LINK_TYPE = "otep-host_pnic"
CORRECT_LINK_TYPE = base.CORRECT_LINK_TYPE
+CLIQUE_TYPE = {
+ "environment": "Mirantis-Liberty-API",
+ "name": "instance_vconnector_clique",
+ "link_types": [
+ "instance-vnic",
+ "vnic-vconnector"
+ ],
+ "focal_point_type": "instance"
+}
+
+TEST_CONFIGURATION = {
+ "distribution": CORRECT_DISTRIBUTION,
+ "distribution_version": CORRECT_DIST_VER,
+ "mechanism_drivers": CORRECT_MECHANISM_DRIVER,
+ "type_drivers": CORRECT_TYPE_DRIVER
+}
+
+
+def get_payload(update: dict = None, delete: list = None):
+ payload = CLIQUE_TYPE.copy()
+ if update:
+ payload.update(update)
+ if delete:
+ for k in delete:
+ del payload[k]
+ return payload
+
+
CLIQUE_TYPES_WITH_SPECIFIC_ID = [
- {
- "environment": "Mirantis-Liberty-API",
- "focal_point_type": "host_pnic",
- "id": CORRECT_ID
- }
+ get_payload(update={'id': CORRECT_ID})
+]
+
+CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION = [
+ get_payload(update={'id': SAMPLE_IDS[0],
+ **TEST_CONFIGURATION},
+ delete=['environment'])
]
+CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION_RESPONSE = {
+ "clique_types": CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION
+}
+
CLIQUE_TYPES_WITH_SPECIFIC_FOCAL_POINT_TYPE = [
- {
- "environment": "Mirantis-Liberty-API",
- "focal_point_type": CORRECT_FOCAL_POINT_POINT_TYPE,
- "id": "58ca73ae3a8a836d10ff3b80"
- },
- {
- "environment": "Mirantis-Liberty-API",
- "focal_point_type": CORRECT_FOCAL_POINT_POINT_TYPE,
- "id": "58ca73ae3a8a836d10ff3b81"
- }
+ get_payload(update={'id': _id,
+ 'focal_point_type': CORRECT_FOCAL_POINT_POINT_TYPE})
+ for _id in SAMPLE_IDS
]
CLIQUE_TYPES_WITH_SPECIFIC_FOCAL_POINT_TYPE_RESPONSE = {
}
CLIQUE_TYPES_WITH_SPECIFIC_LINK_TYPE = [
- {
- "environment": "Mirantis-Liberty-API",
- "link_types": [
- CORRECT_LINK_TYPE
- ],
- "id": "58ca73ae3a8a836d10ff3b80"
- },
- {
- "environment": "Mirantis-Liberty-API",
- "link_types": [
- CORRECT_LINK_TYPE
- ],
- "id": "58ca73ae3a8a836d10ff3b81"
- }
+ get_payload(update={'id': _id,
+ 'link_types': [CORRECT_LINK_TYPE]})
+ for _id in SAMPLE_IDS
]
CLIQUE_TYPES_WITH_SPECIFIC_LINK_TYPE_RESPONSE = {
}
CLIQUE_TYPES = [
- {
- "environment": "Mirantis-Liberty-API",
- "focal_point_type": "vnic",
- "id": "58ca73ae3a8a836d10ff3b80"
- },
- {
- "environment": "Mirantis-Liberty-API",
- "focal_point_type": "vnic",
- "id": "58ca73ae3a8a836d10ff3b81"
- }
+ get_payload(update={'id': _id}) for _id in SAMPLE_IDS
]
CLIQUE_TYPES_RESPONSE = {
NON_DICT_CLIQUE_TYPE = base.NON_DICT_OBJ
-CLIQUE_TYPE_WITHOUT_ENVIRONMENT = {
- "name": "instance_vconnector_clique",
- "link_types": [
- "instance-vnic",
- "vnic-vconnector"
- ],
- "focal_point_type": "instance"
-}
+CLIQUE_TYPE_WITH_RESERVED_NAME = get_payload(
+ update={'environment': RESERVED_ENV_NAME}
+)
-CLIQUE_TYPE_WITH_UNKNOWN_ENVIRONMENT = {
- "environment": base.UNKNOWN_ENV,
- "id": "589a3969761b0555a3ef6093",
- "name": "instance_vconnector_clique",
- "link_types": [
- "instance-vnic",
- "vnic-vconnector"
- ],
- "focal_point_type": "instance"
-}
+CLIQUE_TYPE_WITHOUT_ENV_NAME_AND_CONF = get_payload(
+ delete=['environment']
+)
-CLIQUE_TYPE_WITHOUT_FOCAL_POINT_TYPE = {
- "environment": "Mirantis-Liberty-API",
- "name": "instance_vconnector_clique",
- "link_types": [
- "instance-vnic",
- "vnic-vconnector"
- ]
-}
+CLIQUE_TYPE_WITH_BOTH_ENV_AND_CONF = get_payload(
+ update=TEST_CONFIGURATION
+)
-CLIQUE_TYPE_WITH_WRONG_FOCAL_POINT_TYPE = {
- "environment": "Mirantis-Liberty-API",
- "name": "instance_vconnector_clique",
- "link_types": [
- "instance-vnic",
- "vnic-vconnector"
- ],
- "focal_point_type": WRONG_FOCAL_POINT_TYPE
-}
+CLIQUE_TYPE_WITH_INSUFFICIENT_CONF = get_payload(
+ update={'distribution_version': CORRECT_DIST_VER}
+)
-CLIQUE_TYPE_WITHOUT_LINK_TYPES = {
- "environment": "Mirantis-Liberty-API",
- "name": "instance_vconnector_clique",
- "focal_point_type": "instance"
-}
+CLIQUE_TYPE_WITH_UNKNOWN_ENVIRONMENT = get_payload(
+ update={'environment': base.UNKNOWN_ENV}
+)
-CLIQUE_TYPE_WITH_NON_LIST_LINK_TYPES = {
- "environment": "Mirantis-Liberty-API",
- "name": "instance_vconnector_clique",
- "link_types": "instance-vnic",
- "focal_point_type": "instance"
-}
+CLIQUE_TYPE_WITHOUT_FOCAL_POINT_TYPE = get_payload(delete=['focal_point_type'])
-CLIQUE_TYPE_WITH_WRONG_LINK_TYPE = {
- "environment": "Mirantis-Liberty-API",
- "name": "instance_vconnector_clique",
- "link_types": [
- WRONG_LINK_TYPE,
- "vnic-vconnector"
- ],
- "focal_point_type": "instance"
-}
+CLIQUE_TYPE_WITH_WRONG_FOCAL_POINT_TYPE = get_payload(
+ update={'focal_point_type': WRONG_FOCAL_POINT_TYPE}
+)
-CLIQUE_TYPE_WITHOUT_NAME = {
- "environment": "Mirantis-Liberty-API",
- "link_types": [
- "instance-vnic",
- "vnic-vconnector",
- ],
- "focal_point_type": "instance"
-}
+CLIQUE_TYPE_WITHOUT_LINK_TYPES = get_payload(delete=['link_types'])
-CLIQUE_TYPE = {
- "environment": "Mirantis-Liberty-API",
- "name": "instance_vconnector_clique",
- "link_types": [
- "instance-vnic",
- "vnic-vconnector"
- ],
- "focal_point_type": "instance"
-}
+CLIQUE_TYPE_WITH_NON_LIST_LINK_TYPES = get_payload(
+ update={'link_types': "instance-vnic"}
+)
+
+CLIQUE_TYPE_WITH_WRONG_LINK_TYPE = get_payload(
+ update={'link_types': [WRONG_LINK_TYPE, "vnic-vconnector"]}
+)
+
+CLIQUE_TYPE_WITHOUT_NAME = get_payload(delete=['name'])
+
+CLIQUE_TYPE_WITH_WRONG_MECH_DRIVERS = get_payload(
+ update={'mechanism_drivers': WRONG_MECHANISM_DRIVER}
+)
+
+CLIQUE_TYPE_WITH_WRONG_TYPE_DRIVERS = get_payload(
+ update={'type_drivers': WRONG_TYPE_DRIVER}
+)
\ No newline at end of file
"provision": "None",
"env_type": "development",
"name": "Monitoring",
+ "install_monitoring_client": True,
"api_port": "4567",
"rabbitmq_port": "5671",
"rabbitmq_pass": "sensuaccess",
"last_scanned": "2017-03-16T11:14:54Z",
"listen": True,
"mechanism_drivers": [
- "ovs"
+ "OVS"
],
"name": "Mirantis-Liberty",
"operational": "running",
"scanned": True,
"type": "environment",
"type_drivers": "vxlan",
- "user": "WS7j8oTbWPf3LbNne"
+ "user": "WS7j8oTbWPf3LbNne",
+ "environment_type": "OpenStack"
}
self.original_auth_method = AuthenticationMiddleware.process_request
AuthenticationMiddleware.process_request = mock_auth_method
- ResponderBase.get_constants_by_name = MagicMock(side_effect=
- lambda name: base.CONSTANTS_BY_NAMES[name])
+ ResponderBase.get_constants_by_name = MagicMock(
+ side_effect=lambda name: base.CONSTANTS_BY_NAMES[name]
+ )
+
# mock mongo access
MongoAccess.mongo_connect = MagicMock()
MongoAccess.db = MagicMock()
log_level = 'debug'
self.app = App(log_level=log_level).get_app()
- def validate_get_request(self, url, params={}, headers=None, mocks={},
- side_effects={},
+ def validate_get_request(self, url, params=None, headers=None, mocks=None,
+ side_effects=None,
expected_code=base.SUCCESSFUL_CODE,
expected_response=None):
self.validate_request("GET", url, params, headers, "",
def validate_request(self, action, url, params, headers, body,
mocks, side_effects, expected_code,
expected_response):
- for mock_method, mock_data in mocks.items():
- mock_method.return_value = mock_data
+ if mocks:
+ for mock_method, mock_data in mocks.items():
+ mock_method.return_value = mock_data
- for mock_method, side_effect in side_effects.items():
- mock_method.side_effect = side_effect
+ if side_effects:
+ for mock_method, side_effect in side_effects.items():
+ mock_method.side_effect = side_effect
result = self.simulate_request(action, url, params=params, headers=headers, body=body)
self.assertEqual(result.status, expected_code)
if expected_response:
self.assertEqual(result.json, expected_response)
- def validate_post_request(self, url, headers={}, body="", mocks={},
- side_effects={},
+ def validate_post_request(self, url, headers=None, body="", mocks=None,
+ side_effects=None,
expected_code=base.CREATED_CODE, expected_response=None):
self.validate_request("POST", url, {}, headers, body, mocks, side_effects,
expected_code, expected_response)
- def validate_delete_request(self, url, params={}, headers={}, mocks={},
- side_effects={},
+ def validate_delete_request(self, url, params=None, headers=None, mocks=None,
+ side_effects=None,
expected_code=base.SUCCESSFUL_CODE, expected_response=None):
self.validate_request("DELETE", url, params, headers, "",
mocks, side_effects,
# which accompanies this distribution, and is available at #
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
-from unittest.mock import MagicMock, Mock
-
+import copy
import requests
+from unittest.mock import MagicMock, Mock
from discover.fetchers.api.api_access import ApiAccess
from test.fetch.api_fetch.test_data.api_access import *
def test_parse_illegal_time(self):
time = self.api_access.parse_time(ILLEGAL_TIME)
- self.assertEqual(time, None, "Can't get None when the time format is wrong")
+ self.assertEqual(time, None,
+ "Can't get None when the time format is wrong")
def test_get_existing_token(self):
self.api_access.tokens = VALID_TOKENS
token = self.api_access.get_existing_token(PROJECT)
- self.assertNotEqual(token, VALID_TOKENS[PROJECT], "Can't get existing token")
+ self.assertNotEqual(token, VALID_TOKENS[PROJECT],
+ "Can't get existing token")
def test_get_nonexistent_token(self):
self.api_access.tokens = EMPTY_TOKENS
token = self.api_access.get_existing_token(TEST_PROJECT)
- self.assertEqual(token, None, "Can't get None when the token doesn't " +
- "exist in tokens")
+ self.assertEqual(token, None,
+ "Can't get None when the token doesn't exist "
+ "in tokens")
def test_v2_auth(self):
self.api_access.get_existing_token = MagicMock(return_value=None)
self.response.json = Mock(return_value=CORRECT_AUTH_CONTENT)
# mock authentication info from OpenStack Api
- token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER, TEST_BODY)
+ token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER,
+ TEST_BODY)
self.assertNotEqual(token_details, None, "Can't get the token details")
def test_v2_auth_with_error_content(self):
self.api_access.get_existing_token = MagicMock(return_value=None)
self.response.json = Mock(return_value=ERROR_AUTH_CONTENT)
# authentication content from OpenStack Api will be incorrect
- token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER, TEST_BODY)
- self.assertIs(token_details, None, "Can't get None when the content is wrong")
+ token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER,
+ TEST_BODY)
+ self.assertIs(token_details, None,
+ "Can't get None when the content is wrong")
def test_v2_auth_with_error_token(self):
self.response.status_code = requests.codes.bad_request
self.response.json = Mock(return_value=ERROR_TOKEN_CONTENT)
# authentication info from OpenStack Api will not contain token info
- token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER, TEST_BODY)
+ token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER,
+ TEST_BODY)
self.assertIs(token_details, None, "Can't get None when the content " +
"doesn't contain any token info")
# the time will not be parsed
self.api_access.parse_time = MagicMock(return_value=None)
- token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER, TEST_BODY)
+ token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER,
+ TEST_BODY)
# reset original parse_time method
self.api_access.parse_time = original_method
- self.assertIs(token_details, None, "Can't get None when the time in token " +
- "can't be parsed")
+ self.assertIs(token_details, None,
+ "Can't get None when the time in token can't be parsed")
def test_v2_auth_pwd(self):
self.response.json = Mock(return_value=CORRECT_AUTH_CONTENT)
self.assertNotEqual(token, None, "Can't get token")
def test_get_url(self):
- self.response.json = Mock(return_value=GET_CONTENT)
+ get_response = copy.deepcopy(self.response)
+ get_response.status_code = requests.codes.ok
+ self.requests_get = requests.get
+ requests.get = MagicMock(return_value=get_response)
+ get_response.json = Mock(return_value=GET_CONTENT)
result = self.api_access.get_url(TEST_URL, TEST_HEADER)
# check whether it returns content message when the response is correct
self.assertNotEqual(result, None, "Can't get content when the "
"response is correct")
+ requests.get = self.requests_get
def test_get_url_with_error_response(self):
- self.response.status_code = requests.codes.bad_request
- self.response.json = Mock(return_value=None)
- self.response.text = "Bad request"
+ get_response = copy.deepcopy(self.response)
+ get_response.status_code = requests.codes.bad_request
+ get_response.text = "Bad request"
+ get_response.json = Mock(return_value=GET_CONTENT)
+ self.requests_get = requests.get
+ requests.get = MagicMock(return_value=get_response)
+
# the response will be wrong
result = self.api_access.get_url(TEST_URL, TEST_HEADER)
self.assertEqual(result, None, "Result returned" +
"when the response status is not 200")
+ requests.get = self.requests_get
def test_get_region_url(self):
region_url = self.api_access.get_region_url(REGION_NAME, SERVICE_NAME)
def test_get_region_url_without_service_endpoint(self):
# error service doesn't exist in region service endpoints
- region_url = self.api_access.get_region_url(REGION_NAME, ERROR_SERVICE_NAME)
- self.assertIs(region_url, None, "Can't get None with wrong service name")
+ region_url = self.api_access.get_region_url(REGION_NAME,
+ ERROR_SERVICE_NAME)
+ self.assertIs(region_url, None,
+ "Can't get None with wrong service name")
def test_region_url_nover(self):
- # mock return value of get_region_url, which has something starting from v2
+ # mock return value of get_region_url,
+ # which has something starting from v2
self.api_access.get_region_url = MagicMock(return_value=REGION_URL)
- region_url = self.api_access.get_region_url_nover(REGION_NAME, SERVICE_NAME)
+ region_url = self.api_access.get_region_url_nover(REGION_NAME,
+ SERVICE_NAME)
# get_region_nover will remove everything from v2
- self.assertNotIn("v2", region_url, "Can't get region url without v2 info")
+ self.assertNotIn("v2", region_url,
+ "Can't get region url without v2 info")
def test_get_service_region_endpoints(self):
region = REGIONS[REGION_NAME]
- result = self.api_access.get_service_region_endpoints(region, SERVICE_NAME)
+ result = self.api_access.get_service_region_endpoints(region,
+ SERVICE_NAME)
self.assertNotEqual(result, None, "Can't get service endpoint")
def test_get_service_region_endpoints_with_nonexistent_service(self):
region = REGIONS[REGION_NAME]
- result = self.api_access.get_service_region_endpoints(region, ERROR_SERVICE_NAME)
+ get_endpoints = self.api_access.get_service_region_endpoints
+ result = get_endpoints(region, ERROR_SERVICE_NAME)
self.assertIs(result, None, "Can't get None when the service name " +
"doesn't exist in region's services")
# which accompanies this distribution, and is available at #
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
+import copy
from unittest.mock import MagicMock
from discover.fetchers.api.api_fetch_project_hosts import ApiFetchProjectHosts
from test.fetch.test_fetch import TestFetch
"type in host_type")
def test_add_host_type_with_existent_host_type(self):
+ fetch_host_os_details = self.fetcher.fetch_host_os_details
+ self.fetcher.fetch_host_os_details = MagicMock()
# add nonexistent host type to host type
HOST_DOC["host_type"] = [NONEXISTENT_TYPE]
# try to add existing host type
self.fetcher.add_host_type(HOST_DOC, NONEXISTENT_TYPE, HOST_ZONE)
- self.assertEqual(len(HOST_DOC['host_type']), 1, "Add duplicate host type")
+ self.assertEqual(len(HOST_DOC['host_type']), 1,
+ "Add duplicate host type")
+ self.fetcher.fetch_host_os_details = fetch_host_os_details
def test_add_compute_host_type(self):
- HOST_DOC['host_type'] = []
+ doc = copy.deepcopy(HOST_DOC)
+ doc['host_type'] = []
# clear zone
- HOST_DOC['zone'] = None
+ doc['zone'] = None
# add compute host type
- self.fetcher.add_host_type(HOST_DOC, COMPUTE_TYPE, HOST_ZONE)
+ self.fetcher.add_host_type(doc, COMPUTE_TYPE, HOST_ZONE)
# for compute host type, zone information will be added
- self.assertEqual(HOST_DOC['zone'], HOST_ZONE, "Can't update zone " +
- "name for compute node")
- self.assertEqual(HOST_DOC['parent_id'], HOST_ZONE, "Can't update parent_id " +
- "for compute node")
+ self.assertEqual(doc['zone'], HOST_ZONE,
+ "Can't update zone name for compute node")
+ self.assertEqual(doc['parent_id'], HOST_ZONE,
+ "Can't update parent_id for compute node")
def test_fetch_compute_node_ip_address(self):
# mock ip address information fetched from DB
def test_get_host_details(self):
# test node have nova-conductor attribute, controller type will be added
+ fetch_host_os_details = self.fetcher.fetch_host_os_details
+ self.fetcher.fetch_host_os_details = MagicMock()
result = self.fetcher.get_host_details(AVAILABILITY_ZONE, HOST_NAME)
self.assertIn("Controller", result['host_type'], "Can't put controller type " +
"in the compute node host_type")
+ self.fetcher.fetch_host_os_details = fetch_host_os_details
def test_get_hosts_from_az(self):
+ fetch_host_os_details = self.fetcher.fetch_host_os_details
+ self.fetcher.fetch_host_os_details = MagicMock()
result = self.fetcher.get_hosts_from_az(AVAILABILITY_ZONE)
self.assertNotEqual(result, [], "Can't get hosts information from "
"availability zone")
+ self.fetcher.fetch_host_os_details = fetch_host_os_details
def test_get_for_region(self):
+ fetch_host_os_details = self.fetcher.fetch_host_os_details
+ self.fetcher.fetch_host_os_details = MagicMock()
# mock region url for nova node
self.fetcher.get_region_url = MagicMock(return_value=REGION_URL)
# mock the response from OpenStack Api
result = self.fetcher.get_for_region(self.region, TOKEN)
self.assertNotEqual(result, [], "Can't get hosts information for region")
+ self.fetcher.fetch_host_os_details = fetch_host_os_details
def test_get_for_region_without_token(self):
self.fetcher.get_region_url = MagicMock(return_value=REGION_URL)
self.assertEqual(result, [], "Can't get [] when the response is wrong")
def test_get_for_region_with_error_hypervisors_response(self):
+ fetch_host_os_details = self.fetcher.fetch_host_os_details
+ self.fetcher.fetch_host_os_details = MagicMock()
self.fetcher.get_region_url = MagicMock(return_value=REGION_URL)
# mock error hypervisors response from OpenStack Api
side_effect = [AVAILABILITY_ZONE_RESPONSE, HYPERVISORS_ERROR_RESPONSE]
result = self.fetcher.get_for_region(self.region, TOKEN)
self.assertNotEqual(result, [], "Can't get hosts information when " +
"the hypervisors response is wrong")
+ self.fetcher.fetch_host_os_details = fetch_host_os_details
def test_get(self):
original_method = self.fetcher.get_for_region
result = self.fetcher.get(PROJECT_NAME)
self.assertEqual(result, [], "Can't get [] when the token is invalid")
+ def test_fetch_host_os_details(self):
+ original_method = self.fetcher.run
+ self.fetcher.run = MagicMock(return_value=OS_DETAILS_INPUT)
+ doc = {'host': 'host1'}
+ self.fetcher.fetch_host_os_details(doc)
+ self.assertEqual(doc.get('OS', {}), OS_DETAILS)
+ self.fetcher.run = original_method
+
+
def tearDown(self):
super().tearDown()
ApiFetchProjectHosts.v2_auth_pwd = self._v2_auth_pwd
"zone": "osdna-zone"
}
]
+
+OS_DETAILS_INPUT = """
+NAME="Ubuntu"
+VERSION="16.04 LTS (Xenial Xerus)"
+ID=ubuntu
+ID_LIKE=debian
+PRETTY_NAME="Ubuntu 16.04 LTS"
+VERSION_ID="16.04"
+HOME_URL="http://www.ubuntu.com/"
+SUPPORT_URL="http://help.ubuntu.com/"
+BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"
+UBUNTU_CODENAME=xenial
+ARCHITECURE=x86_64
+"""
+OS_DETAILS = {
+ 'name': 'Ubuntu',
+ 'version': '16.04 LTS (Xenial Xerus)',
+ 'ID': 'ubuntu',
+ 'ID_LIKE': 'debian',
+ 'architecure': 'x86_64'
+}
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+ENV = 'env1'
+CLIQUE_CONSTRAINTS = [
+ {
+ 'focal_point_type': 'instance',
+ 'constraints': ['network']
+ },
+ {
+ 'focal_point_type': 'dummy1',
+ 'constraints': []
+ },
+ {
+ 'focal_point_type': 'dummy2',
+ 'constraints': ['network', 'dummy_constraint']
+ },
+ {
+ 'focal_point_type': 'dummy3',
+ 'constraints': ['dummy_constraint2']
+ }
+]
+CONSTRAINTS = ['network', 'dummy_constraint', 'dummy_constraint2']
+
+LINK_ATTRIBUTES_NONE = {}
+LINK_ATTRIBUTES_NONE_2 = {}
+LINK_ATTRIBUTES_EMPTY = {'attributes': []}
+LINK_ATTR_V1 = {'attributes': {'network': 'v1'}}
+LINK_ATTR_V1_2 = {'attributes': {'network': 'v1'}}
+LINK_ATTR_V2 = {'attributes': {'network': 'v2'}}
+LINK_ATTR_V1_AND_A2V2 = {'attributes': {'network': 'v1', 'attr2': 'v2'}}
+
+LINK_TYPE_1 = {
+ 'link_type': 'instance-vnic',
+ 'source_id': 'instance1',
+ 'target_id': 'vnic1'
+}
+LINK_TYPE_1_REVERSED = {
+ 'link_type': 'instance-vnic',
+ 'source_id': 'vnic1',
+ 'target_id': 'instance1'
+}
+LINK_TYPE_1_2 = {
+ 'link_type': 'instance-vnic',
+ 'source_id': 'instance1',
+ 'target_id': 'vnic2'
+}
+LINK_TYPE_2 = {
+ 'link_type': 'vnic-vconnector',
+ 'source_id': 'vnic1',
+ 'target_id': 'vconnector1'
+}
+LINK_TYPE_3 = {
+ 'implicit': True,
+ 'link_type': 'instance-vconnector',
+ 'source_id': 'instance1',
+ 'target_id': 'vconnector1'
+}
+LINK_TYPE_4_NET1 = {
+ 'environment': ENV,
+ 'implicit': True,
+ 'link_type': 'instance-host_pnic',
+ 'source': 'instance1_dbid',
+ 'source_id': 'instance1',
+ 'target': 'host_pnic1_dbid',
+ 'target_id': 'host_pnic1',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+LINK_TYPE_5_NET2 = {
+ 'environment': ENV,
+ 'link_type': 'host_pnic-switch',
+ 'source_id': 'host_pnic1',
+ 'target': 'switch1_dbid',
+ 'target_id': 'switch1',
+ 'host': 'host2',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID2'}
+}
+LINK_TYPE_6_NET1 = {
+ 'environment': ENV,
+ 'link_type': 'host_pnic-switch',
+ 'source': 'host_pnic1_dbid',
+ 'source_id': 'host_pnic1',
+ 'target': 'switch2_dbid',
+ 'target_id': 'switch2',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+LINK_TYPE_7_NET1 = {
+ 'environment': ENV,
+ 'implicit': True,
+ 'link_type': 'instance-switch',
+ 'source': 'instance1_dbid',
+ 'source_id': 'instance1',
+ 'target': 'switch2_dbid',
+ 'target_id': 'switch2',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+
+LINK_FULL_A2B = {
+ 'environment': ENV,
+ 'link_type': 'instance-vnic',
+ 'source': 'instance1_dbid',
+ 'source_id': 'instance1',
+ 'target': 'vnic1_dbid',
+ 'target_id': 'vnic1',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+LINK_FULL_B2C = {
+ 'environment': ENV,
+ 'link_type': 'vnic-vconnector',
+ 'source': 'vnic1_dbid',
+ 'source_id': 'vnic1',
+ 'target': 'vconnector1_dbid',
+ 'target_id': 'vconnector1',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+LINK_FULL_C2D = {
+ 'environment': ENV,
+ 'link_type': 'vconnector-vedge',
+ 'source': 'vconnector1_dbid',
+ 'source_id': 'vconnector1',
+ 'target': 'vedge1_dbid',
+ 'target_id': 'vedge1',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+LINK_FULL_D2E = {
+ 'environment': ENV,
+ 'link_type': 'vedge-otep',
+ 'source': 'vedge1_dbid',
+ 'source_id': 'vedge1',
+ 'target': 'otep1_dbid',
+ 'target_id': 'otep1',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+LINK_FULL_A2C = {
+ 'environment': ENV,
+ 'implicit': True,
+ 'link_type': 'instance-vconnector',
+ 'source': 'instance1_dbid',
+ 'source_id': 'instance1',
+ 'target': 'vconnector1_dbid',
+ 'target_id': 'vconnector1',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+LINK_FULL_B2D = {
+ 'environment': ENV,
+ 'implicit': True,
+ 'link_type': 'vnic-vedge',
+ 'source': 'vnic1_dbid',
+ 'source_id': 'vnic1',
+ 'target': 'vedge1_dbid',
+ 'target_id': 'vedge1',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+LINK_FULL_C2E = {
+ 'environment': ENV,
+ 'implicit': True,
+ 'link_type': 'vconnector-otep',
+ 'source': 'vconnector1_dbid',
+ 'source_id': 'vconnector1',
+ 'target': 'otep1_dbid',
+ 'target_id': 'otep1',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+LINK_FULL_A2D = {
+ 'environment': ENV,
+ 'implicit': True,
+ 'link_type': 'instance-vedge',
+ 'source': 'instance1_dbid',
+ 'source_id': 'instance1',
+ 'target': 'vedge1_dbid',
+ 'target_id': 'vedge1',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+LINK_FULL_B2E = {
+ 'environment': ENV,
+ 'implicit': True,
+ 'link_type': 'vnic-otep',
+ 'source': 'vnic1_dbid',
+ 'source_id': 'vnic1',
+ 'target': 'otep1_dbid',
+ 'target_id': 'otep1',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+LINK_FULL_A2E = {
+ 'environment': ENV,
+ 'implicit': True,
+ 'link_type': 'instance-otep',
+ 'source': 'instance1_dbid',
+ 'source_id': 'instance1',
+ 'target': 'otep1_dbid',
+ 'target_id': 'otep1',
+ 'host': 'host1',
+ 'link_name': '',
+ 'state': 'up',
+ 'source_label': '',
+ 'target_label': '',
+ 'link_weight': 0,
+ 'attributes': {'network': 'netID1'}
+}
+BASE_LINKS = [
+ {'pass': 0, 'link': LINK_FULL_A2B},
+ {'pass': 0, 'link': LINK_FULL_B2C},
+ {'pass': 0, 'link': LINK_FULL_C2D},
+ {'pass': 0, 'link': LINK_FULL_D2E},
+]
+IMPLICIT_LINKS = [
+ [
+ {'pass': 1, 'link': LINK_FULL_A2C},
+ {'pass': 1, 'link': LINK_FULL_B2D},
+ {'pass': 1, 'link': LINK_FULL_C2E},
+ ],
+ [
+ {'pass': 2, 'link': LINK_FULL_A2D},
+ {'pass': 2, 'link': LINK_FULL_B2E},
+ ],
+ [
+ {'pass': 3, 'link': LINK_FULL_A2E},
+ ],
+ []
+]
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+import bson
+
+from discover.link_finders.find_implicit_links import FindImplicitLinks
+from test.fetch.test_fetch import TestFetch
+from unittest.mock import MagicMock
+from test.fetch.link_finders.test_data.test_find_implicit_links import *
+
+from utils.inventory_mgr import InventoryMgr
+
+
+class TestFindImplicitLinks(TestFetch):
+
+ def setUp(self):
+ super().setUp()
+ self.configure_environment()
+ self.fetcher = FindImplicitLinks()
+ self.fetcher.set_env(ENV)
+ self.fetcher.constraint_attributes = ['network']
+ self.original_write_link = self.inv.write_link
+ self.inv.write_link = lambda x: x
+ self.original_objectid = bson.ObjectId
+ bson.ObjectId = lambda x: x
+
+ def tearDown(self):
+ super().tearDown()
+ bson.ObjectId = self.original_objectid
+ self.inv.write_link = self.original_write_link
+
+ def test_get_constraint_attributes(self):
+ original_find = InventoryMgr.find
+ InventoryMgr.find = MagicMock(return_value=CLIQUE_CONSTRAINTS)
+ constraint_types = self.fetcher.get_constraint_attributes()
+ self.assertEqual(sorted(constraint_types), sorted(CONSTRAINTS))
+ InventoryMgr.find = original_find
+
+ def test_constraints_match(self):
+ matcher = self.fetcher.constraints_match
+ self.assertTrue(matcher(LINK_ATTRIBUTES_NONE, LINK_ATTRIBUTES_NONE_2))
+ self.assertTrue(matcher(LINK_ATTRIBUTES_NONE, LINK_ATTRIBUTES_EMPTY))
+ self.assertTrue(matcher(LINK_ATTRIBUTES_NONE, LINK_ATTR_V1))
+ self.assertTrue(matcher(LINK_ATTRIBUTES_EMPTY, LINK_ATTR_V1))
+ self.assertTrue(matcher(LINK_ATTR_V1, LINK_ATTR_V1_2))
+ self.assertTrue(matcher(LINK_ATTR_V1,
+ LINK_ATTR_V1_AND_A2V2))
+ self.assertFalse(matcher(LINK_ATTR_V1, LINK_ATTR_V2))
+
+ def test_links_match(self):
+ matcher = self.fetcher.links_match
+ self.assertFalse(matcher(LINK_TYPE_1, LINK_TYPE_1_2))
+ self.assertFalse(matcher(LINK_TYPE_1, LINK_TYPE_1_REVERSED))
+ self.assertFalse(matcher(LINK_TYPE_4_NET1, LINK_TYPE_5_NET2))
+ self.assertFalse(matcher(LINK_TYPE_1_2, LINK_TYPE_2))
+ self.assertTrue(matcher(LINK_TYPE_1, LINK_TYPE_2))
+
+ def test_get_link_constraint_attributes(self):
+ getter = self.fetcher.get_link_constraint_attributes
+ self.assertEqual(getter(LINK_TYPE_1, LINK_TYPE_1_2), {})
+ self.assertEqual(getter(LINK_TYPE_1, LINK_TYPE_4_NET1),
+ LINK_TYPE_4_NET1.get('attributes'))
+ self.assertEqual(getter(LINK_TYPE_4_NET1, LINK_TYPE_1),
+ LINK_TYPE_4_NET1.get('attributes'))
+ self.assertEqual(getter(LINK_TYPE_1, LINK_TYPE_5_NET2),
+ LINK_TYPE_5_NET2.get('attributes'))
+ self.assertEqual(getter(LINK_TYPE_4_NET1, LINK_TYPE_6_NET1),
+ LINK_TYPE_4_NET1.get('attributes'))
+
+ def test_get_attr(self):
+ getter = self.fetcher.get_attr
+ self.assertIsNone(getter('host', {}, {}))
+ self.assertIsNone(getter('host', {'host': 'v1'}, {'host': 'v2'}))
+ self.assertEqual(getter('host', {'host': 'v1'}, {}), 'v1')
+ self.assertEqual(getter('host', {}, {'host': 'v2'}), 'v2')
+ self.assertEqual(getter('host', {'host': 'v1'}, {'host': 'v1'}), 'v1')
+
+ def test_add_implicit_link(self):
+ original_write_link = self.inv.write_link
+ self.inv.write_link = lambda x: x
+ original_objectid = bson.ObjectId
+ bson.ObjectId = lambda x: x
+ add_func = self.fetcher.add_implicit_link
+ self.assertEqual(add_func(LINK_TYPE_4_NET1, LINK_TYPE_6_NET1),
+ LINK_TYPE_7_NET1)
+ bson.ObjectId = original_objectid
+ self.inv.write_link = original_write_link
+
+ def test_get_transitive_closure(self):
+ self.fetcher.links = [
+ {'pass': 0, 'link': LINK_FULL_A2B},
+ {'pass': 0, 'link': LINK_FULL_B2C},
+ {'pass': 0, 'link': LINK_FULL_C2D},
+ {'pass': 0, 'link': LINK_FULL_D2E},
+ ]
+ self.fetcher.get_transitive_closure()
+ for pass_no in range(1, len(IMPLICIT_LINKS)):
+ implicit_links = [l for l in self.fetcher.links
+ if l['pass'] == pass_no]
+ self.assertEqual(implicit_links, IMPLICIT_LINKS[pass_no-1],
+ 'incorrect links for pass #{}'.format(pass_no))
"provision": "Deploy",
"env_type": "development",
"name": "Monitoring",
+ "install_monitoring_client": True,
"rabbitmq_port": "5672",
"rabbitmq_pass": "osdna",
"rabbitmq_user": "sensu",
"scanners_package": "discover",
"scanners": {}
}
+LINK_FINDERS_METADATA = {
+ "finders_package": "discover.link_finders",
+ "base_finder": "FindLinks",
+ "link_finders": [
+ "FindLinksForInstanceVnics",
+ "FindLinksForOteps",
+ "FindLinksForVconnectors",
+ "FindLinksForVedges",
+ "FindLinksForVserviceVnics",
+ "FindLinksForPnics",
+ "FindImplicitLinks"
+ ]
+}
TYPE_TO_FETCH = {
"type": "host_pnic",
'input': METADATA_SCANNER_INCORRECT_FETCHER,
'msg': 'failed to find fetcher class f1 '
'in scanner ScanAggregate type #1'
+ ' (could not import module discover.fetchers.f1.f1: '
+ 'No module named \'discover.fetchers.f1\')'
},
{
'input': METADATA_SCANNER_WITH_INCORRECT_CHILD,
from discover.scanner import Scanner
from test.scan.test_scan import TestScan
from unittest.mock import MagicMock, patch
+
+from discover.link_finders.find_links_metadata_parser \
+ import FindLinksMetadataParser
from discover.scan_metadata_parser import ScanMetadataParser
from test.scan.test_data.scanner import *
from monitoring.setup.monitoring_setup_manager import MonitoringSetupManager
def setUp(self):
super().setUp()
- ScanMetadataParser.parse_metadata_file = MagicMock(return_value=METADATA)
+ ScanMetadataParser.parse_metadata_file = \
+ MagicMock(return_value=METADATA)
+ FindLinksMetadataParser.parse_metadata_file = \
+ MagicMock(return_value=LINK_FINDERS_METADATA)
self.scanner = Scanner()
self.scanner.set_env(self.env)
MonitoringSetupManager.create_setup = MagicMock()
set -o errexit
set -o nounset
set -o pipefail
+PYTHONPATH=$PWD/app python3 -m unittest discover -s app/test/api
+PYTHONPATH=$PWD/app python3 -m unittest discover -s app/test/event_based_scan
PYTHONPATH=$PWD/app python3 -m unittest discover -s app/test/fetch
+PYTHONPATH=$PWD/app python3 -m unittest discover -s app/test/scan
+PYTHONPATH=$PWD/app python3 -m unittest discover -s app/test/utils
--- /dev/null
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+import requests
+
+from discover.configuration import Configuration
+from discover.fetcher import Fetcher
+
+
+class ApiAccessBase(Fetcher):
+
+ CONNECT_TIMEOUT = 5
+
+ def __init__(self, api_name=None, config=None):
+ super().__init__()
+ if api_name is None:
+ raise ValueError('ApiAccessBase: api_name must be defined')
+ self.config = {api_name: config} if config else Configuration()
+ self.api_config = self.config.get(api_name)
+ if self.api_config is None:
+ raise ValueError('ApiAccessBase: section "{}" missing in config'
+ .format(api_name))
+ self.host = self.api_config.get('host', '')
+ self.port = self.api_config.get('port', '')
+ if not (self.host and self.port):
+ raise ValueError('Missing definition of host or port ' +
+ 'for {} API access'
+ .format(api_name))
+
+ def get_rel_url(self, relative_url, headers):
+ req_url = self.base_url + relative_url
+ return self.get_url(req_url, headers)
+
+ def get_url(self, req_url, headers):
+ response = requests.get(req_url, headers=headers)
+ if response.status_code != requests.codes.ok:
+ # some error happened
+ if 'reason' in response:
+ msg = ', reason: {}'.format(response.reason)
+ else:
+ msg = ', response: {}'.format(response.text)
+ self.log.error('req_url: {} {}'.format(req_url, msg))
+ return None
+ ret = response.json()
+ return ret
# source_label, target_label: labels for the ends of the link (optional)
def create_link(self, env, src, source_id, target, target_id,
link_type, link_name, state, link_weight,
+ implicit=False,
source_label="", target_label="",
host=None, switch=None,
extra_attributes=None):
"link_weight": link_weight,
"source_label": source_label,
"target_label": target_label,
+ "implicit": implicit,
"attributes": extra_attributes if extra_attributes else {}
}
if host:
if not env_config:
return False
- # Workaround for mechanism_drivers field type
- mechanism_driver = env_config['mechanism_drivers'][0] \
- if isinstance(env_config['mechanism_drivers'], list) \
- else env_config['mechanism_drivers']
+ # Workarounds for mechanism_drivers and distribution_version field types
+ mechanism_driver = env_config.get('mechanism_drivers')
+ if isinstance(mechanism_driver, list):
+ mechanism_driver = mechanism_driver[0]
+ env_distribution_version = env_config.get('distribution_version')
+ if isinstance(env_distribution_version, list):
+ env_distribution_version = env_distribution_version[0]
full_env = {
- 'environment.distribution': env_config['distribution'],
- 'environment.distribution_version':
- {"$in": [env_config['distribution_version']]},
- 'environment.type_drivers': env_config['type_drivers'],
+ 'environment.distribution': env_config.get('distribution'),
+ 'environment.distribution_version': env_distribution_version,
+ 'environment.type_drivers': env_config.get('type_drivers'),
'environment.mechanism_drivers': mechanism_driver
}
return self.is_feature_supported_in_env(full_env, feature)
self.log.error("failed to find master parent " +
master_parent_id)
return False
- folder_id_path = "/".join((master_parent["id_path"], o["parent_id"]))
- folder_name_path = "/".join((master_parent["name_path"], o["parent_text"]))
+ folder_id_path = "/".join((master_parent["id_path"],
+ o["parent_id"]))
+ folder_name_path = "/".join((master_parent["name_path"],
+ o["parent_text"]))
folder = {
"environment": parent["environment"],
"parent_id": master_parent_id,
def __init__(self):
super().__init__()
- self.log_file = os.path.join(FileLogger.LOG_DIRECTORY,
- MongoAccess.LOG_FILENAME)
+ log_dir = FileLogger.LOG_DIRECTORY \
+ if os.path.isdir(FileLogger.LOG_DIRECTORY) \
+ else os.path.abspath('.')
+ self.log_file = os.path.join(log_dir, MongoAccess.LOG_FILENAME)
try:
self.log = FileLogger(self.log_file)
max_call_count_per_con = 100
timeout = 15 # timeout for exec in seconds
+ CONNECT_TIMEOUT = 5
DEFAULT_PORT = 22
pkey=k,
port=self.port if self.port is not None
else self.DEFAULT_PORT,
- password=self.pwd, timeout=30)
+ password=self.pwd,
+ timeout=self.CONNECT_TIMEOUT)
else:
port = None
try:
username=self.user,
password=self.pwd,
port=port,
- timeout=30)
+ timeout=self.CONNECT_TIMEOUT)
except paramiko.ssh_exception.AuthenticationException:
self.log.error('Failed SSH connect to host {}, port={}'
.format(self.host, port))
class_name = ''.join(name_parts)
return class_name
-
@staticmethod
def get_fully_qualified_class(class_name: str = None,
package_name: str = "discover",
module_name = ".".join(module_parts)
try:
class_module = importlib.import_module(module_name)
- except ImportError:
- raise ValueError('could not import module {}'.format(module_name))
+ except ImportError as e:
+ raise ValueError('could not import module {}: {}'
+ .format(module_name, str(e)))
clazz = getattr(class_module, class_name)
return clazz
class_name = ClassResolver.get_class_name_by_module(module_name)
if class_name in ClassResolver.instances:
return 'instance', ClassResolver.instances[class_name]
- clazz = ClassResolver.get_fully_qualified_class(class_name, package_name,
+ clazz = ClassResolver.get_fully_qualified_class(class_name,
+ package_name,
module_name)
return 'class', clazz
+++ /dev/null
-body {
- font-family: Helvetica, sans-serif;
- font-size: 16px;
-}
-
-body a {
- color: #27CCC0;
-}
-
-body a:hover {
- color: #676767;
-}
-
-.navbar-brand img {
- height: 200%;
- margin-top: -5%;
-}
-
-.navbar, h1, h2, h3, h4, h5, h6 {
- font-family: Helvetica, sans-serif;
-}
-
-.navbar-text{
- color: #676767;
-}
-
-.navbar-form.navbar-right{
- padding: 0;
-}
-
-.navbar-form .form-control{
- width: 150px;
-}
+++ /dev/null
-import sphinx_bootstrap_theme
-import os
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-# -- General configuration ------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-# needs_sphinx = '1.0'
-needs_sphinx = '1.3'
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = ['sphinxcontrib.httpdomain', 'sphinx.ext.autodoc',
- 'sphinx.ext.viewcode', 'sphinx.ext.napoleon']
-# Disable javasphinx generation until we have a solution to long build
-# times. readthedocs timesout after 902 seconds.
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
-
-# The encoding of source files.
-# source_encoding = 'utf-8-sig'
-
-# The master toctree document.
-master_doc = 'index'
-
-# General information about the project.
-project = ''
-copyright = '2017, OPNFV. Licensed under CC BY 4.0'
-author = 'Open Platform for NFV'
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-version = 'Latest'
-# The full version, including alpha/beta/rc tags.
-release = 'Latest'
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
-# today = ''
-# Else, today_fmt is used as the format for a strftime call.
-# today_fmt = '%B %d, %Y'
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = []
-
-# The reST default role (used for this markup: `text`) to use for all
-# documents.
-# default_role = None
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-# add_function_parentheses = True
-
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-# add_module_names = True
-
-# If true, sectionauthor and moduleauthor directives will be shown in the
-# output. They are ignored by default.
-# show_authors = False
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-# A list of ignored prefixes for module index sorting.
-# modindex_common_prefix = []
-
-# If true, keep warnings as "system message" paragraphs in the built documents.
-# keep_warnings = False
-
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = False
-
-
-# -- Options for HTML output ----------------------------------------------
-
-# The theme to use for HTML and HTML Help pages. See the documentation for
-# a list of builtin themes.
-html_theme = 'bootstrap'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further. For a list of options available for each theme, see the
-# documentation.
-# html_theme_options = {}
-html_theme_options = {
- 'bootswatch_theme': "journal",
- 'navbar_sidebarrel': False,
-}
-
-# Add any paths that contain custom themes here, relative to this directory.
-# html_theme_path = []
-html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()
-
-# The name for this set of Sphinx documents.
-# "<project> v<release> documentation" by default.
-# html_title = 'OpenDaylight Documentation v0.3.0'
-
-# A shorter title for the navigation bar. Default is the same as html_title.
-# html_short_title = None
-
-# The name of an image file (relative to this directory) to place at the top
-# of the sidebar.
-html_logo = '_static/opnfv-logo.png'
-
-# The name of an image file (relative to this directory) to use as a favicon of
-# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
-# pixels large.
-html_favicon = '_static/favicon.ico'
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
-# Add any extra paths that contain custom files (such as robots.txt or
-# .htaccess) here, relative to this directory. These files are copied
-# directly to the root of the documentation.
-# html_extra_path = []
-
-# If not None, a 'Last updated on:' timestamp is inserted at every page
-# bottom, using the given strftime format.
-# The empty string is equivalent to '%b %d, %Y'.
-# html_last_updated_fmt = None
-
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-# html_use_smartypants = True
-
-# Custom sidebar templates, maps document names to template names.
-# html_sidebars = {}
-
-# Additional templates that should be rendered to pages, maps page names to
-# template names.
-# html_additional_pages = {}
-
-# If false, no module index is generated.
-# html_domain_indices = True
-
-# If false, no index is generated.
-# html_use_index = True
-
-# If true, the index is split into individual pages for each letter.
-# html_split_index = False
-
-# If true, links to the reST sources are added to the pages.
-# html_show_sourcelink = True
-
-# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-# html_show_sphinx = True
-
-# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-# html_show_copyright = True
-
-# If true, an OpenSearch description file will be output, and all pages will
-# contain a <link> tag referring to it. The value of this option must be the
-# base URL from which the finished HTML is served.
-# html_use_opensearch = ''
-
-# This is the file name suffix for HTML files (e.g. ".xhtml").
-# html_file_suffix = None
-
-# Language to be used for generating the HTML full-text search index.
-# Sphinx supports the following languages:
-# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
-# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh'
-# html_search_language = 'en'
-
-# A dictionary with options for the search language support, empty by default.
-# 'ja' uses this config value.
-# 'zh' user can custom change `jieba` dictionary path.
-# html_search_options = {'type': 'default'}
-
-# The name of a javascript file (relative to the configuration directory) that
-# implements a search results scorer. If empty, the default will be used.
-# html_search_scorer = 'scorer.js'
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'OPNFV'
-
-# -- Options for LaTeX output ---------------------------------------------
-
-latex_elements = {
- # The paper size ('letterpaper' or 'a4paper').
- #'papersize': 'letterpaper',
-
- # The font size ('10pt', '11pt' or '12pt').
- #'pointsize': '10pt',
-
- # Additional stuff for the LaTeX preamble.
- #'preamble': '',
-
- # Latex figure (float) alignment
- #'figure_align': 'htbp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-# author, documentclass [howto, manual, or own class]).
-latex_documents = [
- (master_doc, 'OPNFV.tex', 'OPNFV Documentation',
- 'OPNFV Project', 'manual'),
-]
-
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-# latex_logo = None
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-# latex_use_parts = False
-
-# If true, show page references after internal links.
-# latex_show_pagerefs = False
-
-# If true, show URL addresses after external links.
-# latex_show_urls = False
-
-# Documents to append as an appendix to all manuals.
-# latex_appendices = []
-
-# If false, no module index is generated.
-# latex_domain_indices = True
-
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
- (master_doc, 'OPNFVDocs', 'OPNFV Documentation',
- [author], 1)
-]
-
-# If true, show URL addresses after external links.
-# man_show_urls = False
-
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-# dir menu entry, description, category)
-texinfo_documents = [
- (master_doc, 'OPNFVDocs', 'OPNFV Documentation',
- author, 'OPNFV', 'One line description of project.',
- 'Miscellaneous'),
-]
-
-html_sidebars = {'**': ['localtoc.html', 'relations.html'],}
-# Documents to append as an appendix to all manuals.
-# texinfo_appendices = []
-
-# If false, no module index is generated.
-# texinfo_domain_indices = True
-
-# How to display URL addresses: 'footnote', 'no', or 'inline'.
-# texinfo_show_urls = 'footnote'
-
-# If true, do not generate a @detailmenu in the "Top" node's menu.
-# texinfo_no_detailmenu = False
-
-# intersphinx_mapping =
-# {'RTD':('http://opnfvdocsdemo.readthedocs.io/projects/', None)}
+++ /dev/null
-.. This work is licensed under a Creative Commons Attribution 4.0 International License.
-.. http://creativecommons.org/licenses/by/4.0
-.. (c) OPNFV and others.
-
-=====================
-Calipso documentation
-=====================
-
-.. toctree::
- :maxdepth: 1
-
- release/index.rst
--- /dev/null
+***The following are fake stories, although providing real examples of
+real problems that are faced today by cloud providers, and showing
+possible resolutions provided by Calipso:***
+
+***Enterprise use-case story (Calipso ‘S’ release):***
+
+Moz is a website publishing and managing product, Moz provides
+reputation and popularity tracking, helps with distributions, listing,
+and ratings and provides content distributions for industry marketing.
+
+Moz considers moving their main content distribution application to be
+hosted on https://www.dreamhost.com/, which provides shared and
+dedicated IaaS and PaaS hosting based on OpenStack.
+
+As a major milestone for Moz’s due diligence for choosing Dreamhost, Moz
+acquires a shared hosting facility from Dreamhost, that is
+cost-effective and stable, it includes 4 mid-sized Web-servers, 4
+large-sized Application-servers and 2 large-sized DB servers, connected
+using several networks, with some security services.
+
+Dreamhost executives instruct their infrastructure operations department
+to make sure proper SLA and Monitoring is in-place so the due diligence
+and final production deployment of Moz’s services in the Dreamhost
+datacenter goes well and that Moz’s engineers receive excellent service
+experience.
+
+Moz received the following SLA with their current VPS contract:
+
+- 97-day money back guarantee, in case of a single service down event
+ or any dissatisfaction.
+
+- 99.5 % uptime/availability with a weekly total downtime of 30
+ minutes.
+
+- 24/7/365 on-call service with a total of 6 hours MTTR.
+
+- Full HA for all networking services.
+
+- Managed VPS using own Control Panel IaaS provisioning with overall
+ health visibility.
+
+- Scalable RAM, starts with 1GB can grow per request to 16GB from
+ within control panel.
+
+- Guaranteed usage of SSD or equivalent speeds, storage capacity from
+ 30GB to 240GB.
+
+- Backup service based on cinder-backup and Ceph’s dedicated backup
+ volumes, with restoration time below 4 hours.
+
+Dreamhost‘s operations factored all requirement and has decided to
+include real-time monitoring and analysis for the VPS for Moz.
+
+One of the tools used now for Moz environment in Dreamhost is Calipso
+for virtual networking.
+
+Here are some benefits provided by Calipso for Dreamhost operations
+during service cycles:
+
+*Reporting:*
+
+Special handling of virtual networking is in place:
+
+- Dreamhost designed a certain virtual networking setup and
+ connectivity that provides the HA and performance required by the SLA
+ and decided on several physical locations for Moz’s virtual servers
+ in different availability zones.
+
+- Scheduling of discovery has been created, Calipso takes a snapshot of
+ Moz’s environment every Sunday at midnight, reporting on connectivity
+ among all 20 servers (10 main and 10 backups) and overall health of
+ that connectivity.
+
+- Every Sunday morning at 8am, before the week’s automatic
+ snapshotting, the NOC administrator runs a manual discovery and saves
+ that snapshot, she then runs a comparison check against last week’s
+ snapshot and against initial design to find any gaps or changes that
+ might happen due to other shared services deployments, virtual
+ instances and their connectivity are analyzed and reported with
+ Calipso’s topology and health monitoring.
+
+- Reports are saved for a bi-weekly reporting sent to Moz’s networking
+ engineers.
+
+ *Change management:*
+
+ If infrastructure changes needs to happen on any virtual service
+ (routers, switches, firewalls etc.) or on any physical server or
+ physical switch the following special guidelines apply:
+
+- Run a search on Calipso for the name of the virtual service, switch
+ or host. Lookup if Moz environment is using this object (using the
+ object’s attributes).
+
+- Using Calipso’s impact analysis, fill a report stating all Moz’s
+ objects, on which host, connected to which switch that is affected by
+ the planed change.
+
+- Run clique-type scan, using the specific object as ‘focal-point’ to
+ create a dedicated topology with accompanied health report before
+ conducting the change itself, use this a *pre snapshot*.
+
+- Simulate the change, using Moz’s testing environment only, make sure
+ HA services are in places and downtime is confirmed to be in the SLA
+ boundaries.
+
+- Using all reports provided by Calipso, along with application and
+ storage reports, send a detailed change request to NOC and later to
+ the end-customer for review.
+
+- During the change, make sure HA is operational, by running the same
+ clique-type snapshotting every 10 minutes and running a comparison.
+
+- NOC, while waiting for the change to complete, looks at Calipso’s
+ dashboard focused on MOZ’s environment, monitoring results for
+ service down event (as expected), impact on other objects in the
+ service chain - the entire Calipso clique for that object (as
+ expected).
+
+- Once operations has reported back to NOC about change done, run the
+ same snapshotting again as *post snapshot* and run a comparison to
+ make sure all virtual networking are back to the ‘as designed’ stage
+ and all networking services are back.
+
+**Example snapshot taken at one stage on Calipso for the Moz virtual
+networking:**
+
+|image0|
+
+ *Troubleshooting:*
+
+ Dreamhost NOC uses Calipso dashboards for Moz’s environment for
+ their daily health-check. Troubleshooting starts in two cases:
+
+1. When a failure is detected on Calipso for any of Moz’s objects on
+ their virtual networking topologies,
+
+2. When a service case has been opened by Moz with “High Priority,
+ service down” flag.
+
+3. Networking department needs to know which virtual services are
+ connected to which ACI switches ports.
+
+ The following actions are taken, using Calipso dashboards:
+
+- Kickoff a discovery through Calipso API for all objects related to
+ Moz.
+
+- For a service request with no Calipso error detected: using Calipso’s
+ impact analysis, create all cliques for all objects as focal point.
+
+- For an error detected by Calipso: using Calipso’s impact analysis,
+ create cliques for objects with errors as focal point.
+
+- Resulted cliques are then analyzed using detailed messaging facility
+ in Calipso (looking deeply into any message generated regarding the
+ related objects).
+
+- Report with ACI ports to virtual services mappings is sent to
+ networking department for further analysis.
+
+ |image1|
+
+- If this is a failure on any physical device (host or switch) and/or
+ on any physical NIC (switch or host side), Calipso immediately points
+ this out and using the specific set of messages generated the
+ administrator can figure out the root cause (like optical failure,
+ driver, disconnect etc.).
+
+- In virtual object failures Calipso saves time pinpointing the servers
+ where erroneous objects are running, and their previous and new
+ connectivity details.
+
+- Calipso alerts on dependencies for :
+
+1. All related objects in the clique for that objects.
+
+2. Related hosts
+
+3. Related projects and networks
+
+4. Related application (\* in case Murano app has been added)
+
+- Administrators connects directly to the specific servers and now,
+ using the specific object attributes can start he’s manual
+ troubleshooting (actual fixing of the software issues is not
+ currently part of the Calipso features).
+
+- The NOC operators approves closing the service ticket only when all
+ related Calipso cliques are showing up as healthy and connectivity is
+ back to it’s original “as designed” stage, using Calipso older
+ snapshots.
+
+**Lookup of message – to – graph object in messaging facility:**
+
+|image2|
+
+**Finding the right object related to a specific logging/monitoring
+message**:
+
+|image3|
+
+***Service Provider use-case story (Calipso ‘P’ release):***
+
+BoingBoing is a specialized video casting service and blogging site. It
+is using several locations to run their service (regional hubs and
+central corporate campus, some hosted and some are private).
+
+BoingBoing contracted AT&T to build an NFV service for them, deployed on
+2 new hosted regional hubs, to be brought up dynamically for special
+sporting, news or cloture events. On each one of the 2 hosted virtual
+environments the following service chain is created:
+
+1. Two vyatta 5600 virtual routers are front-end routing aggregation
+ function.
+
+2. Two Steelhead virtual wan acceleration appliances connected to
+ central campus for accelerating and caching of video casting
+ services.
+
+3. Two f5 BIG-IP Traffic Management (load balancing) virtual appliances.
+
+4. Two Cisco vASA for virtual firewall and remote-access VPN services.
+
+As a major milestone for BoingBoing’s due diligence for choosing AT&T
+NFV service, BoingBoing acquires 2 shared hosting facilities and
+automatic service from AT&T, that is cost-effective and stable, it
+includes This NFV service consist of a total of 16 virtual appliance
+across those 2 sites, to be created on-demand and maintained with a
+certain SLA once provisioned, all NFV devices are connected using
+several networks, provisioned using VPP ml2 on an OpenStack based
+environment..
+
+AT&T executives instruct their infrastructure operations department to
+make sure proper SLA and Monitoring is in-place so the due diligence and
+final production deployment of BoingBoing’s services in the AT&T
+datacenters goes well and that BoingBoing’s engineers receive excellent
+service experience.
+
+BoingBoing received the following SLA with their current VPS contract:
+
+- 30-day money back guarantee, in case of a single service down event
+ or any dissatisfaction.
+
+- 99.9 % uptime/availability with a weekly total downtime of 10
+ minutes.
+
+- 24/7/365 on-call service with a total of 2 hours MTTR.
+
+- Full HA for all networking services.
+
+- Managed service using Control Panel IaaS provisioning with overall
+ health visibility.
+
+- Dedicated RAM, from16GB to 64GB from within control panel.
+
+- Guaranteed usage of SSD or equivalent speeds, storage capacity from
+ 10GB to 80GB.
+
+- Backup service based on cinder-backup and Ceph’s dedicated backup
+ volumes, with restoration time below 4 hours.
+
+- End-to-end throughput from central campus to dynamically created
+ regional sites to be always above 2Gbps, including all devices on the
+ service chain and the virtual networking in place.
+
+AT&T’s operations factored all requirement and has decided to include
+real-time monitoring and analysis for the NFV environment for
+BoingBoing.
+
+One of the tools used now for BoingBoing environment in AT&T is Calipso
+for virtual networking.
+
+Here are some benefits provided by Calipso for AT&T operations during
+service cycles:
+
+*Reporting:*
+
+Special handling of virtual networking is in place:
+
+- AT&T designed a certain virtual networking (SFC) setup and
+ connectivity that provides the HA and performance required by the SLA
+ and decided on several physical locations for BoingBoing’s virtual
+ appliances in different availability zones.
+
+- Scheduling of discovery has been created, Calipso takes a snapshot of
+ BoingBoing’s environment every Sunday at midnight, reporting on
+ connectivity among all 16 instances (8 per regional site, 4 pairs on
+ each) and overall health of that connectivity.
+
+- Every Sunday morning at 8am, before the week’s automatic
+ snapshotting, the NOC administrator runs a manual discovery and saves
+ that snapshot, she then runs a comparison check against last week’s
+ snapshot and against initial design to find any gaps or changes that
+ might happen due to other shared services deployments, virtual
+ instances and their connectivity are analyzed and reported with
+ Calipso’s topology and health monitoring.
+
+- Reports are saved for a bi-weekly reporting sent to BoingBoing’s
+ networking engineers.
+
+- Throughput is measured by a special traffic sampling technology
+ inside the VPP virtual switches and sent back to Calipso for
+ references to virtual objects and topological inventory. Dependencies
+ are analyzed so SFC topologies are now visualized across all sites
+ and includes graphing facility on the Calipso UI to visualize the
+ throughput.
+
+ *Change management:*
+
+ If infrastructure changes needs to happen on any virtual service
+ (NFV virtual appliances, internal routers, switches, firewalls etc.)
+ or on any physical server or physical switch the following special
+ guidelines apply:
+
+- Run a lookup on Calipso search-engine for the name of the virtual
+ service, switch or host, including names of NFV appliances as updated
+ in the Calipso inventory by the NFV provisioning application. Lookup
+ if BoingBoing environment is using this object (using the object’s
+ attributes).
+
+ **Running a lookup on Calipso search-engine**
+
+|image4|
+
+- Using Calipso’s impact analysis, fill a report stating all
+ BoingBoing’s objects, on which host, connected to which switch that
+ is affected by the planed change.
+
+- Run clique-type scan, using the specific object as ‘focal-point’ to
+ create a dedicated topology with accompanied health report before
+ conducting the change itself, use this a *pre snapshot*.
+
+- Simulate the change, using BoingBoing’s testing environment only,
+ make sure HA services are in places and downtime is confirmed to be
+ in the SLA boundaries.
+
+- Using all reports provided by Calipso, along with application and
+ storage reports, send a detailed change request to NOC and later to
+ the end-customer for review.
+
+- During the change, make sure HA is operational, by running the same
+ clique-type snapshotting every 10 minutes and running a comparison.
+
+- NOC, while waiting for the change to complete, looks at Calipso’s
+ dashboard focused on BoingBoing’s environment, monitoring results for
+ SFC service down event (as expected), impact on other objects in the
+ service chain - the entire Calipso clique for that object (as
+ expected).
+
+- Once operations has reported back to NOC about change done, run the
+ same snapshotting again as *post snapshot* and run a comparison to
+ make sure all virtual networking are back to the ‘as designed’ stage
+ and all networking services are back.
+
+**Example snapshot taken at one stage for the BoingBoing virtual
+networking and SFC:**
+
+|image5|
+
+ *Troubleshooting:*
+
+ AT&T NOC uses Calipso dashboards for BoingBoing’s environment for
+ their daily health-check. Troubleshooting starts in two cases:
+
+1. When a failure is detected on Calipso for any of BoingBoing’s objects
+ on their virtual networking topologies,
+
+2. When a service case has been opened by BoingBoing with “High
+ Priority, SFC down” flag.
+
+ The following actions are taken, using Calipso dashboards:
+
+- Kickoff a discovery through Calipso API for all objects related to
+ BoingBoing.
+
+- For a service request with no Calipso error detected: using Calipso’s
+ impact analysis, create all cliques for all objects as focal point.
+
+- For an error detected by Calipso: using Calipso’s impact analysis,
+ create cliques for objects with errors as focal point.
+
+- Resulted cliques are then analyzed using detailed messaging facility
+ in Calipso (looking deeply into any message generated regarding the
+ related objects).
+
+- If this is a failure on any physical device (host or switch) and/or
+ on any physical NIC (switch or host side), Calipso immediately points
+ this out and using the specific set of messages generated the
+ administrator can figure out the root cause (like optical failure,
+ driver, disconnect etc.).
+
+- In virtual object failures Calipso saves time pinpointing the servers
+ where erroneous objects are running, and their previous and new
+ connectivity details.
+
+- \*Sources of alerts ...OpenStack, Calipso’s and Sensu are built-in
+ sources, other NFV related monitoring and alerting sources can be
+ added to Calipso messaging system.
+
+- Calipso alerts on dependencies for :
+
+1. All related objects in the clique for that objects.
+
+2. Related hosts
+
+3. Related projects and networks
+
+4. Related NFV service and SFC (\* in case NFV tacker has been added)
+
+- Administrators connects directly to the specific servers and now,
+ using the specific object attributes can start he’s manual
+ troubleshooting (actual fixing of the software issues is not
+ currently part of the Calipso features).
+
+- The NOC operators approves closing the service ticket only when all
+ related Calipso cliques are showing up as healthy and connectivity is
+ back to it’s original “as designed” stage, using Calipso older
+ snapshots.
+
+**Calipso’s monitoring dashboard shows virtual services are back to
+operational state:**
+
+|image6|
+
+.. |image0| image:: media/image101.png
+ :width: 7.14372in
+ :height: 2.84375in
+.. |image1| image:: media/image102.png
+ :width: 6.99870in
+ :height: 2.87500in
+.. |image2| image:: media/image103.png
+ :width: 6.50000in
+ :height: 0.49444in
+.. |image3| image:: media/image104.png
+ :width: 6.50000in
+ :height: 5.43472in
+.. |image4| image:: media/image105.png
+ :width: 7.24398in
+ :height: 0.77083in
+.. |image5| image:: media/image106.png
+ :width: 6.50000in
+ :height: 3.58611in
+.. |image6| image:: media/image107.png
+ :width: 7.20996in
+ :height: 2.94792in
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| # | Event name | AMQP event | Handler | Workflow | Scans | Notes |
+==========================+===========================+=====================================+=========================================+==================================================================================================================================================================================================================================================================================+======================================================================================================+==========================================================================================================================================================================================================================================================================================================================================+
-| **Instance** |
+| **Instance** |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 1 | Create Instance | compute.instance.create.end | EventInstanceAdd | 1. Get *instances\_root* from inventory | **Yes** | ** ** |
| | | | | | | |
| | | | | | | |
| | | | | 2. Execute *self.delete\_handler()* | | |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| **Instance Lifecycle** |
+| **Instance Lifecycle** |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 4 | Instance Down | compute.instance.shutdown.start | **Not implemented** | | | |
| | | | | | | |
| | | | | | | |
| | | compute.instance.suspend.end | | | | |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| **Region** |
+| **Region** |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 6 | Add Region | servergroup.create | **Not implemented** | | | |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 8 | Delete Region | servergroup.delete | **Not implemented** | ** ** | ** ** | ** ** |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| **Network** |
+| **Network** |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 9 | Add Network | network.create.end | EventNetworkAdd | 1. If network with specified *id* already exists, log error and **return None** | **No** | ** ** |
| | | | | | | |
| | | | | | | |
| | | | | 2. Execute *self.delete\_handler()* | | |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| **Subnet** |
+| **Subnet** |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 12 | Add Subnet | subnet.create.end | EventSubnetAdd | 1. Get *network\_document* from db | **Yes** {cliques: 1} | 1. I don’t fully understand what `*these lines* <https://cto-github.cisco.com/OSDNA/OSDNA/blob/b8246e3b19732d2f30922791ade23a94b4f52426/app/discover/events/event_subnet_add.py#L123-L126>`__ do. We make sure *ApiAccess.regions* variable is not empty, but why? The widespread usage of static variables is not a good sign anyway. |
| | | | | | | |
| | | | | | | |
| | | | | 6. If no subnets are left in *network\_document*, delete related vservice dhcp, port and vnic documents | | |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| **Port** |
+| **Port** |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 15 | Create Port | port.create.end | EventPortAdd | 1. Check if ports folder exists, create if not. | **Yes** {cliques: 1} | 1. The port and (maybe) port folder will still persist in db even if we abort the execution on step 6. See idea 1 for details. |
| | | | | | | |
| | | | | | | |
| | | | | 6. Execute *self.delete\_handler(vnic)* *for vnic* | | |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| **Router** |
+| **Router** |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 18 | Add Router | router.create.end | EventRouterAdd | 1. Get *host* by id from db | **Yes** {cliques: 1} | 1. Looks like code author confused a lot of stuff here. This class needs to be reviewed thoroughly. |
| | | | | | | |
| | | | | | | |
| | | | | 2. Execute *self.delete\_handler()* | | |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| **Router Interface** |
+| **Router Interface** |
+--------------------------+---------------------------+-------------------------------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 21 | Add Router Interface | router.interface.create | EventInterfaceAdd | 1. Get *network\_doc* from db based on subnet id from interface payload | **Yes** {cliques: 1} | 1. Log message states that we should abort interface adding, though the code does nothing to support that. Moreover, router\_doc can’t be empty at that moment because it’s referenced before. |
| | | | | | | |
--- /dev/null
+| Calipso.io
+| Installation Guide
+
+|image0|
+
+Project “Calipso” tries to illuminate complex virtual networking with
+real time operational state visibility for large and highly distributed
+Virtual Infrastructure Management (VIM).
+
+We believe that Stability is driven by accurate Visibility.
+
+Calipso provides visible insights using smart discovery and virtual
+topological representation in graphs, with monitoring per object in the
+graph inventory to reduce error vectors and troubleshooting, maintenance
+cycles for VIM operators and administrators.
+
+Table of Contents
+
+Calipso.io Installation Guide 1
+
+1 Pre Requisites 3
+
+1.1 Pre Requisites for Calipso “all in one” application 3
+
+1.2 Pre Requisites for Calipso UI application 3
+
+2 Installation Option used with Apex 4
+
+2.1 Micro Services App, single line install 4
+
+3 OPNFV Scenario 5
+
+3.1 APEX automatic configurator and setup 5
+
+3.2 Apex scenario 5
+
+3.3 Calipso functest 6
+
+TBD 6
+
+Pre Requisites
+===============
+
+Pre Requisites for Calipso “all in one” application
+----------------------------------------------------
+
+ Calipso’s main application is written with Python3.5 for Linux
+ Servers, tested successfully on Centos 7.3 and Ubuntu 16.04. When
+ running using micro-services many of the required software packages
+ and libraries are delivered per micro service, but for an “all in
+ one” application case there are several dependencies.
+
+ Here is a list of the required software packages, and the official
+ supported steps required to install them:
+
+1. Python3.5.x for Linux :
+ https://docs.python.org/3.5/using/unix.html#on-linux
+
+2. Pip for Python3 : https://docs.python.org/3/installing/index.html
+
+3. Python3 packages to install using pip3 :
+
+ **sudo pip3 install falcon (>1.1.0)**
+
+ **sudo pip3 install pymongo (>3.4.0)**
+
+ **sudo pip3 install gunicorn (>19.6.0)**
+
+ **sudo pip3 install ldap3 (>2.1.1)**
+
+ **sudo pip3 install setuptools (>34.3.2)**
+
+ **sudo pip3 install python3-dateutil (>2.5.3-2)**
+
+ **sudo pip3 install bcrypt (>3.1.1)**
+
+ **sudo pip3 install bson**
+
+ **sudo pip3 install websocket**
+
+ **sudo pip3 install datetime**
+
+ **sudo pip3 install typing**
+
+ **sudo pip3 install kombu**
+
+ **sudo pip3 install boltons**
+
+ **sudo pip3 install paramiko**
+
+ **sudo pip3 install requests **
+
+ **sudo pip3 install httplib2**
+
+ **sudo pip3 install mysql.connector**
+
+ **sudo pip3 install xmltodict**
+
+ **sudo pip3 install cryptography**
+
+ **sudo pip3 install docker**
+
+1. Git : https://git-scm.com/book/en/v2/Getting-Started-Installing-Git
+
+2. Docker : https://docs.docker.com/engine/installation/
+
+Pre Requisites for Calipso UI application
+------------------------------------------
+
+ Calipso UI is developed and maintained using Meteor Framework
+ (https://www.meteor.com/tutorials). For stability and manageability
+ reasons we decided to always build the latest Calipso UI as a Docker
+ container pre-parameterized for stable and supported behavior. The
+ required steps for installing the Calipso UI with several options
+ are listed below.
+
+Installation Option used with Apex
+==================================
+
+Micro Services App, single line install
+---------------------------------------
+
+ For most users, this will be the fastest and more reliable install
+ option. We currently have Calipso divided into 7 major containers,
+ those are installed using a single installer. The Calipso containers
+ are pre-packaged and fully customized per our design needs. Here are
+ the required steps for installation using this option:
+
+1. Follow steps 1- 5 per section 2.1 above.
+
+2. Install Docker : https://docs.docker.com/engine/installation/
+
+3. Install the following python3 libraries using pip3 : docker, pymongo
+
+4. Although Calipso installer can download all needed containers, if
+ they doesn’t exist locally already, we recommend doing a manual
+ download of all 7 containers, providing better control and logging:
+
+ **sudo docker login** # use your DockerHub username and password to
+ login.
+
+ **sudo docker pull korenlev/calipso:scan** # scan container used to
+ scan VIM
+
+ **sudo docker pull korenlev/calipso:listen** # listen container to
+ attach to VIM’s BUS.
+
+ **sudo docker pull korenlev/calipso:api** # api container for
+ application integration
+
+ **sudo docker pull korenlev/calipso:sensu** # sensu server container
+ for monitoring
+
+ **sudo docker pull korenlev/calipso:mongo** # calipso mongo DB
+ container
+
+ **sudo docker pull korenlev/calipso:ui** # calipso ui container
+
+ **sudo docker pull korenlev/calipso:ldap** # calipso ldap container
+
+1. Check that all containers were downloaded and registered
+ successfully:
+
+ **sudo docker images**
+
+ Expected results (As of Aug 2017):
+
+ **REPOSITORY TAG IMAGE ID CREATED SIZE**
+
+ **korenlev/calipso listen 12086aaedbc3 6 hours ago 1.05GB**
+
+ **korenlev/calipso api 34c4c6c1b03e 6 hours ago 992MB**
+
+ **korenlev/calipso scan 1ee60c4e61d5 6 hours ago 1.1GB**
+
+ **korenlev/calipso sensu a8a17168197a 6 hours ago 1.65GB**
+
+ **korenlev/calipso mongo 17f2d62f4445 22 hours ago 1.31GB**
+
+ **korenlev/calipso ui ab37b366e812 11 days ago 270MB**
+
+ **korenlev/calipso ldap 316bc94b25ad 2 months ago 269MB**
+
+1. Run the calipso installer using single line arguments:
+
+ **python3 calipso/app/install/calipso-installer.py--command
+ start-all --copy q**
+
+ This should launch all calipso modules in sequence along with all
+ needed configuration files placed in /home/calipso.
+
+OPNFV Scenario
+===============
+
+Although calipso is designed for any VIM and for enterprise use-cases
+too, service providers can use additional capability to install calipso
+with Apex for OPNFV.
+
+APEX automatic configurator and setup
+-------------------------------------
+
+ When using apex to install OPNFV, the Triple-O based OpenStack is
+ installed automatically and calipso installation can be initiated
+ automatically after apex completes the VIM installation process for
+ a certain scenario.
+
+ In this case setup\_apex\_environment.py can be used for creating a
+ new environment automatically into mongoDB and UI of Calipso
+ (instead of using the calipso UI to do that as typical user would
+ do), then detailed scanning can start immediately, the following
+ options are available for setup\_apex\_environment.py:
+
+ **-m [MONGO\_CONFIG], --mongo\_config [MONGO\_CONFIG]**
+
+ **name of config file with MongoDB server access details**
+
+ **(Default: /local\_dir/calipso\_mongo\_access.conf)**
+
+ **-d [CONFIG\_DIR], --config\_dir [CONFIG\_DIR]**
+
+ **path to directory with config data (Default:**
+
+ **/home/calipso/apex\_setup\_files)**
+
+ **-i [INSTALL\_DB\_DIR], --install\_db\_dir [INSTALL\_DB\_DIR]**
+
+ **path to directory with DB data (Default:**
+
+ **/home/calipso/Calipso/app/install/db)**
+
+ **-a [APEX], --apex [APEX]**
+
+ **name of environment to Apex host**
+
+ **-e [ENV], --env [ENV]**
+
+ **name of environment to create(Default: Apex-Euphrates)**
+
+ **-l [LOGLEVEL], --loglevel [LOGLEVEL]**
+
+ **logging level (default: "INFO")**
+
+ **-f [LOGFILE], --logfile [LOGFILE]**
+
+ **log file (default:**
+
+ **"/home/calipso/log/apex\_environment\_fetch.log")**
+
+ **-g [GIT], --git [GIT]**
+
+ **URL to clone Git repository (default:**
+
+ **https://git.opnfv.org/calipso)**
+
+Apex scenario
+-------------
+
+ Starting Euphrates 1.0 the following scenario added with Apex
+ installer:
+
+ **os-nosdn-calipso-noha**
+
+ Following CI jobs defined:
+
+ https://build.opnfv.org/ci/job/calipso-verify-euphrates/
+
+ https://build.opnfv.org/ci/job/apex-testsuite-os-nosdn-calipso-noha-baremetal-euphrates/
+
+ https://build.opnfv.org/ci/job/apex-os-nosdn-calipso-noha-baremetal-euphrates/
+
+ Note: destination deploy server needs to have pre-requisites
+ detailed above.
+
+Calipso functest
+----------------
+
+TBD
+----
+
+.. |image0| image:: media/image1.png
+ :width: 6.50000in
+ :height: 4.27153in
--- /dev/null
+| Calipso
+| Developer Guide
+
+|image0|
+
+Project “Calipso” tries to illuminate complex virtual networking with
+real time operational state visibility for large and highly distributed
+Virtual Infrastructure Management (VIM).
+
+We believe that Stability is driven by accurate Visibility.
+
+Calipso provides visible insights using smart discovery and virtual
+topological representation in graphs, with monitoring per object in the
+graph inventory to reduce error vectors and troubleshooting, maintenance
+cycles for VIM operators and administrators.
+
+Project architecture
+====================
+
+Calipso comprises two major parts: application and UI. We’ll focus on
+the former in this developer guide.
+
+Current project structure is as follows:
+
+- root/
+
+ - app/
+
+ - api/
+
+ - responders/
+
+ - auth/
+
+ - resource/
+
+ - *server.py*
+
+ - config/
+
+ - *events.json*
+
+ - *scanners.json*
+
+ - discover/
+
+ - events/
+
+ - listeners/
+
+ - *default\_listener.py*
+
+ - *listener\_base.py*
+
+ - handlers/
+
+ - *event\_base.py*
+
+ - *event\_\*.py*
+
+ - fetchers/
+
+ - aci/
+
+ - api/
+
+ - cli/
+
+ - db/
+
+ - *event\_manager.py*
+
+ - *scan.py*
+
+ - *scan\_manager.py*
+
+ - monitoring/
+
+ - checks/
+
+ - handlers/
+
+ - *monitor.py*
+
+ - setup/
+
+ - *monitoring\_setup\_manager.py*
+
+ - test/
+
+ - api/
+
+ - event\_based\_scan/
+
+ - fetch/
+
+ - scan/
+
+ - utils/
+
+ - ui/
+
+Application structure
+---------------------
+
+‘API’ package
+~~~~~~~~~~~~~
+
+Calipso API is designed to be used by native and third-party
+applications that are planning to use Calipso discovery application.
+
+***api/responders***
+
+This package contains all exposed API endpoint handlers:
+
+*auth* package contains token management handlers,
+
+*resource* package contains resource handlers.
+
+***server.py***
+
+API server startup script. In order for it to work correctly, connection
+arguments for a Mongo database used by a Calipso application instance
+are required:
+
+-m [MONGO\_CONFIG], --mongo\_config [MONGO\_CONFIG]
+
+name of config file with mongo access details
+
+--ldap\_config [LDAP\_CONFIG]
+
+name of the config file with ldap server config
+
+details
+
+-l [LOGLEVEL], --loglevel [LOGLEVEL]
+
+logging level (default: 'INFO')
+
+-b [BIND], --bind [BIND]
+
+binding address of the API server (default
+
+127.0.0.1:8000)
+
+-y [INVENTORY], --inventory [INVENTORY]
+
+name of inventory collection (default: 'inventory')
+
+-t [TOKEN\_LIFETIME], --token-lifetime [TOKEN\_LIFETIME]
+
+lifetime of the token
+
+For detailed reference and endpoints guide, see the API Guide document.
+
+‘Discover’ package
+~~~~~~~~~~~~~~~~~~
+
+‘Discover’ package contains the core Calipso functionality which
+involves:
+
+- scanning a network topology using a defined suite of scanners (see
+ `Scanning concepts <#scanning-concepts>`__, `Scanners configuration
+ file structure <#the-scanners-configuration-file-structure>`__) that
+ use fetchers to get all needed data on objects of the topology;
+
+- tracking live events that modifies the topology in any way (by adding
+ new object, updating existing or deleting them) using a suite of
+ event handlers and event listeners;
+
+- managing the aforementioned suites using specialized manager scripts
+ (*scan\_manager.py* and *event\_manager.py*)
+
+‘Tests’ package
+~~~~~~~~~~~~~~~
+
+‘Tests’ package contains unit tests for main Calipso components: API,
+event handlers, fetchers, scanners and utils.
+
+Other packages
+~~~~~~~~~~~~~~
+
+***Install***
+
+Installation and deployment scripts (with initial data for Calipso
+database).
+
+***Monitoring***
+
+Monitoring configurations, checks and handlers (see
+`Monitoring <#monitoring>`__ section and Monitoring Guide document).
+
+***Utils***
+
+Utility modules for app-wide use (inventory manager, mongo access,
+loggers, etc.).
+
+Scanning Guide
+==============
+
+ Introduction to scanning
+-------------------------
+
+Architecture overview
+~~~~~~~~~~~~~~~~~~~~~
+
+Calipso backend will scan any OpenStack environment to discover the
+objects that it is made of, and place the objects it discovered in a
+MongoDB database.
+
+Following discovery of objects, Calipso will:
+
+| Find what links exist between these objects, and save these links to
+ MongoDB as well.
+| For example, it will create a pnic-network link from a pNIC (physical
+ NIC) and the network it is connected to.
+
+Based on user definitions, it will create a 'clique' for each object
+using the links it previously found. These cliques are later used to
+present graphs for objects being viewed in the Calipso UI. This is not a
+clique by graph theory definition, but more like the social definition
+of clique: a graph of related, interconnected nodes.
+
+
+
+OpenStack Scanning is done using the following methods, in order of
+preference:
+
+1. OpenStack API
+
+2. MySQL DB - fetch any extra detail we can from the infrastructure
+ MySQL DB used by OpenStack
+
+3. CLI - connect by SSH to the hosts in the OpenStack environment to run
+ commands, e.g. ifconfig, that will provide the most in-depth details.
+
+
+
+| *Note*: 'environment' in Calipso means a single deployment of
+ OpenStack, possibly containing multiple tenants (projects), hosts and
+ instances (VMs). A single Calipso instance can handle multiple
+ OpenStack environments.
+| However, we expect that typically Calipso will reside inside an
+ OpenStack control node and will handle just that node's OpenStack
+ environment.
+
+
+
+***Environment***
+
+| The Calipso scan script, written in Python, is called scan.py.
+| It uses Python 3, along with the following libraries:
+
+- pymongo - for MongoDB access
+
+- mysql-connector - For MySQL DB access
+
+- paramiko - for SSH access
+
+- requests - For handling HTTP requests and responses to the OpenStack
+ API
+
+- xmltodict - for handling XML output of CLI commands
+
+- cryptography - used by Paramiko
+
+See Calipso installation guide for environment setup instructions.
+
+***Configuration***
+
+The configuration for accessing the OpenStack environment, by API, DB or
+SSH, is saved in the Calipso MongoDB *“environments\_config”*
+collection.
+
+Calipso can work with a remote MongoDB instance, the details of which
+are read from a configuration file (default: */etc/calipso/mongo.conf*).
+
+| The first column is the configuration key while the second is the
+ configuration value, in the case the value is the server host name or
+ IP address.
+| Other possible keys for MongoDB access:
+
+- port: IP port number
+
+- Other parameters for the PyMongo MongoClient class constructor
+
+Alternate file location can be specified using the CLI -m parameter.
+
+Scanning concepts
+~~~~~~~~~~~~~~~~~
+
+***DB Schema***
+
+Objects are stored in the inventory collection, named *“inventory”* by
+default, along with the accompanying collections, named by
+default: \ *"links", "cliques", "clique\_types" and
+"clique\_constraints"*. For development, separate sets of collections
+can be defined per environment (collection names are created by
+appending the default collection name to the alternative inventory
+collection name).
+
+The inventory, links and cliques collections are all designed to work
+with a multi-environment scenario, so documents are marked with an
+*"environment"* attribute.
+
+The clique\_types collection allows Calipso users (typically
+administrators) to define how the "clique" graphs are to be defined.
+
+It defines a set of link types to be traversed when an object such as an
+instance is clicked in the UI (therefore referred to as the focal
+point). See "Clique Scanning" below. This definition can differ between
+environments.
+
+Example: for focal point type "instance", the link types are often set
+to
+
+- instance-vnic
+
+- vnic-vconnector
+
+- vconnector-vedge
+
+- vedge-pnic
+
+- pnic-network
+
+| The clique\_constraints collection defines a constraint on links
+ traversed for a specific clique when starting from a given focal
+ point.
+| For example: instance cliques are constrained to a specific
+ network. If we wouldn't have this constraint, the resulting graph
+ would stretch to include objects from neighboring networks that are
+ not really related to the instance.
+
+\ ***Hierarchy of Scanning***
+
+The initial scanning is done hierarchically, starting from the
+environment level and discovering lower levels in turn.
+
+Examples:
+
+- Under environment we scan for regions and projects (tenants).
+
+- Under availability zone we have hosts, and under hosts we have
+ instances and host services
+
+The actual scanning order is not always same as the logical hierarchical
+order of objects, to improve scanning performance.
+
+Some objects are referenced multiple times in the hierarchy. For
+example, hosts are always in an availability zone, but can also be part
+of a host aggregate. Such extra references are saved as references to
+the main object.
+
+***Clique Scanning***
+
+| For creating cliques based on the discovered objects and links, clique
+ types need to be defined for the given environment.
+| A clique type specifies the list of link types used in building a
+ clique for a specific focal point object type.
+
+For example, it can define that for instance objects we want to have the
+following link types:
+
+- instance-vnic
+
+- vnic-vconnector
+
+- vconnector-vedge
+
+- vedge-pnic
+
+- pnic-network
+
+
+
+As in many cases the same clique types are used, default clique types
+will be provided with a new Calipso deployment.
+
+\ ***Clique creation algorithm***
+
+- For each clique type CT:
+
+ - For each focal point object F of the type specified as the clique
+ type focal point type:
+
+ - Create a new clique C
+
+ - Add F to the list of objects included in the clique
+
+ - For each link type X-Y of the link types in CT:
+
+ - Find all the source objects of type x that are already
+ included in the clique
+
+ - For each such source object S:
+
+ - for all links L of type X-Y that have S as their source
+
+ - Add the object T of type Y that is the target in L to the
+ list of objects included in the clique
+
+ - Add L to the list of links in the clique C
+
+How to run scans
+----------------
+
+For running environment scans Calipso uses a specialized daemon script
+called *scan manager*. If Calipso application is deployed in docker
+containers, scan manager will run inside the *calipso-scan* container.
+
+Scan manager uses MongoDB connection to fetch requests for environment
+scans and execute them by running a *scan* script. It also performs
+extra checks and procedures connected to scan failure/completion, such
+as marking *environment* as scanned and reporting errors (see
+`details <#scan-manager>`__).
+
+Scan script workflow:
+
+1. Loads specific scanners definitions from a predefined metadata file
+ (which can be extended in order to support scanning of new object
+ types).
+
+2. Runs the root scanner and then children scanners recursively (see
+ `Hierarchy of scanning <#Hierarchy_of_scanning>`__)
+
+ a. Scanners do all necessary work to insert objects in *inventory*.
+
+3. Finalizes the scan and publishes successful scan completion.
+
+Scan manager
+~~~~~~~~~~~~
+
+Scan manager is a script which purpose is to manage the full lifecycle
+of scans requested through API. It runs indefinitely while:
+
+1. Polling the database (*scans* and *scheduled\_scans* collections) for
+ new and scheduled scan requests;
+
+2. Parsing their configurations;
+
+3. Running the scans;
+
+4. Logging the results.
+
+Scan manager can be run in a separate container provided that it has
+connection to the database and the topology source system.
+
+Monitoring
+----------
+
+***Monitoring Subsystem Overview***
+
+Calipso monitoring uses Sensu to remotely track actual state of hosts.
+
+A Sensu server is installed as a Docker image along with the other
+Calipso components.
+
+
+
+Remote hosts send check events to the Sensu server.
+
+We use a filtering of events such that the first occurrence of a check
+is always used, after that cases where status is unchanged are ignored.
+
+When handling a check event, the Calipso Sensu handlers will find the
+matching Calipso object, and update its status.
+
+We also keep the timestamp of the last status update, along with the
+full check output.
+
+Setup of checks and handlers code on the server and the remote hosts can
+be done by Calipso. It is also possible to have this done using another
+tool, e.g. Ansible or Puppet.
+
+More info is available in Monitoring Guide document.
+
+
+
+***Package Structure***
+
+Monitoring package is divided like this:
+
+1. Checks: these are the actual check scripts that will be run on
+the hosts;
+
+2. Handlers: the code that does handling of check events;
+
+3. Setup: code for setting up handlers and checks.
+
+Events Guide
+============
+
+Introduction
+------------
+
+Events
+~~~~~~
+
+Events in general sense are any changes to the monitored topology
+objects that are trackable by Calipso. We currently support subscription
+to Neutron notification queues for several OpenStack distributions as a
+source of events.
+
+The two core concepts of working with events are *listening to events*
+and *event handling*, so the main module groups in play are the *event
+listener* and *event handlers*.
+
+Event listeners
+~~~~~~~~~~~~~~~
+
+An event listener is a module that handles connection to the event
+source, listening to the new events and routing them to respective event
+handlers.
+
+An event listener class should be designed to run indefinitely in
+foreground or background (daemon) while maintaining a connection to the
+source of events (generally a message queue like RabbitMQ or Apache
+Kafka). Each incoming event is examined and, if it has the correct
+format, is routed to the corresponding event handler class. The routing
+can be accomplished through a dedicated event router class using a
+metadata file and a metadata parser (see `Metadata
+parsers <#metadata-parsers>`__).
+
+Event handlers
+~~~~~~~~~~~~~~
+
+An event handler is a specific class that parses the incoming event
+payload and performs a certain CUD (Create/Update/Delete) operation on
+zero or more database objects. Event handler should be independent of
+the event listener implementation.
+
+Event manager
+~~~~~~~~~~~~~
+
+Event manager is a script which purpose is to manage event listeners. It
+runs indefinitely and performs the following operations:
+
+1. Starts a process for each valid entry in *environments\_config*
+ collection that is scanned (*scanned == true*) and has the *listen*
+ flag set to *true*;
+
+2. Checks the *operational* statuses of event listeners and updating
+ them in *environments\_config* collection;
+
+3. Stops the event listeners that no longer qualify for listening (see
+ step 1);
+
+4. Restarts the event listeners that quit unexpectedly;
+
+5. Repeats steps 1-5
+
+Event manager can be run in a separate container provided that it has
+connection to the database and to all events source systems that event
+listeners use.
+
+Contribution
+~~~~~~~~~~~~
+
+You can contribute to Calipso *events* system in several ways:
+
+- create custom event handlers for an existing listener;
+
+- create custom event listeners and reuse existing handlers;
+
+- create custom event handlers and listeners.
+
+See `Creating new event handlers <#creating-new-event-handlers>`__ and
+`Creating new event listeners <#creating-new-event-listeners>`__ for the
+respective guides.
+
+Contribution
+============
+
+This section covers the designed approach to contribution to Calipso.
+
+The main scenario of contribution consists of introducing a new *object*
+type to the discovery engine, defining *links* that connect this new
+object to existing ones, and describing a *clique* (or cliques) that
+makes use of the object and its links. Below we describe how this
+scenario should be implemented, step-by-step.
+
+*Note*: Before writing any new code, you need to create your own
+environment using UI (see User Guide document) or API (see the API guide
+doc). Creating an entry directly in *“environments\_config”* collection
+is not recommended.
+
+Creating new object types
+-------------------------
+
+Before you proceed with creation of new object type, you need to make
+sure the following requirements are met:
+
+- New object type has a unique name and purpose
+
+- New object type has an existing parent object type
+
+First of all, you need to create a fetcher that will take care of
+getting info on objects of the new type, processing it and adding new
+entries in Calipso database.
+
+Creating new fetchers
+~~~~~~~~~~~~~~~~~~~~~
+
+A fetcher is a common name for a class that handles fetching of all
+objects of a certain type that have a common parent object. The source
+of this data may be already implemented in Calipso (like OpenStack API,
+CLI and DB sources) or you may create one yourself.
+
+***Common fetchers***
+
+Fetchers package structure should adhere to the following pattern (where
+*%source\_name%* is a short prefix like *api, cli, db*):
+
+- app
+
+ - discover
+
+ - fetchers
+
+ - *%source\_name%*
+
+ - *%source\_name%*\ \_\ *%fetcher\_name%.*\ py
+
+If you reuse the existing data source, your new fetcher should subclass
+the class located in *%source\_name%\_access* module inside the
+*%source\_name%* directory.
+
+Fetcher class name should repeat the module name, except in CamelCase
+instead of snake\_case.
+
+Example: if you are adding a new cli fetcher, you should subclass
+*CliAccess* class found by *app/discover/fetchers/cli/cli\_access.py*
+path. If the module is named *cli\_fetch\_new\_objects.py*, fetcher
+class should be named *CliFetchNewObjects*.
+
+If you are creating a fetcher that uses new data source, you may
+consider adding an “access” class for this data source to store
+convenience methods. In this case, the “access” class should subclass
+the base Fetcher class (found in *app/discover/fetcher.py*) and the
+fetcher class should subclass the “access” class.
+
+All business logic of a fetcher should be defined inside the overridden
+method from base Fetcher class *get(self, parent\_id)*. You should use
+the second argument that is automatically passed by parent scanner to
+get the parent entity from database and any data you may need. This
+method has to return a list of new objects (dicts) that need to be
+inserted in Calipso database. Their parent object should be passed along
+other fields (see example).
+
+*Note*: types of returned objects should match the one their fetcher is
+designed for.
+
+***Example***:
+
+**app/discover/fetchers/cli/cli\_fetch\_new\_objects.py**
+
+ | **from** discover.fetchers.cli.cli\_access **import** CliAccess
+ | **from** utils.inventory\_mgr **import** InventoryMgr
+ | **class** CliFetchNewObjects(CliAccess):
+ | **def** \_\_init\_\_(self):
+ | super().\_\_init\_\_()
+ | self.inv = InventoryMgr()
+ | **def** get(self, parent\_id):
+ | parent = self.inv.get\_by\_id(self.env, parent\_id)
+ | *# do something
+ *\ objects = [{**"type"**: **"new\_type"**, **"id"**: **"1234"**,
+ **"parent"**: parent},
+ | {**"type"**: **"new\_type"**, **"id"**: **"2345"**, **"parent"**:
+ parent}]
+ | **return** objects
+
+This is an example of a fetcher that deals with the objects of type
+*“new\_type”*. It uses the parent id to fetch the parent object, then
+performs some operations in order to fetch the new objects and
+ultimately returns the objects list, at which point it has gathered all
+required information.
+
+\ ***Folder fetcher***
+
+A special type of fetchers is the folder fetcher. It serves as a dummy
+object used to aggregate objects in a specific point in objects
+hierarchy. If you would like to logically separate children objects from
+parent, you may use folder fetcher found at
+*app/discover/fetchers/folder\_fetcher.py*.
+
+Usage is described `here <#Folder_scanner>`__.
+
+The scanners configuration file structure
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Scanners.json** (full path *app/config/scanners.json*) is an essential
+configuration file that defines scanners hierarchy. It has a forest
+structure, meaning that it is a set of trees, where each tree has a
+*root* scanner, potentially many levels of *children* scanners and
+pointers from parent scanners to children scanners. Scanning hierarchy
+is described `here <#Hierarchy_of_scanning>`__.
+
+A scanner is essentially a list of fetchers with configuration (we’ll
+call those **Fetch types**). Fetch types can be **Simple** and
+**Folder**, described below.
+
+***Simple fetch type***
+
+A simple fetch type looks like this:
+
+ | {
+ | **"type"**: **"project"**,
+ | **"fetcher"**: **"ApiFetchProjects"**,
+ | **"object\_id\_to\_use\_in\_child"**: **"name"**,
+
+ | **"environment\_condition"**: {
+ | **"mechanism\_drivers"**: **"VPP"
+ ** },
+ | **"children\_scanner"**: **"ScanProject"
+ **}
+
+Supported fields include:
+
+- *“fetcher”* – class name of fetcher that the scanner uses;
+
+- *“type”* – object type that the fetcher works with;
+
+- *“children\_scanner”* – (optional) full name of a scanner that should
+ run after current one finishes;
+
+- *“environment\_condition”* – (optional) specific constraints that
+ should be checked against the environment in *environments\_config*
+ collection before execution;
+
+- *“object\_id\_to\_use\_in\_child*\ ” – (optional) which parent field
+ should be passed as parent id to the fetcher (default: “id”).
+
+ \ ***Folder fetch type***
+
+Folder fetch types deal with folder fetchers (described
+`here <#Folder_fetcher>`__) and have a slightly different structure:
+
+ | {
+ | **"type"**: **"aggregates\_folder"**,
+ | **"fetcher"**: {
+ | **"folder"**: **true**,
+ | **"types\_name"**: **"aggregates"**,
+ | **"parent\_type"**: **"region"
+ **},
+
+ **"object\_id\_to\_use\_in\_child"**: **"name"**,
+
+ | **"environment\_condition"**: {
+ | **"mechanism\_drivers"**: **"VPP"
+ ** },
+ | **"children\_scanner"**: **"ScanAggregatesRoot"
+ **}
+
+The only difference is that *“fetcher”* field is now a dictionary with
+the following fields:
+
+- *“folder”* – should always be **true**;
+
+- *“types\_name”* – type name in plural (with added ‘s’) of objects
+ that serve as folder’s children
+
+- *“parent\_type”* – folder’s parent type (basically the parent type of
+ folder’s objects).
+
+Updating scanners
+~~~~~~~~~~~~~~~~~
+
+After creating a new fetcher, you should integrate it into scanners
+hierarchy. There are several possible courses of action:
+
+***Add new scanner as a child of an existing one***
+
+If the parent type of your newly added object type already has a
+scanner, you can add your new scanner as a child of an existing one.
+There are two ways to do that:
+
+1. Add new scanner as a *“children\_scanner”* field to parent scanner
+
+ ***Example***
+
+ Before:
+
+ | **"ScanHost"**: [
+ | {
+ | **"type"**: **"host"**,
+ | **"fetcher"**: **"ApiFetchProjectHosts"**,
+ | }
+ | ],
+
+ After:
+
+ | **"ScanHost"**: [
+ | {
+ | **"type"**: **"host"**,
+ | **"fetcher"**: **"ApiFetchProjectHosts"**,
+ | **"children\_scanner"**: **"NewTypeScanner"
+ **}
+ | ],
+ | **"NewTypeScanner"**: [
+ | {
+ | **"type"**: **"new\_type"**,
+ | **"fetcher"**: **"CliFetchNewType"
+ **}
+ | ]
+
+1. Add new fetch type to parent scanner (in case if children scanner
+ already exists)
+
+ ***Example***
+
+ Before:
+
+ | **"ScanHost"**: [
+ | {
+ | **"type"**: **"host"**,
+ | **"fetcher"**: **"ApiFetchProjectHosts"**,
+ | **"children\_scanner"**: **"ScanHostPnic"
+ **}
+ | ],
+
+ After:
+
+ | **"ScanHost"**: [
+ | {
+ | **"type"**: **"host"**,
+ | **"fetcher"**: **"ApiFetchProjectHosts"**,
+ | **"children\_scanner"**: **"ScanHostPnic"
+ **},
+ | {
+ | **"type"**: **"new\_type"**,
+ | **"fetcher"**: **"CliFetchNewType"
+ **}
+ | ],
+
+***Add new scanner and set an existing one as a child***
+
+***Example***
+
+ Before:
+
+ | **"ScanHost"**: [
+ | {
+ | **"type"**: **"host"**,
+ | **"fetcher"**: **"ApiFetchProjectHosts"**,
+ | **"children\_scanner"**: **"ScanHostPnic"
+ **}
+ | ],
+
+ After:
+
+ | **"NewTypeScanner"**: [
+ | {
+ | **"type"**: **"new\_type"**,
+ | **"fetcher"**: **"CliFetchNewType"**,
+ | **"children\_scanner"**: **"ScanHost"
+ **}
+ | ]
+
+ | **"ScanHost"**: [
+ | {
+ | **"type"**: **"host"**,
+ | **"fetcher"**: **"ApiFetchProjectHosts"**,
+ | **"children\_scanner"**: **"ScanHostPnic"
+ **}
+ | ],
+
+***Other cases***
+
+You may choose to combine approaches or use none of them and create an
+isolated scanner if needed.
+
+Updating constants collection
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Before testing your new scanner and fetcher you need to add the newly
+created object type to *“constants”* collection in Calipso database:
+
+1. **constants.object\_types** document
+
+ Append a *{“value”: “new\_type”, “label”: “new\_type”}* object to
+ **data** list.
+
+1. **constants.scan\_object\_types** document
+
+ Append a *{“value”: “new\_type”, “label”: “new\_type”}* object to
+ **data** list.
+
+1. **constants.object\_types\_for\_links** document
+
+ If you’re planning to build links using this object type (you
+ probably are), append a *{“value”: “new\_type”, “label”:
+ “new\_type”}* object to **data** list.
+
+Setting up monitoring
+~~~~~~~~~~~~~~~~~~~~~
+
+In order to setup monitoring for the new object type you have defined,
+you’ll need to add a Sensu check:
+
+1. Add a check script in app/monitoring/checks:
+
+ a. | Checks should return the following values:
+ | 0: **OK**
+ | 1: **Warning**
+ | 2: **Error**
+
+ b. Checks can print the underlying query results to stdout. Do so
+ within reason, as this output is later stored in the DB, so avoid
+ giving too much output;
+
+ c. Test your script on a remote host:
+
+ i. Write it in */etc/sensu/plugins* directory;
+
+ ii. Update the Sensu configuration on the remote host to run this
+ check;
+
+ iii. Add the check in the “checks” section of
+ */etc/sensu/conf.d/client.json*;
+
+ iv. The name under which you save the check will be used by the
+ handler to determine the DB object that it relates to;
+
+ v. Restart the client with the command: *sudo service
+ sensu-client restart*;
+
+ vi. Check the client log file to see the check is run and
+ produces the expected output (in */var/log/sensu* directory).
+
+ d. Add the script to the source directory (*app/monitoring/checks*).
+
+2. Add a handler in app/monitoring/handlers:
+
+ a. If you use a standard check naming scheme and check an object, the
+ *BasicCheckHandler* can take care of this, but add the object type
+ in *basic\_handling\_types* list in *get\_handler()*;
+
+ b. If you have a more complex naming scheme, override
+ MonitoringCheckHandler. See HandleOtep for example.
+
+3. If you deploy monitoring using Calipso:
+
+ a. Add the check in the *monitoring\_config\_templates* collection.
+
+*Check Naming*
+
+The check name should start with the type of the related object,
+followed by an underscore (“\_”). For example, the name for a check
+related to an OTEP (type “otep”) will start with “otep\_“. It should
+then be followed by the object ID.
+
+
+
+For checks related to links, the check name will have this
+format:
link\_<link type>\_<from\_id>\_<to\_id>
+
+Creating new link types
+-----------------------
+
+After you’ve added a new object type you may consider adding new link
+types to connect objects of new type to existing objects in topology.
+Your new object type may serve as a *source* and/or *target* type for
+the new link type.
+
+The process of new link type creation includes several steps:
+
+1. Write a link finder class;
+
+2. Add the link finder class to the link finders configuration file;
+
+3. Update *“constants”* collection with the new link types.
+
+Writing link finder classes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A new link finder class should:
+
+1. Subclass *app.discover.link\_finders.FindLinks* class;
+
+2. Be located in the *app.discover.link\_finders* package;
+
+3. Define an instance method called *add\_links(self)* with no
+ additional arguments. This method is the only entry point for link
+ finder classes.
+
+*FindLinks* class provides access to inventory manager to its subclasses
+which they should use to their advantage. It also provides a convenience
+method *create\_links(self, …)* for saving links to database. It is
+reasonable to call this method at the end of *add\_links* method.
+
+You may opt to add more than one link type at a time in a single link
+finder.
+
+***Example***
+
+ | **from** discover.find\_links **import** FindLinks
+ | **class** FindLinksForNewType(FindLinks):
+ | **def** add\_links(self):
+ | new\_objects = self.inv.find\_items({\ **"environment"**:
+ self.get\_env(),
+ | **"type"**: **"new\_type"**})
+ | **for** new\_object **in** new\_objects:
+ | old\_object = self.inv.get\_by\_id(environment=self.get\_env(),
+ | item\_id=new\_object[**"old\_object\_id"**])
+ | link\_type = **"old\_type-new\_type"
+ **\ link\_name = **"{}-{}"**.format(old\_object[**"name"**],
+ new\_object[**"name"**])
+ | state = **"up"** *# TBD
+ *\ link\_weight = 0 *# TBD
+ *\ self.create\_link(env=self.get\_env(),
+ | source=old\_object[**"\_id"**],
+ | source\_id=old\_object[**"id"**],
+ | target=new\_object[**"\_id"**],
+ | target\_id=new\_object[**"id"**],
+ | link\_type=link\_type,
+ | link\_name=link\_name,
+ | state=state,
+ | link\_weight=link\_weight)
+
+Updating the link finders configuration file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Default link finders configuration file can be found at
+*/app/config/link\_finders.json* and has the following structure:
+
+ | {
+ | **"finders\_package"**: **"discover.link\_finders"**,
+ | **"base\_finder"**: **"FindLinks"**,
+ | **"link\_finders"**: [
+ | **"FindLinksForInstanceVnics"**,
+ | **"FindLinksForOteps"**,
+ | **"FindLinksForPnics"**,
+ | **"FindLinksForVconnectors"**,
+ | **"FindLinksForVedges"**,
+ | **"FindLinksForVserviceVnics"
+ **]
+ | }
+
+File contents:
+
+- *finders\_package* – python path to the package that contains link
+ finders (relative to $PYTHONPATH environment variable);
+
+- *base\_finder* – base link finder class name;
+
+- *link\_finders* – class names of actual link finders.
+
+If your new fetcher meets the requirements described in `Writing link
+finder classes <#writing-link-finder-classes>`__ section, you can append
+its name to the *“link\_finders”* list in *link\_finders.json* file.
+
+Updating constants collection
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Before testing your new links finder, you need to add the newly created
+link types to *“constants”* collection in Calipso database:
+
+1. **constants.link\_types** document
+
+ Append a *{“value”: “source\_type-target\_type”, “label”:
+ “source\_type-target\_type”}* object to **data** list for each new
+ link type.
+
+Creating custom link finders configuration file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you consider writing a custom list finders configuration file, you
+should also follow the guidelines from 4.2.1-4.2.3 while designing link
+finder classes and including them in the new link finders source file.
+
+The general approach is the following:
+
+1. Custom configuration file should have the same key structure with the
+ basic one;
+
+2. You should create a *base\_finder* class that subclasses the basic
+ FindLinks class (see `Writing link finder
+ classes <#writing-link-finder-classes>`__);
+
+3. Your link finder classes should be located in the same package with
+ your *base\_finder* class;
+
+4. Your link finder classes should subclass your *base\_finder* class
+ and override the *add\_links(self)* method.
+
+Creating new clique types
+-------------------------
+
+Two steps in creating new clique types and including them in clique
+finder are:
+
+1. Designing new clique types
+
+2. Updating clique types collection
+
+Designing new clique types
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A clique type is basically a list of links that will be traversed during
+clique scans (see `Clique creation algorithm <#clique_creation>`__). The
+process of coming up with clique types involves general networking
+concepts knowledge as well as expertise in monitored system details
+(e.g. OpenStack distribution specifics). In a nutshell, it is not a
+trivial process, so the clique design should be considered carefully.
+
+The predefined clique types (in *clique\_types* collection) may give you
+some idea about the rationale behind clique design.
+
+Updating clique types collection
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+After designing the new clique type, you need to update the
+*clique\_types* collection in order for the clique finder to use it. For
+this purpose, you should add a document of the following structure:
+
+ {
+
+ "environment": "ANY",
+
+ "link\_types": [
+
+ "instance-vnic",
+
+ "vnic-vconnector",
+
+ "vconnector-vedge",
+
+ "vedge-otep",
+
+ "otep-vconnector",
+
+ "vconnector-host\_pnic",
+
+ "host\_pnic-network"
+
+ ],
+
+ "name": "instance",
+
+ "focal\_point\_type": "instance"
+
+ }
+
+Document fields are:
+
+- *environment* – can either hold the environment name, for which the
+ new clique type is designed, or **“ANY”** if the new clique type
+ should be added to all environments;
+
+- *name* – display name for the new clique type;
+
+- *focal\_point\_type* – the aggregate object type for the new clique
+ type to use as a starting point;
+
+- *link\_types* – a list of links that constitute the new clique type.
+
+Creating new event handlers
+---------------------------
+
+There are three steps to creating a new event handler:
+
+1. Determining *event types* that will be handled by the new handler;
+
+2. Writing the new handler module and class;
+
+3. Adding the (event type -> handler) mapping to the event handlers
+ configuration file.
+
+Writing custom handler classes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Each event handler should adhere to the following design:
+
+1. Event handler class should subclass
+ the app.discover.events.event\_base.EventBase class;
+
+2. Event handler class should override handle method of EventBase.
+ Business logic of the event handler should be placed inside
+ the handle method;
+
+ a. Handle method accepts two arguments: environment name (str) and
+ notification contents (dict). No other event data will be provided
+ to the method;
+
+ b. Handle method returns an EventResult object, which accepts the
+ following arguments in its constructor:
+
+ i. *result* (mandatory) - determines whether the event handling
+ was successful;
+
+ ii. *retry* (optional) - determines whether the message should be
+ put back in the queue in order to be processed later. This
+ argument is checked only if result was set to False;
+
+ iii. *message* (optional) - (Currently unused) a string comment on
+ handling status;
+
+ iv. *related\_object* (optional) – id of the object related to
+ the handled event;
+
+ v. *display\_context* (optional) – (Calipso UI requirement).
+
+3. Module containing event handler class should have the same name as
+ the relevant handler class except translated
+ from UpperCamelCase to snake\_case.
+
+ ***Example:***
+
+ **app/discover/events/event\_new\_object\_add.py**
+
+ | **from** discover.events.event\_base **import** EventBase,
+ EventResult
+ | **class** EventNewObjectAdd(EventBase):
+ | **def** handle(self, env: str, notification: dict) -> EventResult:
+ | obj\_id = notification[**'payload'**][**'new\_object'**][**'id'**]
+ | obj = {
+ | **'id'**: obj\_id,
+ | **'type'**: **'new\_object'
+ **}
+ | self.inv.set(obj)
+ | **return** EventResult(result=\ **True**)
+
+ Modifications in *events.json*:
+
+ <...>
+
+ | **"event\_handlers"**: {
+ | <...>
+ | **"new\_object.create"**: **"EventNewObjectAdd"**,
+ | <...>**
+ **}
+
+ <...>
+
+After these changes are implemented, any event of type
+new\_object.create will be consumed by the event listener and the
+payload will be passed to EventNewObjectAdd handler which will insert a
+new document in the database.
+
+Event handlers configuration file structure
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Events.json** (full path *app/config/events.json*) is a configuration
+file that contains information about events and event handlers,
+including:
+
+- Event subscription details (queues and exchanges for Neutron
+ notifications);
+
+- Location of event handlers package;
+
+- Mappings between event types and respective event handlers.
+
+The structure of *events.json* is as following:
+
+ | {
+ | **"handlers\_package"**: **"discover.events"**,
+ | **"queues"**: [
+ | {
+ | **"queue"**: **"notifications.nova"**,
+ | **"exchange"**: **"nova"
+ **},
+ | <…>
+ | ],
+ | **"event\_handlers"**: {
+ | **"compute.instance.create.end"**: **"EventInstanceAdd"**,
+ | **"compute.instance.update"**: **"EventInstanceUpdate"**,
+ | **"compute.instance.delete.end"**: **"EventInstanceDelete"**,
+ | **"network.create.end"**: **"EventNetworkAdd"**,
+
+ | <…>**
+ **}
+ | }
+
+The root object contains the following fields:
+
+- **handlers\_package** - python path to the package that contains
+ event handlers (relative to $PYTHONPATH environment variable)
+
+- **queues –** RabbitMQ queues and exchanges to consume messages from
+ (for Neutron notifications case)
+
+- **event\_handlers** – mappings of event types to the respective
+ handlers. The structure suggests that any event can have only one
+ handler.
+
+In order to add a new event handler to the configuration file, you
+should add another mapping to the event\_handlers object, where key is
+the event type being handled and value is the handler class name (module
+name will be determined automatically).
+
+If your event is being published to a queue and/or exchange that the
+listener is not subscribed to, you should add another entry to the
+queues list.
+
+Creating new event listeners
+----------------------------
+
+At the moment, the only guideline for creation of new event listeners is
+that they should subclass the *ListenerBase* class (full path
+*app/discover/events/listeners/listener\_base.py*) and override the
+*listen(self)* method that listens to incoming events indefinitely
+(until terminated by a signal).
+
+In future versions, a comprehensive guide to listeners structure is
+planned.
+
+Metadata parsers
+----------------
+
+Metadata parsers are specialized classes that are designed to verify
+metadata files (found in *app/*\ config directory), use data from them
+to load instances of implementation classes (e.g. scanners, event
+handlers, link finders) in memory, and supply them by request. Scanners
+and link finders configuration files are used in scanner, event handlers
+configuration file – in event listener.
+
+In order to create a new metadata parser, you should consider
+subclassing *MetadataParser* class (found in
+*app/utils/metadata\_parser.py*). *MetadataParser* supports parsing and
+validating of json files out of the box. Entry point for the class is
+the *parse\_metadata\_file* method, which requires the abstract
+*get\_required\_fields* method to be overridden in subclasses. This
+method should return a list of keys that the metadata file is required
+to contain.
+
+For different levels of customization you may consider:
+
+1. Overriding *validate\_metadata* method to provide more precise
+ validation of metadata;
+
+2. Overriding *parse\_metadata\_file* to provide custom metadata
+ handling required by your use case.
+
+.. |image0| image:: media/image1.png
+ :width: 6.50000in
+ :height: 4.27153in
+++ /dev/null
-.. This work is licensed under a Creative Commons Attribution 4.0 International License.
-.. http://creativecommons.org/licenses/by/4.0
-.. (c) OPNFV and others.
-
-.. _calipso-release-guide:
-
-=====================
-Calipso Release Guide
-=====================
-
-.. toctree::
- :maxdepth: 2
-
- about.rst
- admin-guide.rst
- api-guide.rst
- calipso-model.rst
- install-guide-rst
- monotoring-guide.rst
- quickstart-guide.rst
+++ /dev/null
-.. _os-nosdn-calipso-noha:
-
-.. This work is licensed under a Creative Commons Attribution 4.0 International Licence.
-.. http://creativecommons.org/licenses/by/4.0
-.. (c) <optionally add copywriters name>
-
-==============================================
-os-nosdn-calipso-noha overview and description
-==============================================
-
-.. toctree::
- :numbered:
- :maxdepth: 4
-
- apex-scenario-guide.rst
Code Review / calipso.git / commitdiff
