X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=yardstick%2Fbenchmark%2Fcontexts%2Fheat.py;h=374be0b1bc97761a57b35e3e2967dd81aed7b01e;hb=62c775f7261dca0081283fadcaa18e53cf2f181c;hp=aa134d6941a31740a4e2aa2789d10f470fdb1f8c;hpb=653902770572c780777d1dc7a371794b670585b1;p=yardstick.git diff --git a/yardstick/benchmark/contexts/heat.py b/yardstick/benchmark/contexts/heat.py index aa134d694..374be0b1b 100644 --- a/yardstick/benchmark/contexts/heat.py +++ b/yardstick/benchmark/contexts/heat.py @@ -14,10 +14,10 @@ import collections import logging import os import uuid +import errno from collections import OrderedDict import ipaddress -import paramiko import pkg_resources from yardstick.benchmark.contexts.base import Context @@ -25,14 +25,25 @@ from yardstick.benchmark.contexts.model import Network from yardstick.benchmark.contexts.model import PlacementGroup, ServerGroup from yardstick.benchmark.contexts.model import Server from yardstick.benchmark.contexts.model import update_scheduler_hints +from yardstick.common.openstack_utils import get_neutron_client from yardstick.orchestrator.heat import HeatTemplate, get_short_key_uuid -from yardstick.common.constants import YARDSTICK_ROOT_PATH +from yardstick.common import constants as consts +from yardstick.common.utils import source_env +from yardstick.ssh import SSH LOG = logging.getLogger(__name__) DEFAULT_HEAT_TIMEOUT = 3600 +def join_args(sep, *args): + return sep.join(args) + + +def h_join(*args): + return '-'.join(args) + + class HeatContext(Context): """Class that represents a context in the logical model""" @@ -42,38 +53,49 @@ class HeatContext(Context): self.name = None self.stack = None self.networks = OrderedDict() + self.heat_timeout = None self.servers = [] self.placement_groups = [] self.server_groups = [] self.keypair_name = None self.secgroup_name = None self._server_map = {} + self.attrs = {} self._image = None self._flavor = None self.flavors = set() self._user = None self.template_file = None self.heat_parameters = None + self.neutron_client = None # generate an uuid to identify yardstick_key # the first 8 digits of the uuid will be used self.key_uuid = uuid.uuid4() + self.heat_timeout = None self.key_filename = ''.join( - [YARDSTICK_ROOT_PATH, 'yardstick/resources/files/yardstick_key-', + [consts.YARDSTICK_ROOT_PATH, 'yardstick/resources/files/yardstick_key-', get_short_key_uuid(self.key_uuid)]) super(HeatContext, self).__init__() - def assign_external_network(self, networks): + @staticmethod + def assign_external_network(networks): sorted_networks = sorted(networks.items()) external_network = os.environ.get("EXTERNAL_NETWORK", "net04_ext") - have_external_network = [(name, net) - for name, net in sorted_networks if - net.get("external_network")] - # no external net defined, assign it to first network usig os.environ - if sorted_networks and not have_external_network: - sorted_networks[0][1]["external_network"] = external_network + + have_external_network = any(net.get("external_network") for net in networks.values()) + if not have_external_network: + # try looking for mgmt network first + try: + networks['mgmt']["external_network"] = external_network + except KeyError: + if sorted_networks: + # otherwise assign it to first network using os.environ + sorted_networks[0][1]["external_network"] = external_network + return sorted_networks - def init(self, attrs): # pragma: no cover + def init(self, attrs): + self.check_environment() """initializes itself from the supplied arguments""" self.name = attrs["name"] @@ -84,8 +106,8 @@ class HeatContext(Context): self.heat_parameters = attrs.get("heat_parameters") return - self.keypair_name = self.name + "-key" - self.secgroup_name = self.name + "-secgroup" + self.keypair_name = h_join(self.name, "key") + self.secgroup_name = h_join(self.name, "secgroup") self._image = attrs.get("image") @@ -93,12 +115,12 @@ class HeatContext(Context): self.heat_timeout = attrs.get("timeout", DEFAULT_HEAT_TIMEOUT) - self.placement_groups = [PlacementGroup(name, self, pgattrs["policy"]) - for name, pgattrs in attrs.get( + self.placement_groups = [PlacementGroup(name, self, pg_attrs["policy"]) + for name, pg_attrs in attrs.get( "placement_groups", {}).items()] - self.server_groups = [ServerGroup(name, self, sgattrs["policy"]) - for name, sgattrs in attrs.get( + self.server_groups = [ServerGroup(name, self, sg_attrs["policy"]) + for name, sg_attrs in attrs.get( "server_groups", {}).items()] # we have to do this first, because we are injecting external_network @@ -106,21 +128,29 @@ class HeatContext(Context): sorted_networks = self.assign_external_network(attrs["networks"]) self.networks = OrderedDict( - (name, Network(name, self, netattrs)) for name, netattrs in + (name, Network(name, self, net_attrs)) for name, net_attrs in sorted_networks) - for name, serverattrs in sorted(attrs["servers"].items()): - server = Server(name, self, serverattrs) + for name, server_attrs in sorted(attrs["servers"].items()): + server = Server(name, self, server_attrs) self.servers.append(server) self._server_map[server.dn] = server - rsa_key = paramiko.RSAKey.generate(bits=2048, progress_func=None) - rsa_key.write_private_key_file(self.key_filename) - print("Writing %s ..." % self.key_filename) - with open(self.key_filename + ".pub", "w") as pubkey_file: - pubkey_file.write( - "%s %s\n" % (rsa_key.get_name(), rsa_key.get_base64())) - del rsa_key + self.attrs = attrs + SSH.gen_keys(self.key_filename) + + def check_environment(self): + try: + os.environ['OS_AUTH_URL'] + except KeyError: + try: + source_env(consts.OPENRC) + except IOError as e: + if e.errno != errno.EEXIST: + LOG.error('OPENRC file not found') + raise + else: + LOG.error('OS_AUTH_URL not found') @property def image(self): @@ -152,9 +182,14 @@ class HeatContext(Context): for network in self.networks.values(): template.add_network(network.stack_name, network.physical_network, - network.provider) + network.provider, + network.segmentation_id, + network.port_security_enabled, + network.network_type) template.add_subnet(network.subnet_stack_name, network.stack_name, - network.subnet_cidr) + network.subnet_cidr, + network.enable_dhcp, + network.gateway_ip) if network.router: template.add_router(network.router.stack_name, @@ -185,7 +220,7 @@ class HeatContext(Context): try: self.flavors.add(server.flavor["name"]) except KeyError: - self.flavors.add(server.stack_name + "-flavor") + self.flavors.add(h_join(server.stack_name, "flavor")) # add servers with availability policy added_servers = [] @@ -193,7 +228,7 @@ class HeatContext(Context): scheduler_hints = {} for pg in server.placement_groups: update_scheduler_hints(scheduler_hints, added_servers, pg) - # workround for openstack nova bug, check JIRA: YARDSTICK-200 + # workaround for openstack nova bug, check JIRA: YARDSTICK-200 # for details if len(availability_servers) == 2: if not scheduler_hints["different_host"]: @@ -249,9 +284,23 @@ class HeatContext(Context): list(self.networks.values()), scheduler_hints) + def get_neutron_info(self): + if not self.neutron_client: + self.neutron_client = get_neutron_client() + + networks = self.neutron_client.list_networks() + for network in self.networks.values(): + for neutron_net in networks['networks']: + if neutron_net['name'] == network.stack_name: + network.segmentation_id = neutron_net.get('provider:segmentation_id') + # we already have physical_network + # network.physical_network = neutron_net.get('provider:physical_network') + network.network_type = neutron_net.get('provider:network_type') + network.neutron_info = neutron_net + def deploy(self): """deploys template into a stack using cloud""" - print("Deploying context '%s'" % self.name) + LOG.info("Deploying context '%s' START", self.name) heat_template = HeatTemplate(self.name, self.template_file, self.heat_parameters) @@ -266,55 +315,75 @@ class HeatContext(Context): raise SystemExit("\nStack create interrupted") except: LOG.exception("stack failed") + # let the other failures happen, we want stack trace raise - # let the other failures happend, we want stack trace + + # TODO: use Neutron to get segmentation-id + self.get_neutron_info() # copy some vital stack output into server objects for server in self.servers: if server.ports: - # TODO(hafe) can only handle one internal network for now - port = next(iter(server.ports.values())) - server.private_ip = self.stack.outputs[port["stack_name"]] - server.interfaces = {} - for network_name, port in server.ports.items(): - self.make_interface_dict(network_name, port['stack_name'], - server, - self.stack.outputs) + self.add_server_port(server) if server.floating_ip: server.public_ip = \ self.stack.outputs[server.floating_ip["stack_name"]] - print("Context '%s' deployed" % self.name) - - def make_interface_dict(self, network_name, stack_name, server, outputs): - server.interfaces[network_name] = { - "private_ip": outputs[stack_name], - "subnet_id": outputs[stack_name + "-subnet_id"], - "subnet_cidr": outputs[ - "{}-{}-subnet-cidr".format(self.name, network_name)], - "netmask": str(ipaddress.ip_network( - outputs["{}-{}-subnet-cidr".format(self.name, - network_name)]).netmask), - "gateway_ip": outputs[ - "{}-{}-subnet-gateway_ip".format(self.name, network_name)], - "mac_address": outputs[stack_name + "-mac_address"], - "device_id": outputs[stack_name + "-device_id"], - "network_id": outputs[stack_name + "-network_id"], + LOG.info("Deploying context '%s' DONE", self.name) + + def add_server_port(self, server): + # use private ip from first port in first network + try: + private_port = next(iter(server.ports.values()))[0] + except IndexError: + LOG.exception("Unable to find first private port in %s", server.ports) + raise + server.private_ip = self.stack.outputs[private_port["stack_name"]] + server.interfaces = {} + for network_name, ports in server.ports.items(): + for port in ports: + # port['port'] is either port name from mapping or default network_name + server.interfaces[port['port']] = self.make_interface_dict(network_name, + port['port'], + port['stack_name'], + self.stack.outputs) + + def make_interface_dict(self, network_name, port, stack_name, outputs): + private_ip = outputs[stack_name] + mac_address = outputs[h_join(stack_name, "mac_address")] + # these are attributes of the network, not the port + output_subnet_cidr = outputs[h_join(self.name, network_name, + 'subnet', 'cidr')] + + # these are attributes of the network, not the port + output_subnet_gateway = outputs[h_join(self.name, network_name, + 'subnet', 'gateway_ip')] + + return { + "private_ip": private_ip, + "subnet_id": outputs[h_join(stack_name, "subnet_id")], + "subnet_cidr": output_subnet_cidr, + "network": str(ipaddress.ip_network(output_subnet_cidr).network_address), + "netmask": str(ipaddress.ip_network(output_subnet_cidr).netmask), + "gateway_ip": output_subnet_gateway, + "mac_address": mac_address, + "device_id": outputs[h_join(stack_name, "device_id")], + "network_id": outputs[h_join(stack_name, "network_id")], + # this should be == vld_id for NSB tests "network_name": network_name, # to match vnf_generic - "local_mac": outputs[stack_name + "-mac_address"], - "local_ip": outputs[stack_name], - "vld_id": self.networks[network_name].vld_id, + "local_mac": mac_address, + "local_ip": private_ip, } def undeploy(self): """undeploys stack from cloud""" if self.stack: - print("Undeploying context '%s'" % self.name) + LOG.info("Undeploying context '%s' START", self.name) self.stack.delete() self.stack = None - print("Context '%s' undeployed" % self.name) + LOG.info("Undeploying context '%s' DONE", self.name) if os.path.exists(self.key_filename): try: @@ -325,48 +394,80 @@ class HeatContext(Context): super(HeatContext, self).undeploy() + @staticmethod + def generate_routing_table(server): + routes = [ + { + "network": intf["network"], + "netmask": intf["netmask"], + "if": name, + # We have to encode a None gateway as '' for Jinja2 to YAML conversion + "gateway": intf["gateway_ip"] if intf["gateway_ip"] else '', + } + for name, intf in server.interfaces.items() + ] + return routes + def _get_server(self, attr_name): """lookup server info by name from context attr_name: either a name for a server created by yardstick or a dict with attribute name mapping when using external heat templates """ - key_filename = pkg_resources.resource_filename( + pkey = pkg_resources.resource_string( 'yardstick.resources', - 'files/yardstick_key-' + get_short_key_uuid(self.key_uuid)) + h_join('files/yardstick_key', get_short_key_uuid(self.key_uuid))).decode('utf-8') if isinstance(attr_name, collections.Mapping): - cname = attr_name["name"].split(".")[1] - if cname != self.name: + node_name, cname = self.split_name(attr_name['name']) + if cname is None or cname != self.name: return None - public_ip = None - private_ip = None - if "public_ip_attr" in attr_name: - public_ip = self.stack.outputs[attr_name["public_ip_attr"]] - if "private_ip_attr" in attr_name: - private_ip = self.stack.outputs[ - attr_name["private_ip_attr"]] - # Create a dummy server instance for holding the *_ip attributes - server = Server(attr_name["name"].split(".")[0], self, {}) - server.public_ip = public_ip - server.private_ip = private_ip + server = Server(node_name, self, {}) + server.public_ip = self.stack.outputs.get( + attr_name.get("public_ip_attr", object()), None) + + server.private_ip = self.stack.outputs.get( + attr_name.get("private_ip_attr", object()), None) else: - if attr_name not in self._server_map: + server = self._server_map.get(attr_name, None) + if server is None: return None - server = self._server_map[attr_name] - - if server is None: - return None result = { "user": server.context.user, - "key_filename": key_filename, + "pkey": pkey, "private_ip": server.private_ip, "interfaces": server.interfaces, + "routing_table": self.generate_routing_table(server), + # empty IPv6 routing table + "nd_route_tbl": [], } # Target server may only have private_ip if server.public_ip: result["ip"] = server.public_ip return result + + def _get_network(self, attr_name): + if not isinstance(attr_name, collections.Mapping): + network = self.networks.get(attr_name, None) + + else: + # Only take the first key, value + key, value = next(iter(attr_name.items()), (None, None)) + if key is None: + return None + network_iter = (n for n in self.networks.values() if getattr(n, key) == value) + network = next(network_iter, None) + + if network is None: + return None + + result = { + "name": network.name, + "segmentation_id": network.segmentation_id, + "network_type": network.network_type, + "physical_network": network.physical_network, + } + return result