Merge "Integrate Patrole as a new testcase"
[functest.git] / functest / opnfv_tests / vnf / ims / orchestra_openims.py
1 #!/usr/bin/env python
2
3 # Copyright (c) 2016 Orange and others.
4 #
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9
10 """Orchestra OpenIMS testcase implementation."""
11
12 import json
13 import logging
14 import os
15 import socket
16 import time
17 import pkg_resources
18 import yaml
19
20 from functest.core import vnf
21 from functest.utils import config
22
23 from org.openbaton.cli.errors.errors import NfvoException
24 from org.openbaton.cli.agents.agents import MainAgent
25 from snaps.config.flavor import FlavorConfig
26 from snaps.config.image import ImageConfig
27 from snaps.config.network import NetworkConfig, PortConfig, SubnetConfig
28 from snaps.config.router import RouterConfig
29 from snaps.config.security_group import (
30     Direction, Protocol, SecurityGroupConfig, SecurityGroupRuleConfig)
31 from snaps.config.vm_inst import FloatingIpConfig
32 from snaps.config.vm_inst import VmInstanceConfig
33 from snaps.openstack.utils import keystone_utils
34 from snaps.openstack.create_image import OpenStackImage
35 from snaps.openstack.create_flavor import OpenStackFlavor
36 from snaps.openstack.create_security_group import OpenStackSecurityGroup
37 from snaps.openstack.create_network import OpenStackNetwork
38 from snaps.openstack.create_router import OpenStackRouter
39 from snaps.openstack.create_instance import OpenStackVmInstance
40
41 from functest.opnfv_tests.openstack.snaps import snaps_utils
42
43
44 __author__ = "Pauls, Michael <michael.pauls@fokus.fraunhofer.de>"
45 # ----------------------------------------------------------
46 #
47 #               UTILS
48 #
49 # -----------------------------------------------------------
50
51
52 def get_config(parameter, file_path):
53     """
54     Get config parameter.
55
56     Returns the value of a given parameter in file.yaml
57     parameter must be given in string format with dots
58     Example: general.openstack.image_name
59     """
60     with open(file_path) as config_file:
61         file_yaml = yaml.safe_load(config_file)
62     config_file.close()
63     value = file_yaml
64     for element in parameter.split("."):
65         value = value.get(element)
66         if value is None:
67             raise ValueError("The parameter %s is not defined in"
68                              " reporting.yaml", parameter)
69     return value
70
71
72 def servertest(host, port):
73     """Method to test that a server is reachable at IP:port"""
74     args = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM)
75     for family, socktype, proto, _, sockaddr in args:
76         sock = socket.socket(family, socktype, proto)
77         try:
78             sock.connect(sockaddr)
79         except socket.error:
80             return False
81         else:
82             sock.close()
83             return True
84
85
86 def get_userdata(orchestrator=dict):
87     """Build userdata for Open Baton machine"""
88     userdata = "#!/bin/bash\n"
89     userdata += "echo \"Executing userdata...\"\n"
90     userdata += "set -x\n"
91     userdata += "set -e\n"
92     userdata += "echo \"Install curl...\"\n"
93     userdata += "apt-get install curl\n"
94     userdata += "echo \"Inject public key...\"\n"
95     userdata += ("echo \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCuPXrV3"
96                  "geeHc6QUdyUr/1Z+yQiqLcOskiEGBiXr4z76MK4abiFmDZ18OMQlc"
97                  "fl0p3kS0WynVgyaOHwZkgy/DIoIplONVr2CKBKHtPK+Qcme2PVnCtv"
98                  "EqItl/FcD+1h5XSQGoa+A1TSGgCod/DPo+pes0piLVXP8Ph6QS1k7S"
99                  "ic7JDeRQ4oT1bXYpJ2eWBDMfxIWKZqcZRiGPgMIbJ1iEkxbpeaAd9O"
100                  "4MiM9nGCPESmed+p54uYFjwEDlAJZShcAZziiZYAvMZhvAhe6USljc"
101                  "7YAdalAnyD/jwCHuwIrUw/lxo7UdNCmaUxeobEYyyFA1YVXzpNFZya"
102                  "XPGAAYIJwEq/ openbaton@opnfv\" >> /home/ubuntu/.ssh/aut"
103                  "horized_keys\n")
104     userdata += "echo \"Download bootstrap...\"\n"
105     userdata += ("curl -s %s "
106                  "> ./bootstrap\n" % orchestrator['bootstrap']['url'])
107     userdata += ("curl -s %s" "> ./config_file\n" %
108                  orchestrator['bootstrap']['config']['url'])
109     userdata += ("echo \"Disable usage of mysql...\"\n")
110     userdata += "sed -i s/mysql=.*/mysql=no/g /config_file\n"
111     userdata += "echo \"Set autostart of components to 'false'\"\n"
112     userdata += "export OPENBATON_COMPONENT_AUTOSTART=false\n"
113     userdata += "echo \"Execute bootstrap...\"\n"
114     bootstrap = "sh ./bootstrap release -configFile=./config_file"
115     userdata += bootstrap + "\n"
116     userdata += "echo \"Setting 'nfvo.plugin.timeout' to '300000'\"\n"
117     userdata += ("echo \"nfvo.plugin.timeout=600000\" >> "
118                  "/etc/openbaton/openbaton-nfvo.properties\n")
119     userdata += (
120         "wget %s -O /etc/openbaton/openbaton-vnfm-generic-user-data.sh\n" %
121         orchestrator['gvnfm']['userdata']['url'])
122     userdata += "sed -i '113i"'\ \ \ \ '"sleep 60' " \
123                 "/etc/openbaton/openbaton-vnfm-generic-user-data.sh\n"
124     userdata += ("sed -i s/nfvo.marketplace.port=8082/nfvo.marketplace."
125                  "port=8080/g /etc/openbaton/openbaton-nfvo.properties\n")
126     userdata += "echo \"Starting NFVO\"\n"
127     userdata += "service openbaton-nfvo restart\n"
128     userdata += "echo \"Starting Generic VNFM\"\n"
129     userdata += "service openbaton-vnfm-generic restart\n"
130     userdata += "echo \"...end of userdata...\"\n"
131     return userdata
132
133
134 class OpenImsVnf(vnf.VnfOnBoarding):
135     """OpenIMS VNF deployed with openBaton orchestrator"""
136
137     # logger = logging.getLogger(__name__)
138
139     def __init__(self, **kwargs):
140         if "case_name" not in kwargs:
141             kwargs["case_name"] = "orchestra_openims"
142         super(OpenImsVnf, self).__init__(**kwargs)
143         self.logger = logging.getLogger("functest.ci.run_tests.orchestra")
144         self.logger.info("kwargs %s", (kwargs))
145
146         self.case_dir = pkg_resources.resource_filename(
147             'functest', 'opnfv_tests/vnf/ims/')
148         self.data_dir = getattr(config.CONF, 'dir_ims_data')
149         self.test_dir = getattr(config.CONF, 'dir_repo_vims_test')
150         self.created_resources = []
151         self.logger.info("%s VNF onboarding test starting", self.case_name)
152
153         try:
154             self.config = getattr(
155                 config.CONF, 'vnf_{}_config'.format(self.case_name))
156         except BaseException:
157             raise Exception("Orchestra VNF config file not found")
158         config_file = self.case_dir + self.config
159
160         self.mano = dict(
161             get_config("mano", config_file),
162             details={}
163         )
164         self.logger.debug("Orchestrator configuration %s", self.mano)
165
166         self.details['orchestrator'] = dict(
167             name=self.mano['name'],
168             version=self.mano['version'],
169             status='ERROR',
170             result=''
171         )
172
173         self.vnf = dict(
174             get_config(self.case_name, config_file),
175         )
176         self.logger.debug("VNF configuration: %s", self.vnf)
177
178         self.details['vnf'] = dict(
179             name=self.vnf['name'],
180         )
181
182         self.details['test_vnf'] = dict(
183             name=self.case_name,
184         )
185
186         # Orchestra base Data directory creation
187         if not os.path.exists(self.data_dir):
188             os.makedirs(self.data_dir)
189
190         self.images = get_config("tenant_images.orchestrator", config_file)
191         self.images.update(get_config("tenant_images.%s" %
192                                       self.case_name, config_file))
193         self.creds = None
194         self.orchestra_router = None
195
196     def prepare(self):
197         """Prepare testscase (Additional pre-configuration steps)."""
198         super(OpenImsVnf, self).prepare()
199
200         public_auth_url = keystone_utils.get_endpoint(
201             self.snaps_creds, 'identity')
202
203         self.logger.info("Additional pre-configuration steps")
204         self.creds = {
205             "tenant": self.snaps_creds.project_name,
206             "username": self.snaps_creds.username,
207             "password": self.snaps_creds.password,
208             "auth_url": public_auth_url}
209         self.prepare_images()
210         self.prepare_flavor()
211         self.prepare_security_groups()
212         self.prepare_network()
213
214     def prepare_images(self):
215         """Upload images if they doen't exist yet"""
216         self.logger.info("Upload images if they doen't exist yet")
217         for image_name, image_file in self.images.iteritems():
218             self.logger.info("image: %s, file: %s", image_name, image_file)
219             if image_file and image_name:
220                 image = OpenStackImage(
221                     self.snaps_creds,
222                     ImageConfig(name=image_name,
223                                 image_user='cloud',
224                                 img_format='qcow2',
225                                 image_file=image_file,
226                                 public=True))
227                 image.create()
228                 self.created_resources.append(image)
229
230     def prepare_security_groups(self):
231         """Create Open Baton security group if it doesn't exist yet"""
232         self.logger.info(
233             "Creating security group for Open Baton if not yet existing...")
234         sg_rules = list()
235         sg_rules.append(
236             SecurityGroupRuleConfig(
237                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
238                     self.uuid),
239                 direction=Direction.ingress,
240                 protocol=Protocol.tcp,
241                 port_range_min=1,
242                 port_range_max=65535))
243         sg_rules.append(
244             SecurityGroupRuleConfig(
245                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
246                     self.uuid),
247                 direction=Direction.egress,
248                 protocol=Protocol.tcp,
249                 port_range_min=1,
250                 port_range_max=65535))
251         sg_rules.append(
252             SecurityGroupRuleConfig(
253                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
254                     self.uuid),
255                 direction=Direction.ingress,
256                 protocol=Protocol.udp,
257                 port_range_min=1,
258                 port_range_max=65535))
259         sg_rules.append(
260             SecurityGroupRuleConfig(
261                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
262                     self.uuid),
263                 direction=Direction.egress,
264                 protocol=Protocol.udp,
265                 port_range_min=1,
266                 port_range_max=65535))
267         security_group = OpenStackSecurityGroup(
268             self.snaps_creds,
269             SecurityGroupConfig(
270                 name="orchestra-sec-group-allowall-{}".format(
271                     self.uuid),
272                 rule_settings=sg_rules))
273
274         security_group_info = security_group.create()
275         self.created_resources.append(security_group)
276         self.mano['details']['sec_group'] = security_group_info.name
277         self.logger.info(
278             "Security group orchestra-sec-group-allowall prepared")
279
280     def prepare_flavor(self):
281         """Create Open Baton flavor if it doesn't exist yet"""
282         self.logger.info(
283             "Create Flavor for Open Baton NFVO if not yet existing")
284
285         flavor_settings = FlavorConfig(
286             name=self.mano['requirements']['flavor']['name'],
287             ram=self.mano['requirements']['flavor']['ram_min'],
288             disk=self.mano['requirements']['flavor']['disk'],
289             vcpus=self.mano['requirements']['flavor']['vcpus'])
290         flavor = OpenStackFlavor(self.snaps_creds, flavor_settings)
291         flavor_info = flavor.create()
292         self.created_resources.append(flavor)
293         self.mano['details']['flavor'] = {}
294         self.mano['details']['flavor']['name'] = flavor_settings.name
295         self.mano['details']['flavor']['id'] = flavor_info.id
296
297     def prepare_network(self):
298         """Create network/subnet/router if they doen't exist yet"""
299         self.logger.info(
300             "Creating network/subnet/router if they doen't exist yet...")
301         subnet_settings = SubnetConfig(
302             name='{}_subnet-{}'.format(self.case_name, self.uuid),
303             cidr="192.168.100.0/24")
304         network_settings = NetworkConfig(
305             name='{}_net-{}'.format(self.case_name, self.uuid),
306             subnet_settings=[subnet_settings])
307         orchestra_network = OpenStackNetwork(
308             self.snaps_creds, network_settings)
309         orchestra_network_info = orchestra_network.create()
310         self.mano['details']['network'] = {}
311         self.mano['details']['network']['id'] = orchestra_network_info.id
312         self.mano['details']['network']['name'] = orchestra_network_info.name
313         self.mano['details']['external_net_name'] = \
314             snaps_utils.get_ext_net_name(self.snaps_creds)
315         self.created_resources.append(orchestra_network)
316         self.orchestra_router = OpenStackRouter(
317             self.snaps_creds,
318             RouterConfig(
319                 name='{}_router-{}'.format(self.case_name, self.uuid),
320                 external_gateway=self.mano['details']['external_net_name'],
321                 internal_subnets=[
322                     subnet_settings.name]))
323         self.orchestra_router.create()
324         self.created_resources.append(self.orchestra_router)
325         self.logger.info("Created network and router for Open Baton NFVO...")
326
327     def get_vim_descriptor(self):
328         """"Create VIM descriptor to be used for onboarding"""
329         self.logger.info(
330             "Building VIM descriptor with PoP creds: %s",
331             self.creds)
332         self.logger.debug("VIM project/tenant id: %s",
333                           self.snaps_creds.project_name)
334         keystone = keystone_utils.keystone_client(self.snaps_creds)
335         project_id = keystone_utils.get_project(
336             keystone=keystone, project_name=self.snaps_creds.project_name).id
337         vim_json = {
338             "name": "vim-instance",
339             "authUrl": self.creds.get("auth_url"),
340             "tenant": project_id,
341             "username": self.creds.get("username"),
342             "password": self.creds.get("password"),
343             "securityGroups": [
344                 self.mano['details']['sec_group']
345             ],
346             "type": "openstack",
347             "location": {
348                 "name": "opnfv",
349                 "latitude": "52.525876",
350                 "longitude": "13.314400"
351             }
352         }
353         self.logger.info("Built VIM descriptor: %s", vim_json)
354         return vim_json
355
356     def deploy_orchestrator(self):
357         self.logger.info("Deploying Open Baton...")
358         self.logger.info("Details: %s", self.mano['details'])
359         start_time = time.time()
360
361         self.logger.info("Creating orchestra instance...")
362         userdata = get_userdata(self.mano)
363         self.logger.info("flavor: %s\n"
364                          "image: %s\n"
365                          "network_id: %s\n",
366                          self.mano['details']['flavor']['name'],
367                          self.mano['requirements']['image'],
368                          self.mano['details']['network']['id'])
369         self.logger.debug("userdata: %s\n", userdata)
370         # setting up image
371         image_settings = ImageConfig(
372             name=self.mano['requirements']['image'],
373             image_user='ubuntu',
374             exists=True)
375         # setting up port
376         port_settings = PortConfig(
377             name='{}_port-{}'.format(self.case_name, self.uuid),
378             network_name=self.mano['details']['network']['name'])
379         # build configuration of vm
380         orchestra_settings = VmInstanceConfig(
381             name='{}-{}'.format(self.case_name, self.uuid),
382             flavor=self.mano['details']['flavor']['name'],
383             port_settings=[port_settings],
384             security_group_names=[self.mano['details']['sec_group']],
385             floating_ip_settings=[FloatingIpConfig(
386                 name='orchestra_fip-{}'.format(self.uuid),
387                 port_name=port_settings.name,
388                 router_name=self.orchestra_router.router_settings.name)],
389             userdata=str(userdata))
390         orchestra_vm = OpenStackVmInstance(
391             self.snaps_creds, orchestra_settings, image_settings)
392         orchestra_vm.create()
393         self.mano['details']['fip'] = orchestra_vm.get_floating_ip()
394         self.created_resources.append(orchestra_vm)
395         self.mano['details']['id'] = orchestra_vm.get_vm_info()['id']
396         self.logger.info(
397             "Created orchestra instance: %s", self.mano['details']['id'])
398         self.logger.info("Waiting for Open Baton NFVO to be up and running...")
399         timeout = 0
400         while timeout < 20:
401             if servertest(
402                     self.mano['details']['fip'].ip,
403                     "8080"):
404                 break
405             else:
406                 self.logger.info("Open Baton NFVO is not started yet (%ss)",
407                                  (timeout * 60))
408                 time.sleep(60)
409                 timeout += 1
410
411         if timeout >= 20:
412             duration = time.time() - start_time
413             self.details["orchestrator"].update(
414                 status='FAIL', duration=duration)
415             self.logger.error("Open Baton is not started correctly")
416             return False
417
418         self.logger.info("Waiting for all components to be up and running...")
419         time.sleep(60)
420         duration = time.time() - start_time
421         self.details["orchestrator"].update(status='PASS', duration=duration)
422         self.logger.info("Deploy Open Baton NFVO: OK")
423         return True
424
425     def deploy_vnf(self):
426         start_time = time.time()
427         self.logger.info("Deploying %s...", self.vnf['name'])
428
429         main_agent = MainAgent(
430             nfvo_ip=self.mano['details']['fip'].ip,
431             nfvo_port=8080,
432             https=False,
433             version=1,
434             username=self.mano['credentials']['username'],
435             password=self.mano['credentials']['password'])
436
437         self.logger.info(
438             "Create %s Flavor if not existing", self.vnf['name'])
439         flavor_settings = FlavorConfig(
440             name=self.vnf['requirements']['flavor']['name'],
441             ram=self.vnf['requirements']['flavor']['ram_min'],
442             disk=self.vnf['requirements']['flavor']['disk'],
443             vcpus=self.vnf['requirements']['flavor']['vcpus'])
444         flavor = OpenStackFlavor(self.snaps_creds, flavor_settings)
445         flavor_info = flavor.create()
446         self.logger.debug("Flavor id: %s", flavor_info.id)
447
448         self.logger.info("Getting project 'default'...")
449         project_agent = main_agent.get_agent("project", "")
450         for project in json.loads(project_agent.find()):
451             if project.get("name") == "default":
452                 self.mano['details']['project_id'] = project.get("id")
453                 self.logger.info("Found project 'default': %s", project)
454                 break
455
456         vim_json = self.get_vim_descriptor()
457         self.logger.info("Registering VIM: %s", vim_json)
458
459         main_agent.get_agent(
460             "vim", project_id=self.mano['details']['project_id']).create(
461                 entity=json.dumps(vim_json))
462
463         market_agent = main_agent.get_agent(
464             "market", project_id=self.mano['details']['project_id'])
465
466         try:
467             self.logger.info("sending: %s", self.vnf['descriptor']['url'])
468             nsd = market_agent.create(entity=self.vnf['descriptor']['url'])
469             if nsd.get('id') is None:
470                 self.logger.error("NSD not onboarded correctly")
471                 duration = time.time() - start_time
472                 self.details["vnf"].update(status='FAIL', duration=duration)
473                 return False
474             self.mano['details']['nsd_id'] = nsd.get('id')
475             self.logger.info("Onboarded NSD: " + nsd.get("name"))
476
477             nsr_agent = main_agent.get_agent(
478                 "nsr", project_id=self.mano['details']['project_id'])
479
480             self.mano['details']['nsr'] = nsr_agent.create(
481                 self.mano['details']['nsd_id'])
482         except NfvoException:
483             self.logger.exception("failed")
484             duration = time.time() - start_time
485             self.details["vnf"].update(status='FAIL', duration=duration)
486             return False
487
488         if self.mano['details']['nsr'].get('code') is not None:
489             self.logger.error(
490                 "%s cannot be deployed: %s -> %s",
491                 self.vnf['name'],
492                 self.mano['details']['nsr'].get('code'),
493                 self.mano['details']['nsr'].get('message'))
494             self.logger.error("%s cannot be deployed", self.vnf['name'])
495             duration = time.time() - start_time
496             self.details["vnf"].update(status='FAIL', duration=duration)
497             return False
498
499         timeout = 0
500         self.logger.info("Waiting for NSR to go to ACTIVE...")
501         while self.mano['details']['nsr'].get("status") != 'ACTIVE' \
502                 and self.mano['details']['nsr'].get("status") != 'ERROR':
503             timeout += 1
504             self.logger.info("NSR is not yet ACTIVE... (%ss)", 60 * timeout)
505             if timeout == 30:
506                 self.logger.error("INACTIVE NSR after %s sec..", 60 * timeout)
507                 duration = time.time() - start_time
508                 self.details["vnf"].update(status='FAIL', duration=duration)
509                 return False
510             time.sleep(60)
511             self.mano['details']['nsr'] = json.loads(
512                 nsr_agent.find(self.mano['details']['nsr'].get('id')))
513
514         duration = time.time() - start_time
515         if self.mano['details']['nsr'].get("status") == 'ACTIVE':
516             self.details["vnf"].update(status='PASS', duration=duration)
517             self.logger.info("Sleep for 60s to ensure that all "
518                              "services are up and running...")
519             time.sleep(60)
520             result = True
521         else:
522             self.details["vnf"].update(status='FAIL', duration=duration)
523             self.logger.error("NSR: %s", self.mano['details'].get('nsr'))
524             result = False
525         return result
526
527     def test_vnf(self):
528         self.logger.info("Testing VNF OpenIMS...")
529         start_time = time.time()
530         self.logger.info(
531             "Testing if %s works properly...",
532             self.mano['details']['nsr'].get('name'))
533         for vnfr in self.mano['details']['nsr'].get('vnfr'):
534             self.logger.info(
535                 "Checking ports %s of VNF %s",
536                 self.vnf['test'][vnfr.get('name')]['ports'],
537                 vnfr.get('name'))
538             for vdu in vnfr.get('vdu'):
539                 for vnfci in vdu.get('vnfc_instance'):
540                     self.logger.debug(
541                         "Checking ports of VNFC instance %s",
542                         vnfci.get('hostname'))
543                     for floating_ip in vnfci.get('floatingIps'):
544                         self.logger.debug(
545                             "Testing %s:%s",
546                             vnfci.get('hostname'),
547                             floating_ip.get('ip'))
548                         for port in self.vnf['test'][vnfr.get(
549                                 'name')]['ports']:
550                             if servertest(floating_ip.get('ip'), port):
551                                 self.logger.info(
552                                     "VNFC instance %s is reachable at %s:%s",
553                                     vnfci.get('hostname'),
554                                     floating_ip.get('ip'),
555                                     port)
556                             else:
557                                 self.logger.error(
558                                     "VNFC instance %s is not reachable "
559                                     "at %s:%s",
560                                     vnfci.get('hostname'),
561                                     floating_ip.get('ip'),
562                                     port)
563                                 duration = time.time() - start_time
564                                 self.details["test_vnf"].update(
565                                     status='FAIL', duration=duration, esult=(
566                                         "Port %s of server %s -> %s is "
567                                         "not reachable",
568                                         port,
569                                         vnfci.get('hostname'),
570                                         floating_ip.get('ip')))
571                                 self.logger.error("Test VNF: ERROR")
572                                 return False
573         duration = time.time() - start_time
574         self.details["test_vnf"].update(status='PASS', duration=duration)
575         self.logger.info("Test VNF: OK")
576         return True
577
578     def clean(self):
579         self.logger.info("Cleaning %s...", self.case_name)
580         try:
581             main_agent = MainAgent(
582                 nfvo_ip=self.mano['details']['fip'].ip,
583                 nfvo_port=8080, https=False, version=1,
584                 username=self.mano['credentials']['username'],
585                 password=self.mano['credentials']['password'])
586             self.logger.info("Terminating %s...", self.vnf['name'])
587             if self.mano['details'].get('nsr'):
588                 main_agent.get_agent(
589                     "nsr",
590                     project_id=self.mano['details']['project_id']).\
591                         delete(self.mano['details']['nsr'].get('id'))
592                 self.logger.info("Sleeping 60 seconds...")
593                 time.sleep(60)
594             else:
595                 self.logger.info("No need to terminate the VNF...")
596         except (NfvoException, KeyError) as exc:
597             self.logger.error('Unexpected error cleaning - %s', exc)
598         super(OpenImsVnf, self).clean()