2b458e8c1415a8e5da90318323cdb77e5d666ccc
[functest.git] / functest / opnfv_tests / vnf / ims / orchestra_clearwaterims.py
1 #!/usr/bin/env python
2
3 # Copyright (c) 2016 Orange and others.
4 #
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9
10 """Orchestra Clearwater IMS testcase implementation."""
11
12 import json
13 import logging
14 import os
15 import socket
16 import time
17
18 import pkg_resources
19 import yaml
20
21 from functest.core import vnf
22 from functest.opnfv_tests.openstack.snaps import snaps_utils
23 from functest.utils import config
24 from functest.utils import env
25
26 from org.openbaton.cli.errors.errors import NfvoException
27 from org.openbaton.cli.agents.agents import MainAgent
28 from snaps.config.flavor import FlavorConfig
29 from snaps.config.image import ImageConfig
30 from snaps.config.network import NetworkConfig, PortConfig, SubnetConfig
31 from snaps.config.router import RouterConfig
32 from snaps.config.security_group import (
33     Direction, Protocol, SecurityGroupConfig, SecurityGroupRuleConfig)
34 from snaps.config.vm_inst import FloatingIpConfig
35 from snaps.config.vm_inst import VmInstanceConfig
36 from snaps.openstack.utils import keystone_utils
37 from snaps.openstack.create_flavor import OpenStackFlavor
38 from snaps.openstack.create_image import OpenStackImage
39 from snaps.openstack.create_instance import OpenStackVmInstance
40 from snaps.openstack.create_network import OpenStackNetwork
41 from snaps.openstack.create_router import OpenStackRouter
42 from snaps.openstack.create_security_group import OpenStackSecurityGroup
43
44
45 __author__ = "Pauls, Michael <michael.pauls@fokus.fraunhofer.de>"
46 # ----------------------------------------------------------
47 #
48 #               UTILS
49 #
50 # -----------------------------------------------------------
51
52
53 def get_config(parameter, file_path):
54     """
55     Get config parameter.
56
57     Returns the value of a given parameter in file.yaml
58     parameter must be given in string format with dots
59     Example: general.openstack.image_name
60     """
61     with open(file_path) as config_file:
62         file_yaml = yaml.safe_load(config_file)
63     config_file.close()
64     value = file_yaml
65     for element in parameter.split("."):
66         value = value.get(element)
67         if value is None:
68             raise ValueError("The parameter %s is not defined in"
69                              " reporting.yaml", parameter)
70     return value
71
72
73 def servertest(host, port):
74     """Method to test that a server is reachable at IP:port"""
75     args = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM)
76     for family, socktype, proto, _, sockaddr in args:
77         sock = socket.socket(family, socktype, proto)
78         try:
79             sock.connect(sockaddr)
80         except socket.error:
81             return False
82         else:
83             sock.close()
84             return True
85
86
87 def get_userdata(orchestrator=dict):
88     """Build userdata for Open Baton machine"""
89     userdata = "#!/bin/bash\n"
90     userdata += "echo \"Executing userdata...\"\n"
91     userdata += "set -x\n"
92     userdata += "set -e\n"
93     userdata += "echo \"Install curl...\"\n"
94     userdata += "apt-get install curl\n"
95     userdata += "echo \"Inject public key...\"\n"
96     userdata += ("echo \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCuPXrV3"
97                  "geeHc6QUdyUr/1Z+yQiqLcOskiEGBiXr4z76MK4abiFmDZ18OMQlc"
98                  "fl0p3kS0WynVgyaOHwZkgy/DIoIplONVr2CKBKHtPK+Qcme2PVnCtv"
99                  "EqItl/FcD+1h5XSQGoa+A1TSGgCod/DPo+pes0piLVXP8Ph6QS1k7S"
100                  "ic7JDeRQ4oT1bXYpJ2eWBDMfxIWKZqcZRiGPgMIbJ1iEkxbpeaAd9O"
101                  "4MiM9nGCPESmed+p54uYFjwEDlAJZShcAZziiZYAvMZhvAhe6USljc"
102                  "7YAdalAnyD/jwCHuwIrUw/lxo7UdNCmaUxeobEYyyFA1YVXzpNFZya"
103                  "XPGAAYIJwEq/ openbaton@opnfv\" >> /home/ubuntu/.ssh/aut"
104                  "horized_keys\n")
105     userdata += "echo \"Download bootstrap...\"\n"
106     userdata += ("curl -s %s "
107                  "> ./bootstrap\n" % orchestrator['bootstrap']['url'])
108     userdata += ("curl -s %s" "> ./config_file\n" %
109                  orchestrator['bootstrap']['config']['url'])
110     userdata += ("echo \"Disable usage of mysql...\"\n")
111     userdata += "sed -i s/mysql=.*/mysql=no/g /config_file\n"
112     userdata += "echo \"Set autostart of components to 'false'\"\n"
113     userdata += "export OPENBATON_COMPONENT_AUTOSTART=false\n"
114     userdata += "echo \"Execute bootstrap...\"\n"
115     bootstrap = "sh ./bootstrap release -configFile=./config_file"
116     userdata += bootstrap + "\n"
117     userdata += "echo \"Setting 'nfvo.plugin.timeout' to '300000'\"\n"
118     userdata += ("echo \"nfvo.plugin.timeout=600000\" >> "
119                  "/etc/openbaton/openbaton-nfvo.properties\n")
120     userdata += (
121         "wget %s -O /etc/openbaton/openbaton-vnfm-generic-user-data.sh\n" %
122         orchestrator['gvnfm']['userdata']['url'])
123     userdata += "sed -i '113i"'\ \ \ \ '"sleep 60' " \
124                 "/etc/openbaton/openbaton-vnfm-generic-user-data.sh\n"
125     userdata += ("sed -i s/nfvo.marketplace.port=8082/nfvo.marketplace."
126                  "port=8080/g /etc/openbaton/openbaton-nfvo.properties\n")
127     userdata += "echo \"Starting NFVO\"\n"
128     userdata += "service openbaton-nfvo restart\n"
129     userdata += "echo \"Starting Generic VNFM\"\n"
130     userdata += "service openbaton-vnfm-generic restart\n"
131     userdata += "echo \"...end of userdata...\"\n"
132     return userdata
133
134
135 class ClearwaterImsVnf(vnf.VnfOnBoarding):
136     """Clearwater IMS VNF deployed with openBaton orchestrator"""
137
138     # logger = logging.getLogger(__name__)
139
140     def __init__(self, **kwargs):
141         if "case_name" not in kwargs:
142             kwargs["case_name"] = "orchestra_clearwaterims"
143         super(ClearwaterImsVnf, self).__init__(**kwargs)
144         self.logger = logging.getLogger("functest.ci.run_tests.orchestra")
145         self.logger.info("kwargs %s", (kwargs))
146
147         self.case_dir = pkg_resources.resource_filename(
148             'functest', 'opnfv_tests/vnf/ims/')
149         self.data_dir = getattr(config.CONF, 'dir_ims_data')
150         self.test_dir = getattr(config.CONF, 'dir_repo_vims_test')
151         self.created_resources = []
152         self.logger.info("%s VNF onboarding test starting", self.case_name)
153
154         try:
155             self.config = getattr(
156                 config.CONF, 'vnf_{}_config'.format(self.case_name))
157         except BaseException:
158             raise Exception("Orchestra VNF config file not found")
159
160         config_file = self.case_dir + self.config
161
162         self.mano = dict(
163             get_config("mano", config_file),
164             details={}
165         )
166         self.logger.debug("Orchestrator configuration %s", self.mano)
167
168         self.details['orchestrator'] = dict(
169             name=self.mano['name'],
170             version=self.mano['version'],
171             status='ERROR',
172             result=''
173         )
174
175         self.vnf = dict(
176             get_config(self.case_name, config_file),
177         )
178         self.logger.debug("VNF configuration: %s", self.vnf)
179
180         self.details['vnf'] = dict(
181             name=self.vnf['name'],
182         )
183
184         self.details['test_vnf'] = dict(
185             name=self.case_name,
186         )
187
188         # Orchestra base Data directory creation
189         if not os.path.exists(self.data_dir):
190             os.makedirs(self.data_dir)
191
192         self.images = get_config("tenant_images.orchestrator", config_file)
193         self.images.update(
194             get_config(
195                 "tenant_images.%s" %
196                 self.case_name,
197                 config_file))
198         self.creds = None
199         self.orchestra_router = None
200
201     def prepare(self):
202         """Prepare testscase (Additional pre-configuration steps)."""
203         super(ClearwaterImsVnf, self).prepare()
204
205         self.logger.info("Additional pre-configuration steps")
206
207         public_auth_url = keystone_utils.get_endpoint(
208             self.snaps_creds, 'identity')
209         self.creds = {
210             "tenant": self.snaps_creds.project_name,
211             "username": self.snaps_creds.username,
212             "password": self.snaps_creds.password,
213             "auth_url": public_auth_url}
214         self.prepare_images()
215         self.prepare_flavor()
216         self.prepare_security_groups()
217         self.prepare_network()
218
219     def prepare_images(self):
220         """Upload images if they doen't exist yet"""
221         self.logger.info("Upload images if they doen't exist yet")
222         for image_name, image_file in self.images.iteritems():
223             self.logger.info("image: %s, file: %s", image_name, image_file)
224             if image_file and image_name:
225                 image = OpenStackImage(
226                     self.snaps_creds,
227                     ImageConfig(name=image_name,
228                                 image_user='cloud',
229                                 img_format='qcow2',
230                                 image_file=image_file,
231                                 public=True))
232                 image.create()
233                 self.created_resources.append(image)
234
235     def prepare_security_groups(self):
236         """Create Open Baton security group if it doesn't exist yet"""
237         self.logger.info(
238             "Creating security group for Open Baton if not yet existing...")
239         sg_rules = list()
240         sg_rules.append(
241             SecurityGroupRuleConfig(
242                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
243                     self.uuid),
244                 direction=Direction.ingress,
245                 protocol=Protocol.tcp,
246                 port_range_min=1,
247                 port_range_max=65535))
248         sg_rules.append(
249             SecurityGroupRuleConfig(
250                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
251                     self.uuid),
252                 direction=Direction.egress,
253                 protocol=Protocol.tcp,
254                 port_range_min=1,
255                 port_range_max=65535))
256         sg_rules.append(
257             SecurityGroupRuleConfig(
258                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
259                     self.uuid),
260                 direction=Direction.ingress,
261                 protocol=Protocol.udp,
262                 port_range_min=1,
263                 port_range_max=65535))
264         sg_rules.append(
265             SecurityGroupRuleConfig(
266                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
267                     self.uuid),
268                 direction=Direction.egress,
269                 protocol=Protocol.udp,
270                 port_range_min=1,
271                 port_range_max=65535))
272         sg_rules.append(
273             SecurityGroupRuleConfig(
274                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
275                     self.uuid),
276                 direction=Direction.ingress,
277                 protocol=Protocol.icmp))
278         sg_rules.append(
279             SecurityGroupRuleConfig(
280                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
281                     self.uuid),
282                 direction=Direction.egress,
283                 protocol=Protocol.icmp))
284         security_group = OpenStackSecurityGroup(
285             self.snaps_creds,
286             SecurityGroupConfig(
287                 name="orchestra-sec-group-allowall-{}".format(
288                     self.uuid),
289                 rule_settings=sg_rules))
290         security_group_info = security_group.create()
291         self.created_resources.append(security_group)
292         self.mano['details']['sec_group'] = security_group_info.name
293         self.logger.info(
294             "Security group orchestra-sec-group-allowall prepared")
295
296     def prepare_flavor(self):
297         """Create Open Baton flavor if it doesn't exist yet"""
298         self.logger.info(
299             "Create Flavor for Open Baton NFVO if not yet existing")
300
301         flavor_settings = FlavorConfig(
302             name=self.mano['requirements']['flavor']['name'],
303             ram=self.mano['requirements']['flavor']['ram_min'],
304             disk=self.mano['requirements']['flavor']['disk'],
305             vcpus=self.mano['requirements']['flavor']['vcpus'])
306         flavor = OpenStackFlavor(self.snaps_creds, flavor_settings)
307         flavor_info = flavor.create()
308         self.created_resources.append(flavor)
309         self.mano['details']['flavor'] = {}
310         self.mano['details']['flavor']['name'] = flavor_settings.name
311         self.mano['details']['flavor']['id'] = flavor_info.id
312
313     def prepare_network(self):
314         """Create network/subnet/router if they doen't exist yet"""
315         self.logger.info(
316             "Creating network/subnet/router if they doen't exist yet...")
317         subnet_settings = SubnetConfig(
318             name='{}_subnet-{}'.format(self.case_name, self.uuid),
319             cidr="192.168.100.0/24",
320             dns_nameservers=[env.get('NAMESERVER')])
321         network_settings = NetworkConfig(
322             name='{}_net-{}'.format(self.case_name, self.uuid),
323             subnet_settings=[subnet_settings])
324         orchestra_network = OpenStackNetwork(
325             self.snaps_creds, network_settings)
326         orchestra_network_info = orchestra_network.create()
327         self.mano['details']['network'] = {}
328         self.mano['details']['network']['id'] = orchestra_network_info.id
329         self.mano['details']['network']['name'] = orchestra_network_info.name
330         self.mano['details']['external_net_name'] = snaps_utils.\
331             get_ext_net_name(self.snaps_creds)
332         self.created_resources.append(orchestra_network)
333         self.orchestra_router = OpenStackRouter(
334             self.snaps_creds,
335             RouterConfig(
336                 name='{}_router-{}'.format(self.case_name, self.uuid),
337                 external_gateway=self.mano['details']['external_net_name'],
338                 internal_subnets=[
339                     subnet_settings.name]))
340         self.orchestra_router.create()
341         self.created_resources.append(self.orchestra_router)
342         self.logger.info("Created network and router for Open Baton NFVO...")
343
344     def get_vim_descriptor(self):
345         """"Create VIM descriptor to be used for onboarding"""
346         self.logger.info(
347             "Building VIM descriptor with PoP creds: %s",
348             self.creds)
349         self.logger.debug("VIM project/tenant id: %s",
350                           self.snaps_creds.project_name)
351         keystone = keystone_utils.keystone_client(self.snaps_creds)
352         project_id = keystone_utils.get_project(
353             keystone=keystone, project_name=self.snaps_creds.project_name).id
354         vim_json = {
355             "name": "vim-instance",
356             "authUrl": self.creds.get("auth_url"),
357             "tenant": project_id,
358             "username": self.creds.get("username"),
359             "password": self.creds.get("password"),
360             "securityGroups": [
361                 self.mano['details']['sec_group']
362             ],
363             "type": "openstack",
364             "location": {
365                 "name": "opnfv",
366                 "latitude": "52.525876",
367                 "longitude": "13.314400"
368             }
369         }
370         self.logger.info("Built VIM descriptor: %s", vim_json)
371         return vim_json
372
373     def deploy_orchestrator(self):
374         self.logger.info("Deploying Open Baton...")
375         self.logger.info("Details: %s", self.mano['details'])
376         start_time = time.time()
377
378         self.logger.info("Creating orchestra instance...")
379         userdata = get_userdata(self.mano)
380         self.logger.info("flavor: %s\n"
381                          "image: %s\n"
382                          "network_id: %s\n",
383                          self.mano['details']['flavor']['name'],
384                          self.mano['requirements']['image'],
385                          self.mano['details']['network']['id'])
386         self.logger.debug("userdata: %s\n", userdata)
387         # setting up image
388         image_settings = ImageConfig(
389             name=self.mano['requirements']['image'],
390             image_user='ubuntu',
391             exists=True)
392         # setting up port
393         port_settings = PortConfig(
394             name='{}_port-{}'.format(self.case_name, self.uuid),
395             network_name=self.mano['details']['network']['name'])
396
397         # build configuration of vm
398         orchestra_settings = VmInstanceConfig(
399             name=self.case_name,
400             flavor=self.mano['details']['flavor']['name'],
401             port_settings=[port_settings],
402             security_group_names=[self.mano['details']['sec_group']],
403             floating_ip_settings=[FloatingIpConfig(
404                 name='orchestra_fip-{}'.format(self.uuid),
405                 port_name=port_settings.name,
406                 router_name=self.orchestra_router.router_settings.name)],
407             userdata=str(userdata))
408         orchestra_vm = OpenStackVmInstance(
409             self.snaps_creds, orchestra_settings, image_settings)
410         orchestra_vm.create()
411         self.mano['details']['fip'] = orchestra_vm.get_floating_ip()
412         self.created_resources.append(orchestra_vm)
413         self.mano['details']['id'] = orchestra_vm.get_vm_info()['id']
414         self.logger.info(
415             "Created orchestra instance: %s", self.mano['details']['id'])
416         self.logger.info("Waiting for Open Baton NFVO to be up and running...")
417         timeout = 0
418         while timeout < 20:
419             if servertest(
420                     self.mano['details']['fip'].ip,
421                     "8080"):
422                 break
423             else:
424                 self.logger.info(
425                     "Open Baton NFVO is not started yet (%ss)",
426                     (timeout * 60))
427                 time.sleep(60)
428                 timeout += 1
429
430         if timeout >= 20:
431             duration = time.time() - start_time
432             self.details["orchestrator"].update(
433                 status='FAIL', duration=duration)
434             self.logger.error("Open Baton is not started correctly")
435             return False
436
437         self.logger.info("Waiting for all components to be up and running...")
438         time.sleep(60)
439         duration = time.time() - start_time
440         self.details["orchestrator"].update(status='PASS', duration=duration)
441         self.logger.info("Deploy Open Baton NFVO: OK")
442         return True
443
444     def deploy_vnf(self):
445         start_time = time.time()
446         self.logger.info("Deploying %s...", self.vnf['name'])
447
448         main_agent = MainAgent(
449             nfvo_ip=self.mano['details']['fip'].ip,
450             nfvo_port=8080,
451             https=False,
452             version=1,
453             username=self.mano['credentials']['username'],
454             password=self.mano['credentials']['password'])
455
456         self.logger.info(
457             "Create %s Flavor if not existing", self.vnf['name'])
458         flavor_settings = FlavorConfig(
459             name=self.vnf['requirements']['flavor']['name'],
460             ram=self.vnf['requirements']['flavor']['ram_min'],
461             disk=self.vnf['requirements']['flavor']['disk'],
462             vcpus=self.vnf['requirements']['flavor']['vcpus'])
463         flavor = OpenStackFlavor(self.snaps_creds, flavor_settings)
464         flavor_info = flavor.create()
465         self.logger.debug("Flavor id: %s", flavor_info.id)
466
467         self.logger.info("Getting project 'default'...")
468         project_agent = main_agent.get_agent("project", "")
469         for project in json.loads(project_agent.find()):
470             if project.get("name") == "default":
471                 self.mano['details']['project_id'] = project.get("id")
472                 self.logger.info("Found project 'default': %s", project)
473                 break
474
475         vim_json = self.get_vim_descriptor()
476         self.logger.info("Registering VIM: %s", vim_json)
477
478         main_agent.get_agent(
479             "vim", project_id=self.mano['details']['project_id']).create(
480                 entity=json.dumps(vim_json))
481
482         market_agent = main_agent.get_agent(
483             "market", project_id=self.mano['details']['project_id'])
484
485         try:
486             self.logger.info("sending: %s", self.vnf['descriptor']['url'])
487             nsd = market_agent.create(entity=self.vnf['descriptor']['url'])
488             if nsd.get('id') is None:
489                 self.logger.error("NSD not onboarded correctly")
490                 duration = time.time() - start_time
491                 self.details["vnf"].update(status='FAIL', duration=duration)
492                 return False
493             self.mano['details']['nsd_id'] = nsd.get('id')
494             self.logger.info("Onboarded NSD: " + nsd.get("name"))
495
496             nsr_agent = main_agent.get_agent(
497                 "nsr", project_id=self.mano['details']['project_id'])
498
499             self.mano['details']['nsr'] = nsr_agent.create(
500                 self.mano['details']['nsd_id'])
501         except NfvoException:
502             self.logger.exception("failed")
503             duration = time.time() - start_time
504             self.details["vnf"].update(status='FAIL', duration=duration)
505             return False
506
507         if self.mano['details']['nsr'].get('code') is not None:
508             self.logger.error(
509                 "%s cannot be deployed: %s -> %s",
510                 self.vnf['name'],
511                 self.mano['details']['nsr'].get('code'),
512                 self.mano['details']['nsr'].get('message'))
513             self.logger.error("%s cannot be deployed", self.vnf['name'])
514             duration = time.time() - start_time
515             self.details["vnf"].update(status='FAIL', duration=duration)
516             return False
517
518         timeout = 0
519         self.logger.info("Waiting for NSR to go to ACTIVE...")
520         while self.mano['details']['nsr'].get("status") != 'ACTIVE' \
521                 and self.mano['details']['nsr'].get("status") != 'ERROR':
522             timeout += 1
523             self.logger.info("NSR is not yet ACTIVE... (%ss)", 60 * timeout)
524             if timeout == 30:
525                 self.logger.error("INACTIVE NSR after %s sec..", 60 * timeout)
526                 duration = time.time() - start_time
527                 self.details["vnf"].update(status='FAIL', duration=duration)
528                 return False
529             time.sleep(60)
530             self.mano['details']['nsr'] = json.loads(
531                 nsr_agent.find(self.mano['details']['nsr'].get('id')))
532
533         duration = time.time() - start_time
534         if self.mano['details']['nsr'].get("status") == 'ACTIVE':
535             self.details["vnf"].update(status='PASS', duration=duration)
536             self.logger.info("Sleep for 60s to ensure that all "
537                              "services are up and running...")
538             time.sleep(60)
539             result = True
540         else:
541             self.details["vnf"].update(status='FAIL', duration=duration)
542             self.logger.error("NSR: %s", self.mano['details'].get('nsr'))
543             result = False
544         return result
545
546     def test_vnf(self):
547         self.logger.info(
548             "Testing VNF Clearwater IMS is not yet implemented...")
549         start_time = time.time()
550
551         duration = time.time() - start_time
552         self.details["test_vnf"].update(status='PASS', duration=duration)
553         self.logger.info("Test VNF: OK")
554         return True
555
556     def clean(self):
557         self.logger.info("Cleaning %s...", self.case_name)
558         try:
559             main_agent = MainAgent(
560                 nfvo_ip=self.mano['details']['fip'].ip,
561                 nfvo_port=8080, https=False, version=1,
562                 username=self.mano['credentials']['username'],
563                 password=self.mano['credentials']['password'])
564             self.logger.info("Terminating %s...", self.vnf['name'])
565             if self.mano['details'].get('nsr'):
566                 main_agent.get_agent(
567                     "nsr",
568                     project_id=self.mano['details']['project_id']).delete(
569                         self.mano['details']['nsr'].get('id'))
570                 self.logger.info("Sleeping 60 seconds...")
571                 time.sleep(60)
572             else:
573                 self.logger.info("No need to terminate the VNF...")
574         except (NfvoException, KeyError) as exc:
575             self.logger.error('Unexpected error cleaning - %s', exc)
576         super(ClearwaterImsVnf, self).clean()