Switch to snaps in orchestra testcases
[functest.git] / functest / opnfv_tests / vnf / ims / orchestra_clearwaterims.py
1 #!/usr/bin/env python
2
3 # Copyright (c) 2016 Orange and others.
4 #
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9
10 """Orchestra Clearwater IMS testcase implementation."""
11
12 import json
13 import logging
14 import os
15 import socket
16 import time
17
18 import pkg_resources
19 import yaml
20
21 from functest.core import vnf
22 from functest.opnfv_tests.openstack.snaps import snaps_utils
23 from functest.utils import config
24
25 from org.openbaton.cli.errors.errors import NfvoException
26 from org.openbaton.cli.agents.agents import MainAgent
27 from snaps.config.flavor import FlavorConfig
28 from snaps.config.image import ImageConfig
29 from snaps.config.network import NetworkConfig, PortConfig, SubnetConfig
30 from snaps.config.router import RouterConfig
31 from snaps.config.security_group import (
32     Direction, Protocol, SecurityGroupConfig, SecurityGroupRuleConfig)
33 from snaps.config.vm_inst import FloatingIpConfig
34 from snaps.config.vm_inst import VmInstanceConfig
35 from snaps.openstack.utils import keystone_utils
36 from snaps.openstack.create_flavor import OpenStackFlavor
37 from snaps.openstack.create_image import OpenStackImage
38 from snaps.openstack.create_instance import OpenStackVmInstance
39 from snaps.openstack.create_network import OpenStackNetwork
40 from snaps.openstack.create_router import OpenStackRouter
41 from snaps.openstack.create_security_group import OpenStackSecurityGroup
42
43
44 __author__ = "Pauls, Michael <michael.pauls@fokus.fraunhofer.de>"
45 # ----------------------------------------------------------
46 #
47 #               UTILS
48 #
49 # -----------------------------------------------------------
50
51
52 def get_config(parameter, file_path):
53     """
54     Get config parameter.
55
56     Returns the value of a given parameter in file.yaml
57     parameter must be given in string format with dots
58     Example: general.openstack.image_name
59     """
60     with open(file_path) as config_file:
61         file_yaml = yaml.safe_load(config_file)
62     config_file.close()
63     value = file_yaml
64     for element in parameter.split("."):
65         value = value.get(element)
66         if value is None:
67             raise ValueError("The parameter %s is not defined in"
68                              " reporting.yaml", parameter)
69     return value
70
71
72 def servertest(host, port):
73     """Method to test that a server is reachable at IP:port"""
74     args = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM)
75     for family, socktype, proto, _, sockaddr in args:
76         sock = socket.socket(family, socktype, proto)
77         try:
78             sock.connect(sockaddr)
79         except socket.error:
80             return False
81         else:
82             sock.close()
83             return True
84
85
86 def get_userdata(orchestrator=dict):
87     """Build userdata for Open Baton machine"""
88     userdata = "#!/bin/bash\n"
89     userdata += "echo \"Executing userdata...\"\n"
90     userdata += "set -x\n"
91     userdata += "set -e\n"
92     userdata += "echo \"Install curl...\"\n"
93     userdata += "apt-get install curl\n"
94     userdata += "echo \"Inject public key...\"\n"
95     userdata += ("echo \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCuPXrV3"
96                  "geeHc6QUdyUr/1Z+yQiqLcOskiEGBiXr4z76MK4abiFmDZ18OMQlc"
97                  "fl0p3kS0WynVgyaOHwZkgy/DIoIplONVr2CKBKHtPK+Qcme2PVnCtv"
98                  "EqItl/FcD+1h5XSQGoa+A1TSGgCod/DPo+pes0piLVXP8Ph6QS1k7S"
99                  "ic7JDeRQ4oT1bXYpJ2eWBDMfxIWKZqcZRiGPgMIbJ1iEkxbpeaAd9O"
100                  "4MiM9nGCPESmed+p54uYFjwEDlAJZShcAZziiZYAvMZhvAhe6USljc"
101                  "7YAdalAnyD/jwCHuwIrUw/lxo7UdNCmaUxeobEYyyFA1YVXzpNFZya"
102                  "XPGAAYIJwEq/ openbaton@opnfv\" >> /home/ubuntu/.ssh/aut"
103                  "horized_keys\n")
104     userdata += "echo \"Download bootstrap...\"\n"
105     userdata += ("curl -s %s "
106                  "> ./bootstrap\n" % orchestrator['bootstrap']['url'])
107     userdata += ("curl -s %s" "> ./config_file\n" %
108                  orchestrator['bootstrap']['config']['url'])
109     userdata += ("echo \"Disable usage of mysql...\"\n")
110     userdata += "sed -i s/mysql=.*/mysql=no/g /config_file\n"
111     userdata += "echo \"Set autostart of components to 'false'\"\n"
112     userdata += "export OPENBATON_COMPONENT_AUTOSTART=false\n"
113     userdata += "echo \"Execute bootstrap...\"\n"
114     bootstrap = "sh ./bootstrap release -configFile=./config_file"
115     userdata += bootstrap + "\n"
116     userdata += "echo \"Setting 'nfvo.plugin.timeout' to '300000'\"\n"
117     userdata += ("echo \"nfvo.plugin.timeout=600000\" >> "
118                  "/etc/openbaton/openbaton-nfvo.properties\n")
119     userdata += (
120         "wget %s -O /etc/openbaton/openbaton-vnfm-generic-user-data.sh\n" %
121         orchestrator['gvnfm']['userdata']['url'])
122     userdata += "sed -i '113i"'\ \ \ \ '"sleep 60' " \
123                 "/etc/openbaton/openbaton-vnfm-generic-user-data.sh\n"
124     userdata += ("sed -i s/nfvo.marketplace.port=8082/nfvo.marketplace."
125                  "port=8080/g /etc/openbaton/openbaton-nfvo.properties\n")
126     userdata += "echo \"Starting NFVO\"\n"
127     userdata += "service openbaton-nfvo restart\n"
128     userdata += "echo \"Starting Generic VNFM\"\n"
129     userdata += "service openbaton-vnfm-generic restart\n"
130     userdata += "echo \"...end of userdata...\"\n"
131     return userdata
132
133
134 class ClearwaterImsVnf(vnf.VnfOnBoarding):
135     """Clearwater IMS VNF deployed with openBaton orchestrator"""
136
137     # logger = logging.getLogger(__name__)
138
139     def __init__(self, **kwargs):
140         if "case_name" not in kwargs:
141             kwargs["case_name"] = "orchestra_clearwaterims"
142         super(ClearwaterImsVnf, self).__init__(**kwargs)
143         self.logger = logging.getLogger("functest.ci.run_tests.orchestra")
144         self.logger.info("kwargs %s", (kwargs))
145
146         self.case_dir = pkg_resources.resource_filename(
147             'functest', 'opnfv_tests/vnf/ims/')
148         self.data_dir = getattr(config.CONF, 'dir_ims_data')
149         self.test_dir = getattr(config.CONF, 'dir_repo_vims_test')
150         self.created_resources = []
151         self.logger.info("%s VNF onboarding test starting", self.case_name)
152
153         try:
154             self.config = getattr(
155                 config.CONF, 'vnf_{}_config'.format(self.case_name))
156         except BaseException:
157             raise Exception("Orchestra VNF config file not found")
158
159         config_file = self.case_dir + self.config
160
161         self.mano = dict(
162             get_config("mano", config_file),
163             details={}
164         )
165         self.logger.debug("Orchestrator configuration %s", self.mano)
166
167         self.details['orchestrator'] = dict(
168             name=self.mano['name'],
169             version=self.mano['version'],
170             status='ERROR',
171             result=''
172         )
173
174         self.vnf = dict(
175             get_config(self.case_name, config_file),
176         )
177         self.logger.debug("VNF configuration: %s", self.vnf)
178
179         self.details['vnf'] = dict(
180             name=self.vnf['name'],
181         )
182
183         self.details['test_vnf'] = dict(
184             name=self.case_name,
185         )
186
187         # Orchestra base Data directory creation
188         if not os.path.exists(self.data_dir):
189             os.makedirs(self.data_dir)
190
191         self.images = get_config("tenant_images.orchestrator", config_file)
192         self.images.update(
193             get_config(
194                 "tenant_images.%s" %
195                 self.case_name,
196                 config_file))
197         self.creds = None
198         self.orchestra_router = None
199
200     def prepare(self):
201         """Prepare testscase (Additional pre-configuration steps)."""
202         super(ClearwaterImsVnf, self).prepare()
203
204         self.logger.info("Additional pre-configuration steps")
205
206         public_auth_url = keystone_utils.get_endpoint(
207             self.snaps_creds, 'identity')
208         self.creds = {
209             "tenant": self.snaps_creds.project_name,
210             "username": self.snaps_creds.username,
211             "password": self.snaps_creds.password,
212             "auth_url": public_auth_url}
213         self.prepare_images()
214         self.prepare_flavor()
215         self.prepare_security_groups()
216         self.prepare_network()
217
218     def prepare_images(self):
219         """Upload images if they doen't exist yet"""
220         self.logger.info("Upload images if they doen't exist yet")
221         for image_name, image_file in self.images.iteritems():
222             self.logger.info("image: %s, file: %s", image_name, image_file)
223             if image_file and image_name:
224                 image = OpenStackImage(
225                     self.snaps_creds,
226                     ImageConfig(name=image_name,
227                                 image_user='cloud',
228                                 img_format='qcow2',
229                                 image_file=image_file,
230                                 public=True))
231                 image.create()
232                 self.created_resources.append(image)
233
234     def prepare_security_groups(self):
235         """Create Open Baton security group if it doesn't exist yet"""
236         self.logger.info(
237             "Creating security group for Open Baton if not yet existing...")
238         sg_rules = list()
239         sg_rules.append(
240             SecurityGroupRuleConfig(
241                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
242                     self.uuid),
243                 direction=Direction.ingress,
244                 protocol=Protocol.tcp,
245                 port_range_min=1,
246                 port_range_max=65535))
247         sg_rules.append(
248             SecurityGroupRuleConfig(
249                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
250                     self.uuid),
251                 direction=Direction.egress,
252                 protocol=Protocol.tcp,
253                 port_range_min=1,
254                 port_range_max=65535))
255         sg_rules.append(
256             SecurityGroupRuleConfig(
257                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
258                     self.uuid),
259                 direction=Direction.ingress,
260                 protocol=Protocol.udp,
261                 port_range_min=1,
262                 port_range_max=65535))
263         sg_rules.append(
264             SecurityGroupRuleConfig(
265                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
266                     self.uuid),
267                 direction=Direction.egress,
268                 protocol=Protocol.udp,
269                 port_range_min=1,
270                 port_range_max=65535))
271         sg_rules.append(
272             SecurityGroupRuleConfig(
273                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
274                     self.uuid),
275                 direction=Direction.ingress,
276                 protocol=Protocol.icmp))
277         sg_rules.append(
278             SecurityGroupRuleConfig(
279                 sec_grp_name="orchestra-sec-group-allowall-{}".format(
280                     self.uuid),
281                 direction=Direction.egress,
282                 protocol=Protocol.icmp))
283         security_group = OpenStackSecurityGroup(
284             self.snaps_creds,
285             SecurityGroupConfig(
286                 name="orchestra-sec-group-allowall-{}".format(
287                     self.uuid),
288                 rule_settings=sg_rules))
289         security_group_info = security_group.create()
290         self.created_resources.append(security_group)
291         self.mano['details']['sec_group'] = security_group_info.name
292         self.logger.info(
293             "Security group orchestra-sec-group-allowall prepared")
294
295     def prepare_flavor(self):
296         """Create Open Baton flavor if it doesn't exist yet"""
297         self.logger.info(
298             "Create Flavor for Open Baton NFVO if not yet existing")
299
300         flavor_settings = FlavorConfig(
301             name=self.mano['requirements']['flavor']['name'],
302             ram=self.mano['requirements']['flavor']['ram_min'],
303             disk=self.mano['requirements']['flavor']['disk'],
304             vcpus=self.mano['requirements']['flavor']['vcpus'])
305         flavor = OpenStackFlavor(self.snaps_creds, flavor_settings)
306         flavor_info = flavor.create()
307         self.created_resources.append(flavor)
308         self.mano['details']['flavor'] = {}
309         self.mano['details']['flavor']['name'] = flavor_settings.name
310         self.mano['details']['flavor']['id'] = flavor_info.id
311
312     def prepare_network(self):
313         """Create network/subnet/router if they doen't exist yet"""
314         self.logger.info(
315             "Creating network/subnet/router if they doen't exist yet...")
316         subnet_settings = SubnetConfig(
317             name='{}_subnet-{}'.format(self.case_name, self.uuid),
318             cidr="192.168.100.0/24")
319         network_settings = NetworkConfig(
320             name='{}_net-{}'.format(self.case_name, self.uuid),
321             subnet_settings=[subnet_settings])
322         orchestra_network = OpenStackNetwork(
323             self.snaps_creds, network_settings)
324         orchestra_network_info = orchestra_network.create()
325         self.mano['details']['network'] = {}
326         self.mano['details']['network']['id'] = orchestra_network_info.id
327         self.mano['details']['network']['name'] = orchestra_network_info.name
328         self.mano['details']['external_net_name'] = snaps_utils.\
329             get_ext_net_name(self.snaps_creds)
330         self.created_resources.append(orchestra_network)
331         self.orchestra_router = OpenStackRouter(
332             self.snaps_creds,
333             RouterConfig(
334                 name='{}_router-{}'.format(self.case_name, self.uuid),
335                 external_gateway=self.mano['details']['external_net_name'],
336                 internal_subnets=[
337                     subnet_settings.name]))
338         self.orchestra_router.create()
339         self.created_resources.append(self.orchestra_router)
340         self.logger.info("Created network and router for Open Baton NFVO...")
341
342     def get_vim_descriptor(self):
343         """"Create VIM descriptor to be used for onboarding"""
344         self.logger.info(
345             "Building VIM descriptor with PoP creds: %s",
346             self.creds)
347         self.logger.debug("VIM project/tenant id: %s",
348                           self.snaps_creds.project_name)
349         keystone = keystone_utils.keystone_client(self.snaps_creds)
350         project_id = keystone_utils.get_project(
351             keystone=keystone, project_name=self.snaps_creds.project_name).id
352         vim_json = {
353             "name": "vim-instance",
354             "authUrl": self.creds.get("auth_url"),
355             "tenant": project_id,
356             "username": self.creds.get("username"),
357             "password": self.creds.get("password"),
358             "securityGroups": [
359                 self.mano['details']['sec_group']
360             ],
361             "type": "openstack",
362             "location": {
363                 "name": "opnfv",
364                 "latitude": "52.525876",
365                 "longitude": "13.314400"
366             }
367         }
368         self.logger.info("Built VIM descriptor: %s", vim_json)
369         return vim_json
370
371     def deploy_orchestrator(self):
372         self.logger.info("Deploying Open Baton...")
373         self.logger.info("Details: %s", self.mano['details'])
374         start_time = time.time()
375
376         self.logger.info("Creating orchestra instance...")
377         userdata = get_userdata(self.mano)
378         self.logger.info("flavor: %s\n"
379                          "image: %s\n"
380                          "network_id: %s\n",
381                          self.mano['details']['flavor']['name'],
382                          self.mano['requirements']['image'],
383                          self.mano['details']['network']['id'])
384         self.logger.debug("userdata: %s\n", userdata)
385         # setting up image
386         image_settings = ImageConfig(
387             name=self.mano['requirements']['image'],
388             image_user='ubuntu',
389             exists=True)
390         # setting up port
391         port_settings = PortConfig(
392             name='{}_port-{}'.format(self.case_name, self.uuid),
393             network_name=self.mano['details']['network']['name'])
394
395         # build configuration of vm
396         orchestra_settings = VmInstanceConfig(
397             name=self.case_name,
398             flavor=self.mano['details']['flavor']['name'],
399             port_settings=[port_settings],
400             security_group_names=[self.mano['details']['sec_group']],
401             floating_ip_settings=[FloatingIpConfig(
402                 name='orchestra_fip-{}'.format(self.uuid),
403                 port_name=port_settings.name,
404                 router_name=self.orchestra_router.router_settings.name)],
405             userdata=str(userdata))
406         orchestra_vm = OpenStackVmInstance(
407             self.snaps_creds, orchestra_settings, image_settings)
408         orchestra_vm.create()
409         self.mano['details']['fip'] = orchestra_vm.get_floating_ip()
410         self.created_resources.append(orchestra_vm)
411         self.mano['details']['id'] = orchestra_vm.get_vm_info()['id']
412         self.logger.info(
413             "Created orchestra instance: %s", self.mano['details']['id'])
414         self.logger.info("Waiting for Open Baton NFVO to be up and running...")
415         timeout = 0
416         while timeout < 20:
417             if servertest(
418                     self.mano['details']['fip'].ip,
419                     "8080"):
420                 break
421             else:
422                 self.logger.info(
423                     "Open Baton NFVO is not started yet (%ss)",
424                     (timeout * 60))
425                 time.sleep(60)
426                 timeout += 1
427
428         if timeout >= 20:
429             duration = time.time() - start_time
430             self.details["orchestrator"].update(
431                 status='FAIL', duration=duration)
432             self.logger.error("Open Baton is not started correctly")
433             return False
434
435         self.logger.info("Waiting for all components to be up and running...")
436         time.sleep(60)
437         duration = time.time() - start_time
438         self.details["orchestrator"].update(status='PASS', duration=duration)
439         self.logger.info("Deploy Open Baton NFVO: OK")
440         return True
441
442     def deploy_vnf(self):
443         start_time = time.time()
444         self.logger.info("Deploying %s...", self.vnf['name'])
445
446         main_agent = MainAgent(
447             nfvo_ip=self.mano['details']['fip'].ip,
448             nfvo_port=8080,
449             https=False,
450             version=1,
451             username=self.mano['credentials']['username'],
452             password=self.mano['credentials']['password'])
453
454         self.logger.info(
455             "Create %s Flavor if not existing", self.vnf['name'])
456         flavor_settings = FlavorConfig(
457             name=self.vnf['requirements']['flavor']['name'],
458             ram=self.vnf['requirements']['flavor']['ram_min'],
459             disk=self.vnf['requirements']['flavor']['disk'],
460             vcpus=self.vnf['requirements']['flavor']['vcpus'])
461         flavor = OpenStackFlavor(self.snaps_creds, flavor_settings)
462         flavor_info = flavor.create()
463         self.logger.debug("Flavor id: %s", flavor_info.id)
464
465         self.logger.info("Getting project 'default'...")
466         project_agent = main_agent.get_agent("project", "")
467         for project in json.loads(project_agent.find()):
468             if project.get("name") == "default":
469                 self.mano['details']['project_id'] = project.get("id")
470                 self.logger.info("Found project 'default': %s", project)
471                 break
472
473         vim_json = self.get_vim_descriptor()
474         self.logger.info("Registering VIM: %s", vim_json)
475
476         main_agent.get_agent(
477             "vim", project_id=self.mano['details']['project_id']).create(
478                 entity=json.dumps(vim_json))
479
480         market_agent = main_agent.get_agent(
481             "market", project_id=self.mano['details']['project_id'])
482
483         try:
484             self.logger.info("sending: %s", self.vnf['descriptor']['url'])
485             nsd = market_agent.create(entity=self.vnf['descriptor']['url'])
486             if nsd.get('id') is None:
487                 self.logger.error("NSD not onboarded correctly")
488                 duration = time.time() - start_time
489                 self.details["vnf"].update(status='FAIL', duration=duration)
490                 return False
491             self.mano['details']['nsd_id'] = nsd.get('id')
492             self.logger.info("Onboarded NSD: " + nsd.get("name"))
493
494             nsr_agent = main_agent.get_agent(
495                 "nsr", project_id=self.mano['details']['project_id'])
496
497             self.mano['details']['nsr'] = nsr_agent.create(
498                 self.mano['details']['nsd_id'])
499         except NfvoException:
500             self.logger.exception("failed")
501             duration = time.time() - start_time
502             self.details["vnf"].update(status='FAIL', duration=duration)
503             return False
504
505         if self.mano['details']['nsr'].get('code') is not None:
506             self.logger.error(
507                 "%s cannot be deployed: %s -> %s",
508                 self.vnf['name'],
509                 self.mano['details']['nsr'].get('code'),
510                 self.mano['details']['nsr'].get('message'))
511             self.logger.error("%s cannot be deployed", self.vnf['name'])
512             duration = time.time() - start_time
513             self.details["vnf"].update(status='FAIL', duration=duration)
514             return False
515
516         timeout = 0
517         self.logger.info("Waiting for NSR to go to ACTIVE...")
518         while self.mano['details']['nsr'].get("status") != 'ACTIVE' \
519                 and self.mano['details']['nsr'].get("status") != 'ERROR':
520             timeout += 1
521             self.logger.info("NSR is not yet ACTIVE... (%ss)", 60 * timeout)
522             if timeout == 30:
523                 self.logger.error("INACTIVE NSR after %s sec..", 60 * timeout)
524                 duration = time.time() - start_time
525                 self.details["vnf"].update(status='FAIL', duration=duration)
526                 return False
527             time.sleep(60)
528             self.mano['details']['nsr'] = json.loads(
529                 nsr_agent.find(self.mano['details']['nsr'].get('id')))
530
531         duration = time.time() - start_time
532         if self.mano['details']['nsr'].get("status") == 'ACTIVE':
533             self.details["vnf"].update(status='PASS', duration=duration)
534             self.logger.info("Sleep for 60s to ensure that all "
535                              "services are up and running...")
536             time.sleep(60)
537             result = True
538         else:
539             self.details["vnf"].update(status='FAIL', duration=duration)
540             self.logger.error("NSR: %s", self.mano['details'].get('nsr'))
541             result = False
542         return result
543
544     def test_vnf(self):
545         self.logger.info(
546             "Testing VNF Clearwater IMS is not yet implemented...")
547         start_time = time.time()
548
549         duration = time.time() - start_time
550         self.details["test_vnf"].update(status='PASS', duration=duration)
551         self.logger.info("Test VNF: OK")
552         return True
553
554     def clean(self):
555         self.logger.info("Cleaning %s...", self.case_name)
556         try:
557             main_agent = MainAgent(
558                 nfvo_ip=self.mano['details']['fip'].ip,
559                 nfvo_port=8080, https=False, version=1,
560                 username=self.mano['credentials']['username'],
561                 password=self.mano['credentials']['password'])
562             self.logger.info("Terminating %s...", self.vnf['name'])
563             if self.mano['details'].get('nsr'):
564                 main_agent.get_agent(
565                     "nsr",
566                     project_id=self.mano['details']['project_id']).delete(
567                         self.mano['details']['nsr'].get('id'))
568                 self.logger.info("Sleeping 60 seconds...")
569                 time.sleep(60)
570             else:
571                 self.logger.info("No need to terminate the VNF...")
572         except (NfvoException, KeyError) as exc:
573             self.logger.error('Unexpected error cleaning - %s', exc)
574         super(ClearwaterImsVnf, self).clean()