Create security groups with ICMP and SSH rules for vPing tests
[functest.git] / testcases / vPing / CI / libraries / vPing2.py
1 #!/usr/bin/python
2 #
3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # 0.1: This script boots the VM1 and allocates IP address from Nova
11 # Later, the VM2 boots then execute cloud-init to ping VM1.
12 # After successful ping, both the VMs are deleted.
13 # 0.2: measure test duration and publish results under json format
14 #
15 #
16 import argparse
17 import datetime
18 import logging
19 import os
20 import paramiko
21 import pprint
22 import subprocess
23 import sys
24 import time
25 import yaml
26 from scp import SCPClient
27 from novaclient import client as novaclient
28 from neutronclient.v2_0 import client as neutronclient
29 from keystoneclient.v2_0 import client as keystoneclient
30 from glanceclient import client as glanceclient
31
32 pp = pprint.PrettyPrinter(indent=4)
33
34 parser = argparse.ArgumentParser()
35
36 parser.add_argument("-d", "--debug", help="Debug mode", action="store_true")
37 parser.add_argument("-r", "--report",
38                     help="Create json result file",
39                     action="store_true")
40
41 args = parser.parse_args()
42
43 """ logging configuration """
44
45 logger = logging.getLogger('vPing')
46 logger.setLevel(logging.DEBUG)
47
48 ch = logging.StreamHandler()
49
50 if args.debug:
51     ch.setLevel(logging.DEBUG)
52 else:
53     ch.setLevel(logging.INFO)
54
55 formatter = logging.Formatter('%(asctime)s - %(name)s'
56                               '- %(levelname)s - %(message)s')
57
58 ch.setFormatter(formatter)
59 logger.addHandler(ch)
60 paramiko.util.log_to_file("/var/log/paramiko.log")
61
62 REPO_PATH = os.environ['repos_dir']+'/functest/'
63 if not os.path.exists(REPO_PATH):
64     logger.error("Functest repository directory not found '%s'" % REPO_PATH)
65     exit(-1)
66 sys.path.append(REPO_PATH + "testcases/")
67 import functest_utils
68
69 with open("/home/opnfv/functest/conf/config_functest.yaml") as f:
70     functest_yaml = yaml.safe_load(f)
71 f.close()
72
73 HOME = os.environ['HOME'] + "/"
74 # vPing parameters
75 VM_BOOT_TIMEOUT = 180
76 VM_DELETE_TIMEOUT = 100
77 PING_TIMEOUT = functest_yaml.get("vping").get("ping_timeout")
78 TEST_DB = functest_yaml.get("results").get("test_db_url")
79 NAME_VM_1 = functest_yaml.get("vping").get("vm_name_1")
80 NAME_VM_2 = functest_yaml.get("vping").get("vm_name_2")
81 IP_1 = functest_yaml.get("vping").get("ip_1")
82 IP_2 = functest_yaml.get("vping").get("ip_2")
83 # GLANCE_IMAGE_NAME = functest_yaml.get("general"). \
84 #    get("openstack").get("image_name")
85 GLANCE_IMAGE_NAME = "functest-vping"
86 GLANCE_IMAGE_FILENAME = functest_yaml.get("general"). \
87     get("openstack").get("image_file_name")
88 GLANCE_IMAGE_FORMAT = functest_yaml.get("general"). \
89     get("openstack").get("image_disk_format")
90 GLANCE_IMAGE_PATH = functest_yaml.get("general"). \
91     get("directories").get("dir_functest_data") + "/" + GLANCE_IMAGE_FILENAME
92
93
94 FLAVOR = functest_yaml.get("vping").get("vm_flavor")
95
96 # NEUTRON Private Network parameters
97
98 NEUTRON_PRIVATE_NET_NAME = functest_yaml.get("vping"). \
99     get("vping_private_net_name")
100
101 NEUTRON_PRIVATE_SUBNET_NAME = functest_yaml.get("vping"). \
102     get("vping_private_subnet_name")
103
104 NEUTRON_PRIVATE_SUBNET_CIDR = functest_yaml.get("vping"). \
105     get("vping_private_subnet_cidr")
106
107 NEUTRON_ROUTER_NAME = functest_yaml.get("vping"). \
108     get("vping_router_name")
109
110
111 def pMsg(value):
112
113     """pretty printing"""
114     pp.pprint(value)
115
116
117 def waitVmActive(nova, vm):
118
119     # sleep and wait for VM status change
120     sleep_time = 3
121     count = VM_BOOT_TIMEOUT / sleep_time
122     while True:
123         status = functest_utils.get_instance_status(nova, vm)
124         logger.debug("Status: %s" % status)
125         if status == "ACTIVE":
126             return True
127         if status == "ERROR" or status == "error":
128             return False
129         if count == 0:
130             logger.debug("Booting a VM timed out...")
131             return False
132         count -= 1
133         time.sleep(sleep_time)
134     return False
135
136
137 def waitVmDeleted(nova, vm):
138
139     # sleep and wait for VM status change
140     sleep_time = 3
141     count = VM_DELETE_TIMEOUT / sleep_time
142     while True:
143         status = functest_utils.get_instance_status(nova, vm)
144         if not status:
145             return True
146         elif count == 0:
147             logger.debug("Timeout")
148             return False
149         else:
150             # return False
151             count -= 1
152         time.sleep(sleep_time)
153     return False
154
155
156 def create_private_neutron_net(neutron):
157
158     neutron.format = 'json'
159     logger.info('Creating neutron network %s...' % NEUTRON_PRIVATE_NET_NAME)
160     network_id = functest_utils. \
161         create_neutron_net(neutron, NEUTRON_PRIVATE_NET_NAME)
162
163     if not network_id:
164         return False
165     logger.debug("Network '%s' created successfully" % network_id)
166     logger.debug('Creating Subnet....')
167     subnet_id = functest_utils. \
168         create_neutron_subnet(neutron,
169                               NEUTRON_PRIVATE_SUBNET_NAME,
170                               NEUTRON_PRIVATE_SUBNET_CIDR,
171                               network_id)
172     if not subnet_id:
173         return False
174     logger.debug("Subnet '%s' created successfully" % subnet_id)
175     logger.debug('Creating Router...')
176     router_id = functest_utils. \
177         create_neutron_router(neutron, NEUTRON_ROUTER_NAME)
178
179     if not router_id:
180         return False
181
182     logger.debug("Router '%s' created successfully" % router_id)
183     logger.debug('Adding router to subnet...')
184
185     if not functest_utils.add_interface_router(neutron, router_id, subnet_id):
186         return False
187     logger.debug("Interface added successfully.")
188
189     logger.debug('Adding gateway to router...')
190     if not functest_utils.add_gateway_router(neutron, router_id):
191         return False
192     logger.debug("Gateway added successfully.")
193
194     network_dic = {'net_id': network_id,
195                    'subnet_id': subnet_id,
196                    'router_id': router_id}
197     return network_dic
198
199
200 def cleanup(nova, neutron, image_id, network_dic, port_id1, port_id2, secgroup_id):
201
202     # delete both VMs
203     logger.info("Cleaning up...")
204     logger.debug("Deleting image...")
205     if not functest_utils.delete_glance_image(nova, image_id):
206         logger.error("Error deleting the glance image")
207
208     vm1 = functest_utils.get_instance_by_name(nova, NAME_VM_1)
209     if vm1:
210         logger.debug("Deleting '%s'..." % NAME_VM_1)
211         nova.servers.delete(vm1)
212         # wait until VMs are deleted
213         if not waitVmDeleted(nova, vm1):
214             logger.error(
215                 "Instance '%s' with cannot be deleted. Status is '%s'" % (
216                     NAME_VM_1, functest_utils.get_instance_status(nova, vm1)))
217         else:
218             logger.debug("Instance %s terminated." % NAME_VM_1)
219
220     vm2 = functest_utils.get_instance_by_name(nova, NAME_VM_2)
221
222     if vm2:
223         logger.debug("Deleting '%s'..." % NAME_VM_2)
224         vm2 = nova.servers.find(name=NAME_VM_2)
225         nova.servers.delete(vm2)
226
227         if not waitVmDeleted(nova, vm2):
228             logger.error(
229                 "Instance '%s' with cannot be deleted. Status is '%s'" % (
230                     NAME_VM_2, functest_utils.get_instance_status(nova, vm2)))
231         else:
232             logger.debug("Instance %s terminated." % NAME_VM_2)
233
234     # delete created network
235     logger.debug("Deleting network '%s'..." % NEUTRON_PRIVATE_NET_NAME)
236     net_id = network_dic["net_id"]
237     subnet_id = network_dic["subnet_id"]
238     router_id = network_dic["router_id"]
239
240     if not functest_utils.delete_neutron_port(neutron, port_id1):
241         logger.error("Unable to remove port '%s'" % port_id1)
242         return False
243     logger.debug("Port '%s' removed successfully" % port_id1)
244
245     if not functest_utils.delete_neutron_port(neutron, port_id2):
246         logger.error("Unable to remove port '%s'" % port_id2)
247         return False
248     logger.debug("Port '%s' removed successfully" % port_id2)
249
250     if not functest_utils.remove_interface_router(neutron, router_id,
251                                                   subnet_id):
252         logger.error("Unable to remove subnet '%s' from router '%s'" % (
253             subnet_id, router_id))
254         return False
255
256     logger.debug("Interface removed successfully")
257     if not functest_utils.delete_neutron_router(neutron, router_id):
258         logger.error("Unable to delete router '%s'" % router_id)
259         return False
260
261     logger.debug("Router deleted successfully")
262
263     if not functest_utils.delete_neutron_subnet(neutron, subnet_id):
264         logger.error("Unable to delete subnet '%s'" % subnet_id)
265         return False
266
267     logger.debug(
268         "Subnet '%s' deleted successfully" % NEUTRON_PRIVATE_SUBNET_NAME)
269
270     if not functest_utils.delete_neutron_net(neutron, net_id):
271         logger.error("Unable to delete network '%s'" % net_id)
272         return False
273
274     logger.debug(
275         "Network '%s' deleted successfully" % NEUTRON_PRIVATE_NET_NAME)
276
277     if not functest_utils.delete_security_group(neutron, secgroup_id):
278         logger.error("Unable to delete security group '%s'" % secgroup_id)
279         return False
280     logger.debug(
281         "Security group '%s' deleted successfully" % secgroup_id)
282
283     return True
284
285 def push_results(start_time_ts, duration, test_status):
286     try:
287         logger.debug("Pushing result into DB...")
288         scenario = functest_utils.get_scenario(logger)
289         pod_name = functest_utils.get_pod_name(logger)
290         functest_utils.push_results_to_db(TEST_DB,
291                                           "vPing",
292                                           logger, pod_name, scenario,
293                                           payload={'timestart': start_time_ts,
294                                                    'duration': duration,
295                                                    'status': test_status})
296     except:
297         logger.error("Error pushing results into Database '%s'" % sys.exc_info()[0])
298
299
300 def main():
301
302     creds_nova = functest_utils.get_credentials("nova")
303     nova_client = novaclient.Client('2', **creds_nova)
304     creds_neutron = functest_utils.get_credentials("neutron")
305     neutron_client = neutronclient.Client(**creds_neutron)
306     creds_keystone = functest_utils.get_credentials("keystone")
307     keystone_client = keystoneclient.Client(**creds_keystone)
308     glance_endpoint = keystone_client.service_catalog.url_for(service_type='image',
309                                                               endpoint_type='publicURL')
310     glance_client = glanceclient.Client(1, glance_endpoint,
311                                         token=keystone_client.auth_token)
312     EXIT_CODE = -1
313
314     image = None
315     flavor = None
316
317     logger.debug("Creating image '%s' from '%s'..." % (GLANCE_IMAGE_NAME,
318                                                        GLANCE_IMAGE_PATH))
319     image_id = functest_utils.create_glance_image(glance_client,
320                                                   GLANCE_IMAGE_NAME,
321                                                   GLANCE_IMAGE_PATH)
322     if not image_id:
323         logger.error("Failed to create a Glance image...")
324         return(EXIT_CODE)
325     logger.debug("Image '%s' with ID=%s created successfully." %\
326                   (GLANCE_IMAGE_NAME, image_id))
327     # Check if the given image exists
328     image = functest_utils.get_image_id(glance_client, GLANCE_IMAGE_NAME)
329     if image == '':
330         logger.error("ERROR: Glance image '%s' not found." % GLANCE_IMAGE_NAME)
331         logger.info("Available images are: ")
332         pMsg(nova_client.images.list())
333         return(EXIT_CODE)
334
335     network_dic = create_private_neutron_net(neutron_client)
336
337     if not network_dic:
338         logger.error(
339             "There has been a problem when creating the neutron network")
340         return(EXIT_CODE)
341
342     network_id = network_dic["net_id"]
343
344     # Check if the given flavor exists
345
346     try:
347         flavor = nova_client.flavors.find(name=FLAVOR)
348         logger.info("Flavor found '%s'" % FLAVOR)
349     except:
350         logger.error("Flavor '%s' not found." % FLAVOR)
351         logger.info("Available flavors are: ")
352         pMsg(nova_client.flavor.list())
353         return(EXIT_CODE)
354
355     # Deleting instances if they exist
356
357     servers = nova_client.servers.list()
358     for server in servers:
359         if server.name == NAME_VM_1 or server.name == NAME_VM_2:
360             logger.info("Instance %s found. Deleting..." % server.name)
361             server.delete()
362
363     SECGROUP_NAME = "vPing-sg"
364     SECGROUP_DESCR = "Security group for vPing test case"
365     logger.info("Creating security group  '%s'..." % SECGROUP_NAME)
366     SECGROUP = functest_utils.create_security_group(neutron_client,
367                                           SECGROUP_NAME,
368                                           SECGROUP_DESCR)
369     SECGROUP_ID = SECGROUP['id']
370     if not SECGROUP:
371         logger.error("Failed to create the security group...")
372         return(EXIT_CODE)
373     logger.debug("Security group '%s' with ID=%s created successfully." %\
374                   (SECGROUP['name'], SECGROUP_ID))
375
376     logger.debug("Adding ICMP rules in security group '%s'..." % SECGROUP_NAME)
377     if not functest_utils.create_secgroup_rule(neutron_client, SECGROUP_ID, \
378                     'ingress', 'icmp'):
379         logger.error("Failed to create the security group rule...")
380         return(EXIT_CODE)
381
382     logger.debug("Adding SSH rules in security group '%s'..." % SECGROUP_NAME)
383     if not functest_utils.create_secgroup_rule(neutron_client, SECGROUP_ID, \
384                     'ingress', 'tcp', '22', '22'):
385         logger.error("Failed to create the security group rule...")
386         return(EXIT_CODE)
387
388     if not functest_utils.create_secgroup_rule(neutron_client, SECGROUP_ID, \
389                     'egress', 'tcp', '22', '22'):
390         logger.error("Failed to create the security group rule...")
391         return(EXIT_CODE)
392
393     # boot VM 1
394     # basic boot
395     # tune (e.g. flavor, images, network) to your specific
396     # openstack configuration here
397     # we consider start time at VM1 booting
398     start_time_ts = time.time()
399     end_time_ts = start_time_ts
400     logger.info("vPing Start Time:'%s'" % (
401         datetime.datetime.fromtimestamp(start_time_ts).strftime(
402             '%Y-%m-%d %H:%M:%S')))
403
404     # create VM
405     logger.debug("Creating port 'vping-port-1' with IP %s..." % IP_1)
406     port_id1 = functest_utils.create_neutron_port(neutron_client,
407                                                   "vping-port-1", network_id,
408                                                   IP_1)
409     if not port_id1:
410         logger.error("Unable to create port.")
411         return(EXIT_CODE)
412
413     logger.info("Creating instance '%s' with IP %s..." % (NAME_VM_1, IP_1))
414     logger.debug(
415         "Configuration:\n name=%s \n flavor=%s \n image=%s \n "
416         "network=%s \n" % (NAME_VM_1, flavor, image, network_id))
417     vm1 = nova_client.servers.create(
418         name=NAME_VM_1,
419         flavor=flavor,
420         image=image,
421         # nics = [{"net-id": network_id, "v4-fixed-ip": IP_1}]
422         nics=[{"port-id": port_id1}]
423     )
424
425     # wait until VM status is active
426     if not waitVmActive(nova_client, vm1):
427         logger.error("Instance '%s' cannot be booted. Status is '%s'" % (
428             NAME_VM_1, functest_utils.get_instance_status(nova_client, vm1)))
429         cleanup(nova_client, neutron_client, image_id, network_dic,
430                 port_id1, SECGROUP_ID)
431         return (EXIT_CODE)
432     else:
433         logger.info("Instance '%s' is ACTIVE." % NAME_VM_1)
434
435     # Retrieve IP of first VM
436     # logger.debug("Fetching IP...")
437     # server = functest_utils.get_instance_by_name(nova_client, NAME_VM_1)
438     # theoretically there is only one IP address so we take the
439     # first element of the table
440     # Dangerous! To be improved!
441     # test_ip = server.networks.get(NEUTRON_PRIVATE_NET_NAME)[0]
442     test_ip = IP_1
443     logger.debug("Instance '%s' got %s" % (NAME_VM_1, test_ip))
444
445     logger.info("Adding '%s' to security group '%s'..." % (NAME_VM_1, SECGROUP_NAME))
446     functest_utils.add_secgroup_to_instance(nova_client, vm1.id, SECGROUP['id'])
447
448     # boot VM 2
449     # we will boot then execute a ping script with cloud-init
450     # the long chain corresponds to the ping procedure converted with base 64
451     # tune (e.g. flavor, images, network) to your specific openstack
452     #  configuration here
453
454
455     # create VM
456     logger.debug("Creating port 'vping-port-2' with IP %s..." % IP_2)
457     port_id2 = functest_utils.create_neutron_port(neutron_client,
458                                                   "vping-port-2", network_id,
459                                                   IP_2)
460
461     if not port_id2:
462         logger.error("Unable to create port.")
463         return(EXIT_CODE)
464
465     logger.info("Creating instance '%s' with IP %s..." % (NAME_VM_2, IP_2))
466     logger.debug(
467         "Configuration:\n name=%s \n flavor=%s \n image=%s \n "
468         "network=%s \n" % (NAME_VM_2, flavor, image, network_id))
469     vm2 = nova_client.servers.create(
470         name=NAME_VM_2,
471         flavor=flavor,
472         image=image,
473         nics=[{"port-id": port_id2}]
474     )
475
476     if not waitVmActive(nova_client, vm2):
477         logger.error("Instance '%s' cannot be booted. Status is '%s'" % (
478             NAME_VM_2, functest_utils.get_instance_status(nova_client, vm2)))
479         cleanup(nova_client, neutron_client, image_id, network_dic,
480                 port_id1, port_id2, SECGROUP_ID)
481         return (EXIT_CODE)
482     else:
483         logger.info("Instance '%s' is ACTIVE." % NAME_VM_2)
484
485     logger.info("Adding '%s' to security group '%s'..." % (NAME_VM_2, SECGROUP_NAME))
486     functest_utils.add_secgroup_to_instance(nova_client, vm2.id, SECGROUP['id'])
487
488     logger.info("Creating floating IP for VM '%s'..." % NAME_VM_2)
489     floatip = functest_utils.create_floating_ip(neutron_client)
490     if floatip == None:
491         logger.error("Cannot create floating IP.")
492         cleanup(nova_client, neutron_client, image_id, network_dic,
493             port_id1, port_id2, SECGROUP_ID)
494         return (EXIT_CODE)
495     logger.info("Floating IP created: '%s'" % floatip)
496
497     logger.info("Associating floating ip: '%s' to VM '%s' " % (floatip, NAME_VM_2))
498     if not functest_utils.add_floating_ip(nova_client, vm2.id, floatip):
499         logger.error("Cannot associate floating IP to VM.")
500         cleanup(nova_client, neutron_client, image_id, network_dic,
501             port_id1, port_id2, SECGROUP_ID)
502         return (EXIT_CODE)
503
504     logger.info("Trying to establish SSH connection to %s..." % floatip)
505     username='cirros'
506     password='cubswin:)'
507     ssh = paramiko.SSHClient()
508     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
509
510     timeout = 50
511     while timeout > 0:
512         try:
513             ssh.connect(floatip, username=username, password=password, timeout=2)
514             logger.debug("SSH connection established to %s." % floatip)
515             break
516         except Exception, e:
517             #print e
518             logger.debug("Waiting for %s..." % floatip)
519             time.sleep(6)
520             timeout -= 1
521
522     if timeout == 0: # 300 sec timeout (5 min)
523         logger.error("Cannot establish connection to IP '%s'. Aborting" % floatip)
524         cleanup(nova_client, neutron_client, image_id, network_dic,
525             port_id1, port_id2, SECGROUP_ID)
526         return (EXIT_CODE)
527
528     scp = SCPClient(ssh.get_transport())
529
530     ping_script = REPO_PATH + "testcases/vPing/CI/libraries/ping.sh"
531     try:
532         scp.put(ping_script,"~/")
533     except Exception, e:
534         logger.error("Cannot SCP the file '%s' to VM '%s'" % (ping_script,floatip))
535
536
537     #SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "+\
538     #    "-o LogLevel=quiet"
539
540     #cmd1 = "sshpass -p 'cubswin:)' scp " + SSH_OPTS + " " + \
541     #    ping_script + " cirros@"+floatip+":~/ping.sh"
542     #cmd2 = "sshpass -p 'cubswin:)' ssh " + SSH_OPTS + \
543     #    " cirros@"+floatip+" 'chmod 755 ~/ping.sh '"
544     #cmd3 = "sshpass -p 'cubswin:)' ssh " + SSH_OPTS + \
545     #    " cirros@"+floatip+" '~/ping.sh "+IP_1+"'"
546
547     cmd = 'chmod 755 ~/ping.sh'
548     (stdin, stdout, stderr) = ssh.exec_command(cmd)
549     for line in stdout.readlines():
550         print line
551
552     logger.info("Waiting for ping...")
553     sec = 0
554     duration = 0
555
556     cmd = '~/ping.sh ' + IP_1
557     flag = False
558     while True:
559         time.sleep(1)
560         # we do the SCP every time in the loop because while testing, I observed
561         # that for some strange reason, the cirros VM was deleting the file if
562         # do the scp only once
563         (stdin, stdout, stderr) = ssh.exec_command(cmd)
564         output = stdout.readlines()
565         #for line in output:
566         #    print line
567
568         # print "--"+console_log
569         # report if the test is failed
570         for line in output:
571             if "vPing OK" in line:
572                 logger.info("vPing detected!")
573
574                 # we consider start time at VM1 booting
575                 end_time_ts = time.time()
576                 duration = round(end_time_ts - start_time_ts, 1)
577                 logger.info("vPing duration:'%s' s." % duration)
578                 EXIT_CODE = 0
579                 flag = True
580                 break
581             elif sec == PING_TIMEOUT:
582                 logger.info("Timeout reached.")
583                 flag = True
584                 break
585         if flag :
586             break
587         logger.debug("Pinging %s. Waiting for response..." % IP_1)
588         sec += 1
589
590     cleanup(nova_client, neutron_client, image_id, network_dic,
591             port_id1, port_id2, SECGROUP_ID)
592
593     test_status = "NOK"
594     if EXIT_CODE == 0:
595         logger.info("vPing OK")
596         test_status = "OK"
597     else:
598         duration = 0
599         logger.error("vPing FAILED")
600
601     if args.report:
602         push_results(start_time_ts, duration, test_status)
603
604     exit(EXIT_CODE)
605
606 if __name__ == '__main__':
607     main()