testcases/vPing/CI/libraries/vPing_ssh.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # 0.1: This script boots the VM1 and allocates IP address from Nova
  11 # Later, the VM2 boots then execute cloud-init to ping VM1.
  12 # After successful ping, both the VMs are deleted.
  13 # 0.2: measure test duration and publish results under json format
  14 #
  15 #
  16 import argparse
  17 import datetime
  18 import logging
  19 import os
  20 import paramiko
  21 import pprint
  22 import subprocess
  23 import sys
  24 import time
  25 import yaml
  26 from scp import SCPClient
  27 from novaclient import client as novaclient
  28 from neutronclient.v2_0 import client as neutronclient
  29 from keystoneclient.v2_0 import client as keystoneclient
  30 from glanceclient import client as glanceclient
  31
  32 pp = pprint.PrettyPrinter(indent=4)
  33
  34 parser = argparse.ArgumentParser()
  35
  36 parser.add_argument("-d", "--debug", help="Debug mode", action="store_true")
  37 parser.add_argument("-r", "--report",
  38                     help="Create json result file",
  39                     action="store_true")
  40 parser.add_argument("-n", "--noclean",
  41                     help="Don't clean the created resources for this test.",
  42                     action="store_true")
  43
  44 args = parser.parse_args()
  45
  46 """ logging configuration """
  47
  48 logger = logging.getLogger('vPing_ssh')
  49 logger.setLevel(logging.DEBUG)
  50
  51 ch = logging.StreamHandler()
  52
  53 if args.debug:
  54     ch.setLevel(logging.DEBUG)
  55 else:
  56     ch.setLevel(logging.INFO)
  57
  58 formatter = logging.Formatter('%(asctime)s - %(name)s'
  59                               '- %(levelname)s - %(message)s')
  60
  61 ch.setFormatter(formatter)
  62 logger.addHandler(ch)
  63 paramiko.util.log_to_file("/var/log/paramiko.log")
  64
  65 REPO_PATH = os.environ['repos_dir']+'/functest/'
  66 if not os.path.exists(REPO_PATH):
  67     logger.error("Functest repository directory not found '%s'" % REPO_PATH)
  68     exit(-1)
  69 sys.path.append(REPO_PATH + "testcases/")
  70 import functest_utils
  71
  72 with open("/home/opnfv/functest/conf/config_functest.yaml") as f:
  73     functest_yaml = yaml.safe_load(f)
  74 f.close()
  75
  76 HOME = os.environ['HOME'] + "/"
  77 # vPing parameters
  78 VM_BOOT_TIMEOUT = 180
  79 VM_DELETE_TIMEOUT = 100
  80 PING_TIMEOUT = functest_yaml.get("vping").get("ping_timeout")
  81 TEST_DB = functest_yaml.get("results").get("test_db_url")
  82 NAME_VM_1 = functest_yaml.get("vping").get("vm_name_1")
  83 NAME_VM_2 = functest_yaml.get("vping").get("vm_name_2")
  84 # GLANCE_IMAGE_NAME = functest_yaml.get("general"). \
  85 #    get("openstack").get("image_name")
  86 GLANCE_IMAGE_NAME = "functest-vping"
  87 GLANCE_IMAGE_FILENAME = functest_yaml.get("general"). \
  88     get("openstack").get("image_file_name")
  89 GLANCE_IMAGE_FORMAT = functest_yaml.get("general"). \
  90     get("openstack").get("image_disk_format")
  91 GLANCE_IMAGE_PATH = functest_yaml.get("general"). \
  92     get("directories").get("dir_functest_data") + "/" + GLANCE_IMAGE_FILENAME
  93
  94
  95 FLAVOR = functest_yaml.get("vping").get("vm_flavor")
  96
  97 # NEUTRON Private Network parameters
  98
  99 NEUTRON_PRIVATE_NET_NAME = functest_yaml.get("vping"). \
 100     get("vping_private_net_name")
 101 NEUTRON_PRIVATE_SUBNET_NAME = functest_yaml.get("vping"). \
 102     get("vping_private_subnet_name")
 103 NEUTRON_PRIVATE_SUBNET_CIDR = functest_yaml.get("vping"). \
 104     get("vping_private_subnet_cidr")
 105 NEUTRON_ROUTER_NAME = functest_yaml.get("vping"). \
 106     get("vping_router_name")
 107
 108 SECGROUP_NAME = functest_yaml.get("vping"). \
 109     get("vping_sg_name")
 110 SECGROUP_DESCR = functest_yaml.get("vping"). \
 111     get("vping_sg_descr")
 112
 113
 114 def pMsg(value):
 115
 116     """pretty printing"""
 117     pp.pprint(value)
 118
 119
 120 def waitVmActive(nova, vm):
 121
 122     # sleep and wait for VM status change
 123     sleep_time = 3
 124     count = VM_BOOT_TIMEOUT / sleep_time
 125     while True:
 126         status = functest_utils.get_instance_status(nova, vm)
 127         logger.debug("Status: %s" % status)
 128         if status == "ACTIVE":
 129             return True
 130         if status == "ERROR" or status == "error":
 131             return False
 132         if count == 0:
 133             logger.debug("Booting a VM timed out...")
 134             return False
 135         count -= 1
 136         time.sleep(sleep_time)
 137     return False
 138
 139
 140 def waitVmDeleted(nova, vm):
 141
 142     # sleep and wait for VM status change
 143     sleep_time = 3
 144     count = VM_DELETE_TIMEOUT / sleep_time
 145     while True:
 146         status = functest_utils.get_instance_status(nova, vm)
 147         if not status:
 148             return True
 149         elif count == 0:
 150             logger.debug("Timeout")
 151             return False
 152         else:
 153             # return False
 154             count -= 1
 155         time.sleep(sleep_time)
 156     return False
 157
 158
 159 def create_private_neutron_net(neutron):
 160
 161     # Check if the network already exists
 162     network_id = functest_utils.get_network_id(neutron,NEUTRON_PRIVATE_NET_NAME)
 163     subnet_id = functest_utils.get_subnet_id(neutron,NEUTRON_PRIVATE_SUBNET_NAME)
 164     router_id = functest_utils.get_router_id(neutron,NEUTRON_ROUTER_NAME)
 165
 166     if network_id != '' and subnet_id != ''  and router_id != '' :
 167         logger.info("Using existing network '%s'..." % NEUTRON_PRIVATE_NET_NAME)
 168     else:
 169         neutron.format = 'json'
 170         logger.info('Creating neutron network %s...' % NEUTRON_PRIVATE_NET_NAME)
 171         network_id = functest_utils. \
 172             create_neutron_net(neutron, NEUTRON_PRIVATE_NET_NAME)
 173
 174         if not network_id:
 175             return False
 176         logger.debug("Network '%s' created successfully" % network_id)
 177         logger.debug('Creating Subnet....')
 178         subnet_id = functest_utils. \
 179             create_neutron_subnet(neutron,
 180                                   NEUTRON_PRIVATE_SUBNET_NAME,
 181                                   NEUTRON_PRIVATE_SUBNET_CIDR,
 182                                   network_id)
 183         if not subnet_id:
 184             return False
 185         logger.debug("Subnet '%s' created successfully" % subnet_id)
 186         logger.debug('Creating Router...')
 187         router_id = functest_utils. \
 188             create_neutron_router(neutron, NEUTRON_ROUTER_NAME)
 189
 190         if not router_id:
 191             return False
 192
 193         logger.debug("Router '%s' created successfully" % router_id)
 194         logger.debug('Adding router to subnet...')
 195
 196         if not functest_utils.add_interface_router(neutron, router_id, subnet_id):
 197             return False
 198         logger.debug("Interface added successfully.")
 199
 200         logger.debug('Adding gateway to router...')
 201         if not functest_utils.add_gateway_router(neutron, router_id):
 202             return False
 203         logger.debug("Gateway added successfully.")
 204
 205     network_dic = {'net_id': network_id,
 206                    'subnet_id': subnet_id,
 207                    'router_id': router_id}
 208     return network_dic
 209
 210 def create_security_group(neutron_client):
 211     sg_id = functest_utils.get_security_group_id(neutron_client, SECGROUP_NAME)
 212     if sg_id != '':
 213         logger.info("Using existing security group '%s'..." % SECGROUP_NAME)
 214     else:
 215         logger.info("Creating security group  '%s'..." % SECGROUP_NAME)
 216         SECGROUP = functest_utils.create_security_group(neutron_client,
 217                                               SECGROUP_NAME,
 218                                               SECGROUP_DESCR)
 219         if not SECGROUP:
 220             logger.error("Failed to create the security group...")
 221             return False
 222
 223         sg_id = SECGROUP['id']
 224
 225         logger.debug("Security group '%s' with ID=%s created successfully." %\
 226                       (SECGROUP['name'], sg_id))
 227
 228         logger.debug("Adding ICMP rules in security group '%s'..." % SECGROUP_NAME)
 229         if not functest_utils.create_secgroup_rule(neutron_client, sg_id, \
 230                         'ingress', 'icmp'):
 231             logger.error("Failed to create the security group rule...")
 232             return False
 233
 234         logger.debug("Adding SSH rules in security group '%s'..." % SECGROUP_NAME)
 235         if not functest_utils.create_secgroup_rule(neutron_client, sg_id, \
 236                         'ingress', 'tcp', '22', '22'):
 237             logger.error("Failed to create the security group rule...")
 238             return False
 239
 240         if not functest_utils.create_secgroup_rule(neutron_client, sg_id, \
 241                         'egress', 'tcp', '22', '22'):
 242             logger.error("Failed to create the security group rule...")
 243             return False
 244     return sg_id
 245
 246 def cleanup(nova, neutron, image_id, network_dic, sg_id, floatingip):
 247     if args.noclean:
 248         logger.debug("The OpenStack resources are not deleted.")
 249         return True
 250
 251     # delete both VMs
 252     logger.info("Cleaning up...")
 253     logger.debug("Deleting image...")
 254     if not functest_utils.delete_glance_image(nova, image_id):
 255         logger.error("Error deleting the glance image")
 256
 257     vm1 = functest_utils.get_instance_by_name(nova, NAME_VM_1)
 258     if vm1:
 259         logger.debug("Deleting '%s'..." % NAME_VM_1)
 260         nova.servers.delete(vm1)
 261         # wait until VMs are deleted
 262         if not waitVmDeleted(nova, vm1):
 263             logger.error(
 264                 "Instance '%s' with cannot be deleted. Status is '%s'" % (
 265                     NAME_VM_1, functest_utils.get_instance_status(nova, vm1)))
 266         else:
 267             logger.debug("Instance %s terminated." % NAME_VM_1)
 268
 269     vm2 = functest_utils.get_instance_by_name(nova, NAME_VM_2)
 270
 271     if vm2:
 272         logger.debug("Deleting '%s'..." % NAME_VM_2)
 273         vm2 = nova.servers.find(name=NAME_VM_2)
 274         nova.servers.delete(vm2)
 275
 276         if not waitVmDeleted(nova, vm2):
 277             logger.error(
 278                 "Instance '%s' with cannot be deleted. Status is '%s'" % (
 279                     NAME_VM_2, functest_utils.get_instance_status(nova, vm2)))
 280         else:
 281             logger.debug("Instance %s terminated." % NAME_VM_2)
 282
 283     # delete created network
 284     logger.debug("Deleting network '%s'..." % NEUTRON_PRIVATE_NET_NAME)
 285     net_id = network_dic["net_id"]
 286     subnet_id = network_dic["subnet_id"]
 287     router_id = network_dic["router_id"]
 288
 289     if not functest_utils.remove_interface_router(neutron, router_id,
 290                                                   subnet_id):
 291         logger.error("Unable to remove subnet '%s' from router '%s'" % (
 292             subnet_id, router_id))
 293         return False
 294
 295     logger.debug("Interface removed successfully")
 296     if not functest_utils.delete_neutron_router(neutron, router_id):
 297         logger.error("Unable to delete router '%s'" % router_id)
 298         return False
 299
 300     logger.debug("Router deleted successfully")
 301
 302     if not functest_utils.delete_neutron_subnet(neutron, subnet_id):
 303         logger.error("Unable to delete subnet '%s'" % subnet_id)
 304         return False
 305
 306     logger.debug(
 307         "Subnet '%s' deleted successfully" % NEUTRON_PRIVATE_SUBNET_NAME)
 308
 309     if not functest_utils.delete_neutron_net(neutron, net_id):
 310         logger.error("Unable to delete network '%s'" % net_id)
 311         return False
 312
 313     logger.debug(
 314         "Network '%s' deleted successfully" % NEUTRON_PRIVATE_NET_NAME)
 315
 316     if not functest_utils.delete_security_group(neutron, sg_id):
 317         logger.error("Unable to delete security group '%s'" % sg_id)
 318         return False
 319     logger.debug(
 320         "Security group '%s' deleted successfully" % sg_id)
 321
 322     logger.debug("Releasing floating ip '%s'..." % floatingip['fip_addr'])
 323     if not functest_utils.delete_floating_ip(nova, floatingip['fip_id']):
 324         logger.error("Unable to delete floatingip '%s'" % floatingip['fip_addr'])
 325         return False
 326     logger.debug(
 327         "Floating IP '%s' deleted successfully" % floatingip['fip_addr'])
 328     return True
 329
 330
 331 def push_results(start_time_ts, duration, test_status):
 332     try:
 333         logger.debug("Pushing result into DB...")
 334         scenario = functest_utils.get_scenario(logger)
 335         pod_name = functest_utils.get_pod_name(logger)
 336         functest_utils.push_results_to_db(TEST_DB,
 337                                           "functest",
 338                                           "vPing",
 339                                           logger, pod_name, scenario,
 340                                           payload={'timestart': start_time_ts,
 341                                                    'duration': duration,
 342                                                    'status': test_status})
 343     except:
 344         logger.error("Error pushing results into Database '%s'" % sys.exc_info()[0])
 345
 346
 347 def main():
 348
 349     creds_nova = functest_utils.get_credentials("nova")
 350     nova_client = novaclient.Client('2', **creds_nova)
 351     creds_neutron = functest_utils.get_credentials("neutron")
 352     neutron_client = neutronclient.Client(**creds_neutron)
 353     creds_keystone = functest_utils.get_credentials("keystone")
 354     keystone_client = keystoneclient.Client(**creds_keystone)
 355     glance_endpoint = keystone_client.service_catalog.url_for(service_type='image',
 356                                                               endpoint_type='publicURL')
 357     glance_client = glanceclient.Client(1, glance_endpoint,
 358                                         token=keystone_client.auth_token)
 359     EXIT_CODE = -1
 360
 361     image_id = None
 362     flavor = None
 363
 364     # Check if the given image exists
 365     image_id = functest_utils.get_image_id(glance_client, GLANCE_IMAGE_NAME)
 366     if image_id != '':
 367         logger.info("Using existing image '%s'..." % GLANCE_IMAGE_NAME)
 368     else:
 369         logger.info("Creating image '%s' from '%s'..." % (GLANCE_IMAGE_NAME,
 370                                                        GLANCE_IMAGE_PATH))
 371         image_id = functest_utils.create_glance_image(glance_client,
 372                                                       GLANCE_IMAGE_NAME,
 373                                                       GLANCE_IMAGE_PATH)
 374         if not image_id:
 375             logger.error("Failed to create a Glance image...")
 376             return(EXIT_CODE)
 377         logger.debug("Image '%s' with ID=%s created successfully." %\
 378                   (GLANCE_IMAGE_NAME, image_id))
 379
 380
 381     network_dic = create_private_neutron_net(neutron_client)
 382     if not network_dic:
 383         logger.error(
 384             "There has been a problem when creating the neutron network")
 385         return(EXIT_CODE)
 386
 387     network_id = network_dic["net_id"]
 388
 389     sg_id = create_security_group(neutron_client)
 390
 391     # Check if the given flavor exists
 392     try:
 393         flavor = nova_client.flavors.find(name=FLAVOR)
 394         logger.info("Using existing Flavor '%s'..." % FLAVOR)
 395     except:
 396         logger.error("Flavor '%s' not found." % FLAVOR)
 397         logger.info("Available flavors are: ")
 398         pMsg(nova_client.flavor.list())
 399         return(EXIT_CODE)
 400
 401     # Deleting instances if they exist
 402     servers = nova_client.servers.list()
 403     for server in servers:
 404         if server.name == NAME_VM_1 or server.name == NAME_VM_2:
 405             logger.info("Instance %s found. Deleting..." % server.name)
 406             server.delete()
 407
 408
 409     # boot VM 1
 410     start_time_ts = time.time()
 411     end_time_ts = start_time_ts
 412     logger.info("vPing Start Time:'%s'" % (
 413         datetime.datetime.fromtimestamp(start_time_ts).strftime(
 414             '%Y-%m-%d %H:%M:%S')))
 415
 416     logger.info("Creating instance '%s'..." % NAME_VM_1)
 417     logger.debug(
 418         "Configuration:\n name=%s \n flavor=%s \n image=%s \n "
 419         "network=%s \n" % (NAME_VM_1, flavor, image_id, network_id))
 420     vm1 = nova_client.servers.create(
 421         name=NAME_VM_1,
 422         flavor=flavor,
 423         image=image_id,
 424         nics=[{"net-id": network_id}]
 425     )
 426
 427     # wait until VM status is active
 428     if not waitVmActive(nova_client, vm1):
 429         logger.error("Instance '%s' cannot be booted. Status is '%s'" % (
 430             NAME_VM_1, functest_utils.get_instance_status(nova_client, vm1)))
 431         cleanup(nova_client, neutron_client, image_id, network_dic, sg_id, floatingip)
 432         return (EXIT_CODE)
 433     else:
 434         logger.info("Instance '%s' is ACTIVE." % NAME_VM_1)
 435
 436     # Retrieve IP of first VM
 437     test_ip = vm1.networks.get(NEUTRON_PRIVATE_NET_NAME)[0]
 438     logger.debug("Instance '%s' got private ip '%s'." % (NAME_VM_1, test_ip))
 439
 440     logger.info("Adding '%s' to security group '%s'..." % (NAME_VM_1, SECGROUP_NAME))
 441     functest_utils.add_secgroup_to_instance(nova_client, vm1.id, sg_id)
 442
 443     # boot VM 2
 444     logger.info("Creating instance '%s'..." % NAME_VM_2)
 445     logger.debug(
 446         "Configuration:\n name=%s \n flavor=%s \n image=%s \n "
 447         "network=%s \n" % (NAME_VM_2, flavor, image_id, network_id))
 448     vm2 = nova_client.servers.create(
 449         name=NAME_VM_2,
 450         flavor=flavor,
 451         image=image_id,
 452         nics=[{"net-id": network_id}]
 453     )
 454
 455     if not waitVmActive(nova_client, vm2):
 456         logger.error("Instance '%s' cannot be booted. Status is '%s'" % (
 457             NAME_VM_2, functest_utils.get_instance_status(nova_client, vm2)))
 458         cleanup(nova_client, neutron_client, image_id, network_dic, sg_id, floatip_dic)
 459         return (EXIT_CODE)
 460     else:
 461         logger.info("Instance '%s' is ACTIVE." % NAME_VM_2)
 462
 463     logger.info("Adding '%s' to security group '%s'..." % (NAME_VM_2, SECGROUP_NAME))
 464     functest_utils.add_secgroup_to_instance(nova_client, vm2.id, sg_id)
 465
 466     logger.info("Creating floating IP for VM '%s'..." % NAME_VM_2)
 467     floatip_dic = functest_utils.create_floating_ip(neutron_client)
 468     floatip = floatip_dic['fip_addr']
 469     floatip_id = floatip_dic['fip_id']
 470
 471     if floatip == None:
 472         logger.error("Cannot create floating IP.")
 473         cleanup(nova_client, neutron_client, image_id, network_dic, sg_id, floatip_dic)
 474         return (EXIT_CODE)
 475     logger.info("Floating IP created: '%s'" % floatip)
 476
 477     logger.info("Associating floating ip: '%s' to VM '%s' " % (floatip, NAME_VM_2))
 478     if not functest_utils.add_floating_ip(nova_client, vm2.id, floatip):
 479         logger.error("Cannot associate floating IP to VM.")
 480         cleanup(nova_client, neutron_client, image_id, network_dic, sg_id, floatip_dic)
 481         return (EXIT_CODE)
 482
 483     logger.info("Trying to establish SSH connection to %s..." % floatip)
 484     username='cirros'
 485     password='cubswin:)'
 486     ssh = paramiko.SSHClient()
 487     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 488
 489     timeout = 50
 490     nolease = False
 491     discover_count = 0
 492     while timeout > 0:
 493         try:
 494             ssh.connect(floatip, username=username, password=password, timeout=2)
 495             logger.debug("SSH connection established to %s." % floatip)
 496             break
 497         except Exception, e:
 498             logger.debug("Waiting for %s..." % floatip)
 499             time.sleep(6)
 500             timeout -= 1
 501         console_log = vm2.get_console_output()
 502         if "Sending discover" in console_log and \
 503             discover_count % 4 == 0 and not nolease :
 504             logger.debug("Console-log '%s': Sending discover..." % NAME_VM_2)
 505         elif "No lease, failing" in console_log and\
 506                 "network failed" in console_log and not nolease:
 507                 nolease = True
 508                 logger.debug("Console-log '%s': No lease, failing..." % NAME_VM_2)
 509                 logger.info("The instance failed to get an IP from "\
 510                             "the DHCP agent. The test will probably timeout...")
 511         discover_count += 1
 512
 513     if timeout == 0: # 300 sec timeout (5 min)
 514         logger.error("Cannot establish connection to IP '%s'. Aborting" % floatip)
 515         cleanup(nova_client, neutron_client, image_id, network_dic, sg_id, floatip_dic)
 516         return (EXIT_CODE)
 517
 518     scp = SCPClient(ssh.get_transport())
 519
 520     ping_script = REPO_PATH + "testcases/vPing/CI/libraries/ping.sh"
 521     try:
 522         scp.put(ping_script,"~/")
 523     except Exception, e:
 524         logger.error("Cannot SCP the file '%s' to VM '%s'" % (ping_script,floatip))
 525
 526
 527     cmd = 'chmod 755 ~/ping.sh'
 528     (stdin, stdout, stderr) = ssh.exec_command(cmd)
 529     for line in stdout.readlines():
 530         print line
 531
 532     logger.info("Waiting for ping...")
 533     sec = 0
 534     duration = 0
 535
 536     cmd = '~/ping.sh ' + test_ip
 537     flag = False
 538     while True:
 539         time.sleep(1)
 540         (stdin, stdout, stderr) = ssh.exec_command(cmd)
 541         output = stdout.readlines()
 542
 543
 544         for line in output:
 545             if "vPing OK" in line:
 546                 logger.info("vPing detected!")
 547
 548                 # we consider start time at VM1 booting
 549                 end_time_ts = time.time()
 550                 duration = round(end_time_ts - start_time_ts, 1)
 551                 logger.info("vPing duration:'%s' s." % duration)
 552                 EXIT_CODE = 0
 553                 flag = True
 554                 break
 555             elif sec == PING_TIMEOUT:
 556                 logger.info("Timeout reached.")
 557                 flag = True
 558                 break
 559         if flag :
 560             break
 561         logger.debug("Pinging %s. Waiting for response..." % test_ip)
 562         sec += 1
 563
 564     cleanup(nova_client, neutron_client, image_id, network_dic, sg_id, floatip_dic)
 565
 566     test_status = "NOK"
 567     if EXIT_CODE == 0:
 568         logger.info("vPing OK")
 569         test_status = "OK"
 570     else:
 571         duration = 0
 572         logger.error("vPing FAILED")
 573
 574     if args.report:
 575         push_results(start_time_ts, duration, test_status)
 576
 577     exit(EXIT_CODE)
 578
 579 if __name__ == '__main__':
 580     main()