VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py

   1 #!/usr/bin/python
   2
   3 ##
   4 ## Copyright (c) 2010-2017 Intel Corporation
   5 ##
   6 ## Licensed under the Apache License, Version 2.0 (the "License");
   7 ## you may not use this file except in compliance with the License.
   8 ## You may obtain a copy of the License at
   9 ##
  10 ##     http://www.apache.org/licenses/LICENSE-2.0
  11 ##
  12 ## Unless required by applicable law or agreed to in writing, software
  13 ## distributed under the License is distributed on an "AS IS" BASIS,
  14 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 ## See the License for the specific language governing permissions and
  16 ## limitations under the License.
  17 ##
  18
  19 from __future__ import print_function
  20
  21 import os
  22 import stat
  23 import sys
  24 import time
  25 import subprocess
  26 import getopt
  27 import re
  28 import logging
  29 from logging.handlers import RotatingFileHandler
  30 from logging import handlers
  31 from prox_ctrl import prox_ctrl
  32 import ConfigParser
  33 import ast
  34 import atexit
  35 import csv
  36
  37 version="19.6.30"
  38 env = "rapid.env" #Default string for environment
  39 test_file = "basicrapid.test" #Default string for test
  40 machine_map_file = "machine.map" #Default string for machine map file
  41 loglevel="DEBUG" # sets log level for writing to file
  42 screenloglevel="INFO" # sets log level for writing to screen
  43 runtime=10 # time in seconds for 1 test run
  44 configonly = False # IF True, the system will upload all the necessary config fiels to the VMs, but not start PROX and the actual testing
  45
  46 def usage():
  47         print("usage: runrapid    [--version] [-v]")
  48         print("                   [--env ENVIRONMENT_NAME]")
  49         print("                   [--test TEST_NAME]")
  50         print("                   [--map MACHINE_MAP_FILE]")
  51         print("                   [--runtime TIME_FOR_TEST]")
  52         print("                   [--configonly False|True]")
  53         print("                   [--log DEBUG|INFO|WARNING|ERROR|CRITICAL]")
  54         print("                   [-h] [--help]")
  55         print("")
  56         print("Command-line interface to runrapid")
  57         print("")
  58         print("optional arguments:")
  59         print("  -v,  --version                 Show program's version number and exit")
  60         print("  --env ENVIRONMENT_NAME         Parameters will be read from ENVIRONMENT_NAME. Default is %s."%env)
  61         print("  --test TEST_NAME               Test cases will be read from TEST_NAME. Default is %s."%test_file)
  62         print("  --map MACHINE_MAP_FILE Machine mapping will be read from MACHINE_MAP_FILE. Default is %s."%machine_map_file)
  63         print("  --runtime                      Specify time in seconds for 1 test run")
  64         print("  --configonly                   If True, only upload all config files to the VMs, do not run the tests. Default is %s."%configonly)
  65         print("  --log                          Specify logging level for log file output, default is DEBUG")
  66         print("  --screenlog                    Specify logging level for screen output, default is INFO")
  67         print("  -h, --help                     Show help message and exit.")
  68         print("")
  69
  70 try:
  71         opts, args = getopt.getopt(sys.argv[1:], "vh", ["version","help", "env=", "test=", "map=", "runtime=","configonly=","log=","screenlog="])
  72 except getopt.GetoptError as err:
  73         print("===========================================")
  74         print(str(err))
  75         print("===========================================")
  76         usage()
  77         sys.exit(2)
  78 if args:
  79         usage()
  80         sys.exit(2)
  81 for opt, arg in opts:
  82         if opt in ("-h", "--help"):
  83                 usage()
  84                 sys.exit()
  85         if opt in ("-v", "--version"):
  86                 print("Rapid Automated Performance Indication for Dataplane "+version)
  87                 sys.exit()
  88         if opt in ("--env"):
  89                 env = arg
  90         if opt in ("--test"):
  91                 test_file = arg
  92         if opt in ("--map"):
  93                 machine_map_file = arg
  94         if opt in ("--runtime"):
  95                 runtime = arg
  96         if opt in ("--configonly"):
  97                 configonly = arg
  98                 if configonly == 'True':
  99                         configonly = True
 100                         print('No actual runs, only uploading configuration files')
 101                 else:
 102                         configonly = False
 103                         print('--configonly parameter is defaulted to False')
 104         if opt in ("--log"):
 105                 loglevel = arg
 106                 print ("Log level: "+ loglevel)
 107         if opt in ("--screenlog"):
 108                 screenloglevel = arg
 109                 print ("Screen Log level: "+ screenloglevel)
 110
 111 print ("Using '"+env+"' as name for the environment")
 112 print ("Using '"+test_file+"' for test case definition")
 113 print ("Using '"+machine_map_file+"' for machine mapping")
 114 print ("Runtime: "+ runtime)
 115
 116 class bcolors:
 117         HEADER = '\033[95m'
 118         OKBLUE = '\033[94m'
 119         OKGREEN = '\033[92m'
 120         WARNING = '\033[93m'
 121         FAIL = '\033[91m'
 122         ENDC = '\033[0m'
 123         BOLD = '\033[1m'
 124         UNDERLINE = '\033[4m'
 125
 126 # create formatters
 127 screen_formatter = logging.Formatter("%(message)s")
 128 file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 129
 130 # get a top-level logger,
 131 # set its log level,
 132 # BUT PREVENT IT from propagating messages to the root logger
 133 #
 134 log = logging.getLogger()
 135 numeric_level = getattr(logging, loglevel.upper(), None)
 136 if not isinstance(numeric_level, int):
 137         raise ValueError('Invalid log level: %s' % loglevel)
 138 log.setLevel(numeric_level)
 139 log.propagate = 0
 140
 141 # create a console handler
 142 # and set its log level to the command-line option
 143 #
 144 console_handler = logging.StreamHandler(sys.stdout)
 145 #console_handler.setLevel(logging.INFO)
 146 numeric_screenlevel = getattr(logging, screenloglevel.upper(), None)
 147 if not isinstance(numeric_screenlevel, int):
 148         raise ValueError('Invalid screenlog level: %s' % screenloglevel)
 149 console_handler.setLevel(numeric_screenlevel)
 150 console_handler.setFormatter(screen_formatter)
 151
 152 # create a file handler
 153 # and set its log level
 154 #
 155 log_file = 'RUN{}.{}.log'.format(env,test_file)
 156 file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10)
 157 #file_handler = log.handlers.TimedRotatingFileHandler(log_file, 'D', 1, 5)
 158 file_handler.setLevel(numeric_level)
 159 file_handler.setFormatter(file_formatter)
 160
 161 # add handlers to the logger
 162 #
 163 log.addHandler(file_handler)
 164 log.addHandler(console_handler)
 165
 166 # Check if log exists and should therefore be rolled
 167 needRoll = os.path.isfile(log_file)
 168
 169
 170 # This is a stale log, so roll it
 171 if needRoll:
 172         # Add timestamp
 173         log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime())
 174
 175         # Roll over on application start
 176         log.handlers[0].doRollover()
 177
 178 # Add timestamp
 179 log.debug('\n---------\nLog started on %s.\n---------\n' % time.asctime())
 180
 181 log.debug("runrapid.py version: "+version)
 182 #========================================================================
 183 def connect_socket(client):
 184         attempts = 1
 185         log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 186         sock = None
 187         while True:
 188                 sock = client.prox_sock()
 189                 if sock is not None:
 190                         break
 191                 attempts += 1
 192                 if attempts > 20:
 193                         log.exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 194                         raise Exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 195                 time.sleep(2)
 196                 log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 197         log.info("Connected to PROX on %s" % client.ip())
 198         return sock
 199
 200 def connect_client(client):
 201         attempts = 1
 202         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 203         while True:
 204                 try:
 205                         client.connect()
 206                         break
 207                 except RuntimeWarning, ex:
 208                         attempts += 1
 209                         if attempts > 20:
 210                                 log.exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 211                                 raise Exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 212                         time.sleep(2)
 213                         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 214         log.debug("Connected to VM on %s" % client.ip())
 215
 216 def run_iteration(gensock,sutsock):
 217         sleep_time = 2
 218         # Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
 219         time.sleep(sleep_time)
 220         abs_old_rx, abs_old_non_dp_rx, abs_old_tx, abs_old_non_dp_tx, abs_old_drop, abs_old_tx_fail, abs_old_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,tasks)
 221         abs_old_rx = abs_old_rx - abs_old_non_dp_rx
 222         abs_old_tx = abs_old_tx - abs_old_non_dp_tx
 223         gensock.start(gencores)
 224         time.sleep(sleep_time)
 225         if sutsock!='none':
 226                 old_sut_rx, old_sut_non_dp_rx, old_sut_tx, old_sut_non_dp_tx, old_sut_drop, old_sut_tx_fail, old_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 227                 old_sut_rx = old_sut_rx - old_sut_non_dp_rx
 228                 old_sut_tx = old_sut_tx - old_sut_non_dp_tx
 229         old_rx, old_non_dp_rx, old_tx, old_non_dp_tx, old_drop, old_tx_fail, old_tsc, tsc_hz = gensock.core_stats(genstatcores,tasks)
 230         old_rx = old_rx - old_non_dp_rx
 231         old_tx = old_tx - old_non_dp_tx
 232         # Measure latency statistics per second
 233         n_loops = 0
 234         lat_min = 0
 235         lat_max = 0
 236         lat_avg = 0
 237         used_avg = 0
 238         while n_loops < float(runtime):
 239                 n_loops +=1
 240                 time.sleep(1)
 241                 lat_min_sample, lat_max_sample, lat_avg_sample, used_sample = gensock.lat_stats(latcores)
 242                 if lat_min > lat_min_sample:
 243                         lat_min = lat_min_sample
 244                 if lat_max < lat_max_sample:
 245                         lat_max = lat_max_sample
 246                 lat_avg = lat_avg + lat_avg_sample
 247                 used_avg = used_avg + used_sample
 248                 lat_avg = lat_avg / n_loops
 249         used_avg = used_avg / n_loops
 250         # Get statistics after some execution time
 251         new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = gensock.core_stats(genstatcores,tasks)
 252         new_rx = new_rx - new_non_dp_rx
 253         new_tx = new_tx - new_non_dp_tx
 254         if sutsock!='none':
 255                 new_sut_rx, new_sut_non_dp_rx, new_sut_tx, new_sut_non_dp_tx, new_sut_drop, new_sut_tx_fail, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 256                 new_sut_rx = new_sut_rx - new_sut_non_dp_rx
 257                 new_sut_tx = new_sut_tx - new_sut_non_dp_tx
 258         #Stop generating
 259         gensock.stop(gencores)
 260         time.sleep(sleep_time)
 261         abs_new_rx, abs_new_non_dp_rx, abs_new_tx, abs_new_non_dp_tx, abs_new_drop, abs_new_tx_fail, abs_new_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,tasks)
 262         abs_new_rx = abs_new_rx - abs_new_non_dp_rx
 263         abs_new_tx = abs_new_tx - abs_new_non_dp_tx
 264         drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 265         rx = new_rx - old_rx     # rx is all packets received by the nop task = all packets received in the gen VM
 266         tx = new_tx - old_tx     # tx is all generated packets actually accepted by the interface
 267         abs_dropped = (abs_new_tx - abs_old_tx) - (abs_new_rx - abs_old_rx)
 268         tsc = new_tsc - old_tsc  # time difference between the 2 measurements, expressed in cycles.
 269         pps_req_tx = (tx+drop-rx)*tsc_hz*1.0/(tsc*1000000)
 270         pps_tx = tx*tsc_hz*1.0/(tsc*1000000)
 271         pps_rx = rx*tsc_hz*1.0/(tsc*1000000)
 272         if sutsock!='none':
 273                 sut_rx = new_sut_rx - old_sut_rx
 274                 sut_tx = new_sut_tx - old_sut_tx
 275                 sut_tsc = new_sut_tsc - old_sut_tsc
 276                 pps_sut_tx = sut_tx*sut_tsc_hz*1.0/(sut_tsc*1000000)
 277                 pps_sut_tx_str = '{:>9.3f}'.format(pps_sut_tx)
 278         else:
 279                 pps_sut_tx = 0
 280                 pps_sut_tx_str = 'NO MEAS.'
 281         if (tx == 0):
 282                 log.critical("TX = 0. Test interrupted since no packet has been sent.")
 283                 raise Exception("TX = 0")
 284         return(pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,(abs_new_tx_fail - abs_old_tx_fail),(abs_new_tx - abs_old_tx),lat_min,used_avg)
 285
 286 def new_speed(speed,minspeed,maxspeed,success):
 287         if success:
 288                 minspeed = speed
 289         else:
 290                 maxspeed = speed
 291         newspeed = (maxspeed+minspeed)/2.0
 292         return (newspeed,minspeed,maxspeed)
 293
 294 def get_pps(speed,size):
 295         # speed is given in % of 10Gb/s, returning Mpps
 296         return (speed * 100.0 / (8*(size+24)))
 297
 298 def get_speed(packet_speed,size):
 299         # return speed in Gb/s
 300         return (packet_speed / 1000.0 * (8*(size+24)))
 301
 302
 303 def run_flow_size_test(gensock,sutsock):
 304         gensock.start(latcores)
 305         for size in packet_size_list:
 306                 size = size-4
 307                 gensock.set_size(gencores,0,size) # This is setting the frame size
 308                 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 309                 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 310                 # This will only work when using sending UDP packets. For different protocls and ehternet types, we would need a different calculation
 311                 log.info("+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 312                 log.info("| UDP, "+ '{:>5}'.format(size+4) +" bytes, different number of flows by randomizing SRC & DST UDP port                                                                                           |")
 313                 log.info("+--------+--------------------+----------------+----------------+----------------+------------------------+----------------+----------------+----------------+------------+")
 314                 log.info("| Flows  |  Speed requested   | core generated | Sent by Gen NIC| Forward by SUT |      core received     |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
 315                 log.info("+--------+--------------------+----------------+----------------+----------------+------------------------+----------------+----------------+----------------+------------+")
 316                 for flow_number in flow_size_list:
 317                         attempts = 0
 318                         gensock.reset_stats()
 319                         if sutsock!='none':
 320                                 sutsock.reset_stats()
 321                         source_port,destination_port = flows[flow_number]
 322                         gensock.set_random(gencores,0,34,source_port,2)
 323                         gensock.set_random(gencores,0,36,destination_port,2)
 324                         endpps_sut_tx_str = 'NO_RESULTS'
 325                         maxspeed = speed = STARTSPEED
 326                         minspeed = 0
 327                         while (maxspeed-minspeed > ACCURACY):
 328                                 attempts += 1
 329                                 endwarning =''
 330                                 print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 331                                 sys.stdout.flush()
 332                                 # Start generating packets at requested speed (in % of a 10Gb/s link)
 333                                 gensock.speed(speed / len(gencores), gencores)
 334                                 time.sleep(1)
 335                                 # Get statistics now that the generation is stable and initial ARP messages are dealt with
 336                                 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used = run_iteration(gensock,sutsock)
 337                                 drop_rate = 100.0*abs_dropped/abs_tx
 338                                 if lat_used < 0.95:
 339                                         lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 340                                 else:
 341                                         lat_warning = ''
 342                                 if ((drop_rate < DROP_RATE_TRESHOLD) or (abs_dropped==DROP_RATE_TRESHOLD ==0)) and (lat_avg< LAT_AVG_TRESHOLD) and (lat_max < LAT_MAX_TRESHOLD):
 343                                         lat_avg_prefix = bcolors.ENDC
 344                                         lat_max_prefix = bcolors.ENDC
 345                                         abs_drop_rate_prefix = bcolors.ENDC
 346                                         drop_rate_prefix = bcolors.ENDC
 347                                         if ((get_pps(speed,size) - pps_tx)/get_pps(speed,size))>0.01:
 348                                                 speed_prefix = bcolors.WARNING
 349                                                 if abs_tx_fail > 0:
 350                                                         gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
 351                                                 else:
 352                                                         gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(get_pps(speed,size), pps_tx) + bcolors.ENDC
 353                                         else:
 354                                                 speed_prefix = bcolors.ENDC
 355                                                 gen_warning = ''
 356                                         endspeed = speed
 357                                         endpps_req_tx = pps_req_tx
 358                                         endpps_tx = pps_tx
 359                                         endpps_sut_tx_str = pps_sut_tx_str
 360                                         endpps_rx = pps_rx
 361                                         endlat_avg = lat_avg
 362                                         endlat_max = lat_max
 363                                         endabs_dropped = abs_dropped
 364                                         enddrop_rate = drop_rate
 365                                         if lat_warning or gen_warning:
 366                                                 endwarning = '|        | {:167.167} |'.format(lat_warning + gen_warning)
 367                                         success = True
 368                                         success_message='%  | SUCCESS'
 369                                 else:
 370                                         success_message='%  | FAILED'
 371                                         gen_warning = ''
 372                                         abs_drop_rate_prefix = bcolors.ENDC
 373                                         if ((abs_dropped>0) and (DROP_RATE_TRESHOLD ==0)):
 374                                                 abs_drop_rate_prefix = bcolors.FAIL
 375                                         if (drop_rate < DROP_RATE_TRESHOLD):
 376                                                 drop_rate_prefix = bcolors.ENDC
 377                                         else:
 378                                                 drop_rate_prefix = bcolors.FAIL
 379                                         if (lat_avg< LAT_AVG_TRESHOLD):
 380                                                 lat_avg_prefix = bcolors.ENDC
 381                                         else:
 382                                                 lat_avg_prefix = bcolors.FAIL
 383                                         if (lat_max< LAT_MAX_TRESHOLD):
 384                                                 lat_max_prefix = bcolors.ENDC
 385                                         else:
 386                                                 lat_max_prefix = bcolors.FAIL
 387                                         if (((get_pps(speed,size) - pps_tx)/get_pps(speed,size))<0.001):
 388                                                 speed_prefix = bcolors.ENDC
 389                                         else:
 390                                                 speed_prefix = bcolors.FAIL
 391                                         success = False
 392                                 log.debug('|step{:>3}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% '+speed_prefix +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | ' + '{:>9.3f}'.format(pps_tx) +' Mpps | '+ bcolors.ENDC  + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+bcolors.OKBLUE + '{:>4.1f}'.format(get_speed(pps_rx,size)) + 'Gb/s{:>9.3f}'.format(pps_rx)+' Mpps'+bcolors.ENDC+' | '+lat_avg_prefix+ '{:>9.0f}'.format(lat_avg)+' us   | '+lat_max_prefix+ '{:>9.0f}'.format(lat_max)+' us   | '+ abs_drop_rate_prefix + '{:>14d}'.format(abs_dropped)+drop_rate_prefix+ ' |''{:>9.2f}'.format(drop_rate)+bcolors.ENDC+ success_message +lat_warning + gen_warning)
 393                                 speed,minspeed,maxspeed = new_speed(speed,minspeed,maxspeed,success)
 394                         if endpps_sut_tx_str !=  'NO_RESULTS':
 395                                 log.info('|{:>7}'.format(str(flow_number))+" | " + '{:>5.1f}'.format(endspeed) + '% ' + speed_prefix + '{:>6.3f}'.format(get_pps(endspeed,size)) + ' Mpps | '+ '{:>9.3f}'.format(endpps_req_tx)+ ' Mpps | '+ bcolors.ENDC + '{:>9.3f}'.format(endpps_tx) +' Mpps | ' + '{:>9}'.format(endpps_sut_tx_str) +' Mpps | '+bcolors.OKBLUE + '{:>4.1f}'.format(get_speed(pps_rx,size)) + 'Gb/s{:>9.3f}'.format(endpps_rx)+' Mpps'+bcolors.ENDC+' | '+ '{:>9.0f}'.format(endlat_avg)+' us   | '+ '{:>9.0f}'.format(endlat_max)+' us   | '+ '{:>14d}'.format(endabs_dropped)+ ' |'+'{:>9.2f}'.format(enddrop_rate)+ '%  |')
 396                                 if endwarning:
 397                                         log.info (endwarning)
 398                                 log.info("+--------+--------------------+----------------+----------------+----------------+------------------------+----------------+----------------+----------------+------------+")
 399                                 writer.writerow({'flow':flow_number,'size':(size+4),'endspeed':endspeed,'endspeedpps':get_pps(endspeed,size),'endpps_req_tx':endpps_req_tx,'endpps_tx':endpps_tx,'endpps_sut_tx_str':endpps_sut_tx_str,'endpps_rx':endpps_rx,'endlat_avg':endlat_avg,'endlat_max':endlat_max,'endabs_dropped':endabs_dropped,'enddrop_rate':enddrop_rate})
 400                         else:
 401                                 log.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
 402         gensock.stop(latcores)
 403
 404
 405 def run_fixed_rate(gensock,sutsock):
 406         log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------+")
 407         log.info("| UDP, 1 flow, different packet sizes                                                                                                                       |")
 408         log.info("+-----+------------------+-------------+-------------+-------------+-------------+-------------+-------------+-----------+-----------+---------+------------+")
 409         log.info("|Pktsz| Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core| Avg. Latency| Max. Latency|   Sent    |  Received |  Lost   | Total Lost |")
 410         log.info("+-----+------------------+-------------+-------------+-------------+-------------+-------------+-------------+-----------+-----------+---------+------------+")
 411         sleep_time = 3
 412         gensock.start(latcores)
 413         for size in packet_size_list:
 414                 # Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
 415                 time.sleep(sleep_time)
 416                 size = size-4
 417                 gensock.reset_stats()
 418                 if sutsock!='none':
 419                         sutsock.reset_stats()
 420                 gensock.set_size(gencores,0,size) # This is setting the frame size
 421                 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 422                 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 423                 # This will only work when using sending UDP packets. For different protocls and ehternet types, we would need a differnt calculation
 424                 pps_sut_tx_str = 'NO_RESULTS'
 425                 speed = STARTSPEED
 426                 # Start generating packets at requested speed (in % of a 10Gb/s link)
 427                 gensock.speed(speed / len(gencores), gencores)
 428                 duration = float(runtime)
 429                 first = 1
 430                 tot_drop = 0
 431                 if sutsock!='none':
 432                         old_sut_rx, old_sut_non_dp_rx, old_sut_tx, old_sut_non_dp_tx, old_sut_drop, old_sut_tx_fail, old_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 433                         old_sut_rx = old_sut_rx - old_sut_non_dp_rx
 434                         old_sut_tx = old_sut_tx - old_sut_non_dp_tx
 435                 old_rx, old_non_dp_rx, old_tx, old_non_dp_tx, old_drop, old_tx_fail, old_tsc, tsc_hz = gensock.core_stats(genstatcores,tasks)
 436                 old_rx = old_rx - old_non_dp_rx
 437                 old_tx = old_tx - old_non_dp_tx
 438                 gensock.start(gencores)
 439                 while (duration > 0):
 440                         time.sleep(0.5)
 441                         lat_min, lat_max, lat_avg, lat_used = gensock.lat_stats(latcores)
 442                         if lat_used < 0.95:
 443                                 lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 444                         else:
 445                                 lat_warning = ''
 446                         # Get statistics after some execution time
 447                         new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = gensock.core_stats(genstatcores,tasks)
 448                         new_rx = new_rx - new_non_dp_rx
 449                         new_tx = new_tx - new_non_dp_tx
 450                         if sutsock!='none':
 451                                 new_sut_rx, new_sut_non_dp_rx, new_sut_tx, new_sut_non_dp_tx, new_sut_drop, new_sut_tx_fail, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 452                                 new_sut_rx = new_sut_rx - new_sut_non_dp_rx
 453                                 new_sut_tx = new_sut_tx - new_sut_non_dp_tx
 454                                 drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 455                                 rx = new_rx - old_rx     # rx is all packets received by the nop task = all packets received in the gen VM
 456                                 tx = new_tx - old_tx     # tx is all generated packets actually accepted by the interface
 457                                 tsc = new_tsc - old_tsc  # time difference between the 2 measurements, expressed in cycles.
 458                         if tsc == 0 :
 459                                 continue
 460                         if sutsock!='none':
 461                                 sut_rx = new_sut_rx - old_sut_rx
 462                                 sut_tx = new_sut_tx - old_sut_tx
 463                                 sut_tsc = new_sut_tsc - old_sut_tsc
 464                                 if sut_tsc == 0 :
 465                                         continue
 466                         duration = duration - 1
 467                         old_drop = new_drop
 468                         old_rx = new_rx
 469                         old_tx = new_tx
 470                         old_tsc = new_tsc
 471                         pps_req_tx = (tx+drop-rx)*tsc_hz*1.0/(tsc*1000000)
 472                         pps_tx = tx*tsc_hz*1.0/(tsc*1000000)
 473                         pps_rx = rx*tsc_hz*1.0/(tsc*1000000)
 474                         if sutsock!='none':
 475                                 old_sut_tx = new_sut_tx
 476                                 old_sut_rx = new_sut_rx
 477                                 old_sut_tsc= new_sut_tsc
 478                                 pps_sut_tx = sut_tx*sut_tsc_hz*1.0/(sut_tsc*1000000)
 479                                 pps_sut_tx_str = '{:>7.3f}'.format(pps_sut_tx)
 480                         else:
 481                                 pps_sut_tx = 0
 482                                 pps_sut_tx_str = 'NO MEAS.'
 483                         if (tx == 0):
 484                                 log.critical("TX = 0. Test interrupted since no packet has been sent.")
 485                                 raise Exception("TX = 0")
 486                         tot_drop = tot_drop + tx - rx
 487
 488                         if pps_sut_tx_str !=  'NO_RESULTS':
 489                                 # First second mpps are not valid as there is no alignement between time the generator is started and per seconds stats
 490                                 if (first):
 491                                         log.info('|{:>4}'.format(size+4)+" |" + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+'             |' +'             |'  +'             |'+ '             |'+ '{:>8.0f}'.format(lat_avg)+' us  |'+'{:>8.0f}'.format(lat_max)+' us  | ' + '{:>9.0f}'.format(tx) + ' | '+ '{:>9.0f}'.format(rx) + ' | '+ '{:>7.0f}'.format(tx-rx) + ' | '+'{:>7.0f}'.format(tot_drop) +'    |'+lat_warning)
 492                                 else:
 493                                         log.info('|{:>4}'.format(size+4)+" |" + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+ '{:>7.3f}'.format(pps_req_tx)+' Mpps |'+ '{:>7.3f}'.format(pps_tx) +' Mpps |' + '{:>7}'.format(pps_sut_tx_str) +' Mpps |'+ '{:>7.3f}'.format(pps_rx)+' Mpps |'+ '{:>8.0f}'.format(lat_avg)+' us  |'+'{:>8.0f}'.format(lat_max)+' us  | ' + '{:>9.0f}'.format(tx) + ' | '+ '{:>9.0f}'.format(rx) + ' | '+ '{:>7.0f}'.format(tx-rx) + ' | '+ '{:>7.0f}'.format(tot_drop) +'    |'+lat_warning)
 494                         else:
 495                                 log.debug('|{:>7}'.format(str(size))+" | Speed 0 or close to 0")
 496                         first = 0
 497                         if (duration <= 0):
 498                                 #Stop generating
 499                                 gensock.stop(gencores)
 500                                 time.sleep(sleep_time)
 501                                 lat_min, lat_max, lat_avg, lat_used = gensock.lat_stats(latcores)
 502                                 if lat_used < 0.95:
 503                                         lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 504                                 else:
 505                                         lat_warning = ''
 506                                 # Get statistics after some execution time
 507                                 new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = gensock.core_stats(genstatcores,tasks)
 508                                 new_rx = new_rx - new_non_dp_rx
 509                                 new_tx = new_tx - new_non_dp_tx
 510                                 if sutsock!='none':
 511                                         new_sut_rx, new_sut_non_dp_rx, new_sut_tx, new_sut_non_dp_tx, new_sut_drop, new_sut_tx_fail, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 512                                         new_sut_rx = new_sut_rx - new_sut_non_dp_rx
 513                                         new_sut_tx = new_sut_tx - new_sut_non_dp_tx
 514                                 drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 515                                 rx = new_rx - old_rx     # rx is all packets received by the nop task = all packets received in the gen VM
 516                                 tx = new_tx - old_tx     # tx is all generated packets actually accepted by the interface
 517                                 tsc = new_tsc - old_tsc  # time difference between the 2 measurements, expressed in cycles.
 518                                 tot_drop = tot_drop + tx - rx
 519                                 if sutsock!='none':
 520                                         sut_rx = new_sut_rx - old_sut_rx
 521                                         sut_tx = new_sut_tx - old_sut_tx
 522                                         sut_tsc = new_sut_tsc - old_sut_tsc
 523                                 if pps_sut_tx_str !=  'NO_RESULTS':
 524                                         log.info('|{:>4}'.format(size+4)+" |" + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+'             |' +'             |'  +'             |'+ '             |'+ '{:>8.0f}'.format(lat_avg)+' us  |'+'{:>8.0f}'.format(lat_max)+' us  | ' + '{:>9.0f}'.format(tx) + ' | '+ '{:>9.0f}'.format(rx) + ' | '+ '{:>7.0f}'.format(tx-rx) + ' | '+ '{:>7.0f}'.format(tot_drop) +'    |'+lat_warning)
 525                 log.info("+-----+------------------+-------------+-------------+-------------+-------------+-------------+-------------+-----------+-----------+---------+------------+")
 526         gensock.stop(latcores)
 527
 528 def run_measure_swap(sutsock):
 529         log.info("+------------------------------------------------------------------------------------------------------+")
 530         log.info("| Measuring packets on SWAP system                                                                     |")
 531         log.info("+-----------+------------+------------+------------+------------+------------+------------+------------+")
 532         log.info("|    Time   |    RX      |     TX     | non DP RX  | non DP TX  |   TX - RX  | nonDP TX-RX|  DROP TOT  |")
 533         log.info("+-----------+------------+------------+------------+------------+------------+------------+------------+")
 534         sutsock.reset_stats()
 535         duration = float(runtime)
 536         first = 1
 537         tot_drop = 0
 538         old_rx, old_non_dp_rx, old_tx, old_non_dp_tx, old_drop, old_tx_fail, old_tsc, tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 539         while (duration > 0):
 540                 time.sleep(0.5)
 541                 # Get statistics after some execution time
 542                 new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 543                 drop = new_drop-old_drop
 544                 rx = new_rx - old_rx
 545                 tx = new_tx - old_tx
 546                 non_dp_rx = new_non_dp_rx - old_non_dp_rx
 547                 non_dp_tx = new_non_dp_tx - old_non_dp_tx
 548                 tsc = new_tsc - old_tsc
 549                 if tsc == 0 :
 550                         continue
 551                 duration = duration - 1
 552                 old_drop = new_drop
 553                 old_rx = new_rx
 554                 old_tx = new_tx
 555                 old_non_dp_rx = new_non_dp_rx
 556                 old_non_dp_tx = new_non_dp_tx
 557                 old_tsc = new_tsc
 558                 tot_drop = tot_drop + tx - rx
 559
 560                 log.info('|{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(non_dp_rx)+' | '+'{:>10.0f}'.format(non_dp_tx)+' | ' + '{:>10.0f}'.format(tx-rx) + ' | '+ '{:>10.0f}'.format(non_dp_tx-non_dp_rx) + ' | '+'{:>10.0f}'.format(tot_drop) +' |')
 561         log.info("+------------------------------------------------------------------------------------------------------+")
 562
 563
 564 def run_irqtest(sock):
 565         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 566         log.info("| Measuring time probably spent dealing with an interrupt. Interrupting DPDK cores for more than 50us might be problematic   ")
 567         log.info("| and result in packet loss. The first row shows the interrupted time buckets: first number is the bucket between 0us and    ")
 568         log.info("| that number expressed in us and so on. The numbers in the other rows show how many times per second, the program was       ")
 569         log.info("| interrupted for a time as specified by its bucket. '0' is printed when there are no interrupts in this bucket throughout   ")
 570         log.info("| the duration of the test. This is to avoid rounding errors in the case of 0.0                                              ")
 571         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 572         sys.stdout.flush()
 573         buckets=sock.show_irq_buckets(1)
 574         print('Measurement ongoing ... ',end='\r')
 575         sock.stop(irqcores)
 576         old_irq = [[0 for x in range(len(buckets)+1)] for y in range(len(irqcores)+1)]
 577         irq = [[0 for x in range(len(buckets)+1)] for y in range(len(irqcores)+1)]
 578         irq[0][0] = 'bucket us'
 579         for j,bucket in enumerate(buckets,start=1):
 580                 irq[0][j] = '<'+ bucket
 581         irq[0][-1] = '>'+ buckets [-2]
 582         sock.start(irqcores)
 583         time.sleep(2)
 584         for j,bucket in enumerate(buckets,start=1):
 585                 for i,irqcore in enumerate(irqcores,start=1):
 586                         old_irq[i][j] = sock.irq_stats(irqcore,j-1)
 587         time.sleep(float(runtime))
 588         sock.stop(irqcores)
 589         for i,irqcore in enumerate(irqcores,start=1):
 590                 irq[i][0]='core %s '%irqcore
 591                 for j,bucket in enumerate(buckets,start=1):
 592                         diff =  sock.irq_stats(irqcore,j-1) - old_irq[i][j]
 593                         if diff == 0:
 594                                 irq[i][j] = '0'
 595                         else:
 596                                 irq[i][j] = str(round(diff/float(runtime), 2))
 597         for row in irq:
 598                 log.info(''.join(['{:>12}'.format(item) for item in row]))
 599
 600 def run_impairtest(gensock,sutsock):
 601         size=PACKETSIZE-4
 602         log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 603         log.info("| Generator is sending UDP (1 flow) packets ("+ '{:>5}'.format(size+4) +" bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test        |")
 604         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 605         log.info("| Test   |  Speed requested   | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
 606         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 607         attempts = 0
 608         gensock.set_size(gencores,0,size) # This is setting the frame size
 609         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 610         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 611         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 612         gensock.start(latcores)
 613         speed = STARTSPEED
 614         gensock.speed(speed / len(gencores), gencores)
 615         while True:
 616                 attempts += 1
 617                 print('Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 618                 sys.stdout.flush()
 619                 time.sleep(1)
 620                 # Get statistics now that the generation is stable and NO ARP messages any more
 621                 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used = run_iteration(gensock,sutsock)
 622                 drop_rate = 100.0*abs_dropped/abs_tx
 623                 if lat_used < 0.95:
 624                         lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 625                 else:
 626                         lat_warning = ''
 627                 log.info('|{:>7}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | '+ '{:>9.3f}'.format(pps_tx) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9.3f}'.format(pps_rx)+' Mpps | '+ '{:>9.0f}'.format(lat_avg)+' us   | '+ '{:>9.0f}'.format(lat_max)+' us   | '+ '{:>14d}'.format(abs_dropped)+ ' |''{:>9.2f}'.format(drop_rate)+ '%  |'+lat_warning)
 628                 writer.writerow({'flow':'1','size':(size+4),'endspeed':speed,'endspeedpps':get_pps(speed,size),'endpps_req_tx':pps_req_tx,'endpps_tx':pps_tx,'endpps_sut_tx_str':pps_sut_tx_str,'endpps_rx':pps_rx,'endlat_avg':lat_avg,'endlat_max':lat_max,'endabs_dropped':abs_dropped,'enddrop_rate':drop_rate})
 629         gensock.stop(latcores)
 630
 631 def run_warmuptest(gensock):
 632 # Running at low speed to make sure the ARP messages can get through.
 633 # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
 634 # Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
 635 # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
 636         gensock.speed(WARMUPSPEED / len(gencores), gencores)
 637         size=PACKETSIZE-4
 638         gensock.set_size(gencores,0,size) # This is setting the frame size
 639         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 640         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 641         gensock.set_value(gencores,0,56,1,1)
 642         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 643         source_port,destination_port = flows[FLOWSIZE]
 644         gensock.set_random(gencores,0,34,source_port,2)
 645         gensock.set_random(gencores,0,36,destination_port,2)
 646         gensock.start(genstatcores)
 647         time.sleep(WARMUPTIME)
 648         gensock.stop(genstatcores)
 649         gensock.set_value(gencores,0,56,50,1)
 650
 651 global sutstatcores
 652 global genstatcores
 653 global latcores
 654 global gencores
 655 global irqcores
 656 global PACKETSIZE
 657 global packet_size_list
 658 global FLOWSIZE
 659 global flow_size_list
 660 global WARMUPTIME
 661 global WARMUPSPEED
 662 global required_number_of_test_machines
 663 # To generate a desired number of flows, PROX will randomize the bits in source and destination ports, as specified by the bit masks in the flows variable.
 664 flows={\
 665 1:      ['1000000000000000','1000000000000000'],\
 666 2:      ['1000000000000000','100000000000000X'],\
 667 4:      ['100000000000000X','100000000000000X'],\
 668 8:      ['100000000000000X','10000000000000XX'],\
 669 16:     ['10000000000000XX','10000000000000XX'],\
 670 32:     ['10000000000000XX','1000000000000XXX'],\
 671 64:     ['1000000000000XXX','1000000000000XXX'],\
 672 128:    ['1000000000000XXX','100000000000XXXX'],\
 673 256:    ['100000000000XXXX','100000000000XXXX'],\
 674 512:    ['100000000000XXXX','10000000000XXXXX'],\
 675 1024:   ['10000000000XXXXX','10000000000XXXXX'],\
 676 2048:   ['10000000000XXXXX','1000000000XXXXXX'],\
 677 4096:   ['1000000000XXXXXX','1000000000XXXXXX'],\
 678 8192:   ['1000000000XXXXXX','100000000XXXXXXX'],\
 679 16384:  ['100000000XXXXXXX','100000000XXXXXXX'],\
 680 32768:  ['100000000XXXXXXX','10000000XXXXXXXX'],\
 681 65535:  ['10000000XXXXXXXX','10000000XXXXXXXX'],\
 682 131072: ['10000000XXXXXXXX','1000000XXXXXXXXX'],\
 683 262144: ['1000000XXXXXXXXX','1000000XXXXXXXXX'],\
 684 524280: ['1000000XXXXXXXXX','100000XXXXXXXXXX'],\
 685 1048576:['100000XXXXXXXXXX','100000XXXXXXXXXX'],}
 686 clients =[]
 687 socks =[]
 688 socks_control =[]
 689 vmDPIP =[]
 690 vmAdminIP =[]
 691 vmDPmac =[]
 692 hexDPIP =[]
 693 config_file =[]
 694 prox_socket =[]
 695 prox_launch_exit =[]
 696 auto_start =[]
 697 mach_type =[]
 698 sock_type =[]
 699
 700 data_file = 'RUN{}.{}.csv'.format(env,test_file)
 701 data_csv_file = open(data_file,'w')
 702 testconfig = ConfigParser.RawConfigParser()
 703 testconfig.read(test_file)
 704 required_number_of_test_machines = testconfig.get('DEFAULT', 'total_number_of_test_machines')
 705 config = ConfigParser.RawConfigParser()
 706 config.read(env)
 707 machine_map = ConfigParser.RawConfigParser()
 708 machine_map.read(machine_map_file)
 709 key = config.get('ssh', 'key')
 710 total_number_of_machines = config.get('rapid', 'total_number_of_machines')
 711 if int(required_number_of_test_machines) > int(total_number_of_machines):
 712         log.exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
 713         raise Exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
 714 for vm in range(1, int(total_number_of_machines)+1):
 715         vmAdminIP.append(config.get('M%d'%vm, 'admin_ip'))
 716         vmDPmac.append(config.get('M%d'%vm, 'dp_mac'))
 717         vmDPIP.append(config.get('M%d'%vm, 'dp_ip'))
 718         ip = vmDPIP[-1].split('.')
 719         hexDPIP.append(hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2))
 720 machine_index = []
 721 for vm in range(1, int(required_number_of_test_machines)+1):
 722         machine_index.append(int(machine_map.get('TestM%d'%vm, 'machine_index'))-1)
 723         prox_socket.append(testconfig.getboolean('TestM%d'%vm, 'prox_socket'))
 724 for vm in range(1, int(required_number_of_test_machines)+1):
 725         if prox_socket[vm-1]:
 726                 prox_launch_exit.append(testconfig.getboolean('TestM%d'%vm, 'prox_launch_exit'))
 727                 config_file.append(testconfig.get('TestM%d'%vm, 'config_file'))
 728                 with open('{}_{}_parameters{}.lua'.format(env,test_file,vm), "w") as f:
 729                         f.write('name="%s"\n'% testconfig.get('TestM%d'%vm, 'name'))
 730                         f.write('local_ip="%s"\n'% vmDPIP[machine_index[vm-1]])
 731                         f.write('local_hex_ip="%s"\n'% hexDPIP[machine_index[vm-1]])
 732                         if re.match('(l2){0,1}gen(_bare){0,1}\.cfg',config_file[-1]):
 733                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
 734                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
 735                                 genstatcores = gencores + latcores
 736                                 auto_start.append(False)
 737                                 mach_type.append('gen')
 738                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
 739                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
 740                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 741                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 742                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 743                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 744                         elif re.match('(l2){0,1}gen_gw\.cfg',config_file[-1]):
 745                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
 746                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
 747                                 genstatcores = gencores + latcores
 748                                 auto_start.append(False)
 749                                 mach_type.append('gen')
 750                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
 751                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
 752                                 gwVMindex = int(testconfig.get('TestM%d'%vm, 'gw_vm')) -1
 753                                 f.write('gw_ip="%s"\n'% vmDPIP[machine_index[gwVMindex]])
 754                                 f.write('gw_hex_ip="%s"\n'% hexDPIP[machine_index[gwVMindex]])
 755                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 756                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 757                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 758                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 759                         elif  re.match('(l2){0,1}swap.*\.cfg',config_file[-1]):
 760                                 sutstatcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'swapcores'))
 761                                 auto_start.append(True)
 762                                 mach_type.append('sut')
 763                                 f.write('swapcores="%s"\n'% ','.join(map(str, sutstatcores)))
 764                         elif config_file[-1] == 'secgw1.cfg':
 765                                 auto_start.append(True)
 766                                 mach_type.append('none')
 767                                 f.write('secgwcores="%s"\n'% ','.join(map(str, ast.literal_eval(testconfig.get('TestM%d'%vm, 'secgwcores')))))
 768                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 769                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 770                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 771                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 772                         elif config_file[-1] == 'secgw2.cfg':
 773                                 sutstatcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'secgwcores'))
 774                                 auto_start.append(True)
 775                                 mach_type.append('sut')
 776                                 f.write('secgwcores="%s"\n'% ','.join(map(str, sutstatcores)))
 777                         elif config_file[-1] == 'impair.cfg':
 778                                 auto_start.append(True)
 779                                 mach_type.append('none')
 780                                 f.write('impaircores="%s"\n'% ','.join(map(str, ast.literal_eval(testconfig.get('TestM%d'%vm, 'impaircores')))))
 781                         elif config_file[-1] == 'irq.cfg':
 782                                 irqcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'irqcores'))
 783                                 auto_start.append(False)
 784                                 mach_type.append('irq')
 785                                 f.write('irqcores="%s"\n'% ','.join(map(str, irqcores)))
 786                 f.close
 787 #####################################################################################
 788 def exit_handler():
 789         log.debug ('exit cleanup')
 790         for index, sock in enumerate(socks):
 791                 if socks_control[index]:
 792                         sock.quit()
 793         for client in clients:
 794                 client.close()
 795         data_csv_file.close
 796         sys.exit(0)
 797
 798 atexit.register(exit_handler)
 799
 800 for vm in range(0, int(required_number_of_test_machines)):
 801         if prox_socket[vm]:
 802                 clients.append(prox_ctrl(vmAdminIP[machine_index[vm]], key+'.pem','root'))
 803                 connect_client(clients[-1])
 804 # Creating script to bind the right network interface to the poll mode driver
 805                 devbindfile = '{}_{}_devbindvm{}.sh'.format(env,test_file, vm+1)
 806                 with open("devbind.sh") as f:
 807                         newText=f.read().replace('MACADDRESS', vmDPmac[machine_index[vm]])
 808                         with open(devbindfile, "w") as f:
 809                                 f.write(newText)
 810                 st = os.stat(devbindfile)
 811                 os.chmod(devbindfile, st.st_mode | stat.S_IEXEC)
 812                 clients[-1].scp_put('./%s'%devbindfile, '/root/devbind.sh')
 813                 cmd = '/root/devbind.sh'
 814                 clients[-1].run_cmd(cmd)
 815                 log.debug("devbind.sh running on VM%d"%(vm+1))
 816                 clients[-1].scp_put('./%s'%config_file[vm], '/root/%s'%config_file[vm])
 817                 clients[-1].scp_put('./{}_{}_parameters{}.lua'.format(env,test_file, vm+1), '/root/parameters.lua')
 818                 if not configonly:
 819                         if prox_launch_exit[vm]:
 820                                 log.debug("Starting PROX on VM%d"%(vm+1))
 821                                 if auto_start[vm]:
 822                                         cmd = '/root/prox/build/prox -t -o cli -f /root/%s'%config_file[vm]
 823                                 else:
 824                                         cmd = '/root/prox/build/prox -e -t -o cli -f /root/%s'%config_file[vm]
 825                                 clients[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1))
 826                         socks_control.append(prox_launch_exit[vm])
 827                         socks.append(connect_socket(clients[-1]))
 828                         sock_type.append(mach_type[vm])
 829 socks.append('none')
 830 socks_control.append(False)
 831
 832 def get_BinarySearchParams() :
 833         global DROP_RATE_TRESHOLD
 834         global LAT_AVG_TRESHOLD
 835         global LAT_MAX_TRESHOLD
 836         global ACCURACY
 837         global STARTSPEED
 838         DROP_RATE_TRESHOLD = float(testconfig.get('BinarySearchParams', 'drop_rate_threshold'))
 839         LAT_AVG_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_avg_threshold'))
 840         LAT_MAX_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_max_threshold'))
 841         ACCURACY = float(testconfig.get('BinarySearchParams', 'accuracy'))
 842         STARTSPEED = float(testconfig.get('BinarySearchParams', 'startspeed'))
 843
 844 if configonly:
 845         sys.exit()
 846 ####################################################
 847 # Run test cases
 848 # Best to run the flow test at the end since otherwise the tests coming after might be influenced by the big number of entries in the switch flow tables
 849 ####################################################
 850 gensock_index = sock_type.index('gen') if 'gen' in sock_type else -1
 851 sutsock_index = sock_type.index('sut') if 'sut' in sock_type else -1
 852 irqsock_index = sock_type.index('irq') if 'irq' in sock_type else -1
 853 number_of_tests = testconfig.get('DEFAULT', 'number_of_tests')
 854 with data_csv_file:
 855         fieldnames = ['flow','size','endspeed','endspeedpps','endpps_req_tx','endpps_tx','endpps_sut_tx_str','endpps_rx','endlat_avg','endlat_max','endabs_dropped','enddrop_rate']
 856         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 857         writer.writeheader()
 858         for test_nr in range(1, int(number_of_tests)+1):
 859                 test=testconfig.get('test%d'%test_nr,'test')
 860                 tasks= ast.literal_eval(testconfig.get('test%d'%test_nr, 'tasks'))
 861                 log.info(test)
 862                 if test == 'flowsizetest':
 863                         get_BinarySearchParams()
 864                         packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
 865                         flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
 866                         run_flow_size_test(socks[gensock_index],socks[sutsock_index])
 867                 elif test == 'fixed_rate':
 868                         packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
 869                         STARTSPEED = float(testconfig.get('test%d'%test_nr, 'speed'))
 870                         run_fixed_rate(socks[gensock_index],socks[sutsock_index])
 871                 elif test == 'measureswap':
 872                         #packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
 873                         run_measure_swap(socks[sutsock_index])
 874                 elif test == 'impairtest':
 875                         get_BinarySearchParams()
 876                         PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
 877                         run_impairtest(socks[gensock_index],socks[sutsock_index])
 878                 elif test == 'irqtest':
 879                         run_irqtest(socks[irqsock_index])
 880                 elif test == 'warmuptest':
 881                         PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
 882                         FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
 883                         WARMUPSPEED = int(testconfig.get('test%d'%test_nr, 'warmupspeed'))
 884                         WARMUPTIME = int(testconfig.get('test%d'%test_nr, 'warmuptime'))
 885                         run_warmuptest(socks[gensock_index])
 886 ####################################################