VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py

   1 #!/usr/bin/python
   2
   3 ##
   4 ## Copyright (c) 2010-2019 Intel Corporation
   5 ##
   6 ## Licensed under the Apache License, Version 2.0 (the "License");
   7 ## you may not use this file except in compliance with the License.
   8 ## You may obtain a copy of the License at
   9 ##
  10 ##     http://www.apache.org/licenses/LICENSE-2.0
  11 ##
  12 ## Unless required by applicable law or agreed to in writing, software
  13 ## distributed under the License is distributed on an "AS IS" BASIS,
  14 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 ## See the License for the specific language governing permissions and
  16 ## limitations under the License.
  17 ##
  18
  19 from __future__ import print_function
  20
  21 import os
  22 import stat
  23 import sys
  24 import time
  25 import subprocess
  26 import getopt
  27 import re
  28 import logging
  29 from logging.handlers import RotatingFileHandler
  30 from logging import handlers
  31 from prox_ctrl import prox_ctrl
  32 import ConfigParser
  33 import ast
  34 import atexit
  35 import csv
  36
  37 version="19.6.30"
  38 env = "rapid.env" #Default string for environment
  39 test_file = "basicrapid.test" #Default string for test
  40 machine_map_file = "machine.map" #Default string for machine map file
  41 loglevel="DEBUG" # sets log level for writing to file
  42 screenloglevel="INFO" # sets log level for writing to screen
  43 runtime=10 # time in seconds for 1 test run
  44 configonly = False # IF True, the system will upload all the necessary config fiels to the VMs, but not start PROX and the actual testing
  45
  46 def usage():
  47         print("usage: runrapid    [--version] [-v]")
  48         print("                   [--env ENVIRONMENT_NAME]")
  49         print("                   [--test TEST_NAME]")
  50         print("                   [--map MACHINE_MAP_FILE]")
  51         print("                   [--runtime TIME_FOR_TEST]")
  52         print("                   [--configonly False|True]")
  53         print("                   [--log DEBUG|INFO|WARNING|ERROR|CRITICAL]")
  54         print("                   [-h] [--help]")
  55         print("")
  56         print("Command-line interface to runrapid")
  57         print("")
  58         print("optional arguments:")
  59         print("  -v,  --version                 Show program's version number and exit")
  60         print("  --env ENVIRONMENT_NAME         Parameters will be read from ENVIRONMENT_NAME. Default is %s."%env)
  61         print("  --test TEST_NAME               Test cases will be read from TEST_NAME. Default is %s."%test_file)
  62         print("  --map MACHINE_MAP_FILE Machine mapping will be read from MACHINE_MAP_FILE. Default is %s."%machine_map_file)
  63         print("  --runtime                      Specify time in seconds for 1 test run")
  64         print("  --configonly                   If this option is specified, only upload all config files to the VMs, do not run the tests")
  65         print("  --log                          Specify logging level for log file output, default is DEBUG")
  66         print("  --screenlog                    Specify logging level for screen output, default is INFO")
  67         print("  -h, --help                     Show help message and exit.")
  68         print("")
  69
  70 try:
  71         opts, args = getopt.getopt(sys.argv[1:], "vh", ["version","help", "env=", "test=", "map=", "runtime=","configonly","log=","screenlog="])
  72 except getopt.GetoptError as err:
  73         print("===========================================")
  74         print(str(err))
  75         print("===========================================")
  76         usage()
  77         sys.exit(2)
  78 if args:
  79         usage()
  80         sys.exit(2)
  81 for opt, arg in opts:
  82         if opt in ["-h", "--help"]:
  83                 usage()
  84                 sys.exit()
  85         if opt in ["-v", "--version"]:
  86                 print("Rapid Automated Performance Indication for Dataplane "+version)
  87                 sys.exit()
  88         if opt in ["--env"]:
  89                 env = arg
  90         if opt in ["--test"]:
  91                 test_file = arg
  92         if opt in ["--map"]:
  93                 machine_map_file = arg
  94         if opt in ["--runtime"]:
  95                 runtime = arg
  96         if opt in ["--configonly"]:
  97                 configonly = True
  98                 print('No actual runs, only uploading configuration files')
  99         if opt in ["--log"]:
 100                 loglevel = arg
 101                 print ("Log level: "+ loglevel)
 102         if opt in ["--screenlog"]:
 103                 screenloglevel = arg
 104                 print ("Screen Log level: "+ screenloglevel)
 105
 106 print ("Using '"+env+"' as name for the environment")
 107 print ("Using '"+test_file+"' for test case definition")
 108 print ("Using '"+machine_map_file+"' for machine mapping")
 109 print ("Runtime: "+ runtime)
 110
 111 class bcolors:
 112         HEADER = '\033[95m'
 113         OKBLUE = '\033[94m'
 114         OKGREEN = '\033[92m'
 115         WARNING = '\033[93m'
 116         FAIL = '\033[91m'
 117         ENDC = '\033[0m'
 118         BOLD = '\033[1m'
 119         UNDERLINE = '\033[4m'
 120
 121 # create formatters
 122 screen_formatter = logging.Formatter("%(message)s")
 123 file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 124
 125 # get a top-level logger,
 126 # set its log level,
 127 # BUT PREVENT IT from propagating messages to the root logger
 128 #
 129 log = logging.getLogger()
 130 numeric_level = getattr(logging, loglevel.upper(), None)
 131 if not isinstance(numeric_level, int):
 132         raise ValueError('Invalid log level: %s' % loglevel)
 133 log.setLevel(numeric_level)
 134 log.propagate = 0
 135
 136 # create a console handler
 137 # and set its log level to the command-line option
 138 #
 139 console_handler = logging.StreamHandler(sys.stdout)
 140 #console_handler.setLevel(logging.INFO)
 141 numeric_screenlevel = getattr(logging, screenloglevel.upper(), None)
 142 if not isinstance(numeric_screenlevel, int):
 143         raise ValueError('Invalid screenlog level: %s' % screenloglevel)
 144 console_handler.setLevel(numeric_screenlevel)
 145 console_handler.setFormatter(screen_formatter)
 146
 147 # create a file handler
 148 # and set its log level
 149 #
 150 log_file = 'RUN{}.{}.log'.format(env,test_file)
 151 file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10)
 152 #file_handler = log.handlers.TimedRotatingFileHandler(log_file, 'D', 1, 5)
 153 file_handler.setLevel(numeric_level)
 154 file_handler.setFormatter(file_formatter)
 155
 156 # add handlers to the logger
 157 #
 158 log.addHandler(file_handler)
 159 log.addHandler(console_handler)
 160
 161 # Check if log exists and should therefore be rolled
 162 needRoll = os.path.isfile(log_file)
 163
 164
 165 # This is a stale log, so roll it
 166 if needRoll:
 167         # Add timestamp
 168         log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime())
 169
 170         # Roll over on application start
 171         log.handlers[0].doRollover()
 172
 173 # Add timestamp
 174 log.debug('\n---------\nLog started on %s.\n---------\n' % time.asctime())
 175
 176 log.debug("runrapid.py version: "+version)
 177 #========================================================================
 178 def connect_socket(client):
 179         attempts = 1
 180         log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 181         sock = None
 182         while True:
 183                 sock = client.prox_sock()
 184                 if sock is not None:
 185                         break
 186                 attempts += 1
 187                 if attempts > 20:
 188                         log.exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 189                         raise Exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 190                 time.sleep(2)
 191                 log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 192         log.info("Connected to PROX on %s" % client.ip())
 193         return sock
 194
 195 def connect_client(client):
 196         attempts = 1
 197         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 198         while True:
 199                 try:
 200                         client.connect()
 201                         break
 202                 except RuntimeWarning, ex:
 203                         attempts += 1
 204                         if attempts > 20:
 205                                 log.exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 206                                 raise Exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 207                         time.sleep(2)
 208                         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 209         log.debug("Connected to VM on %s" % client.ip())
 210
 211 def run_iteration(gensock,sutsock):
 212         sleep_time = 2
 213         # Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
 214         time.sleep(sleep_time)
 215         abs_old_rx, abs_old_non_dp_rx, abs_old_tx, abs_old_non_dp_tx, abs_old_drop, abs_old_tx_fail, abs_old_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,tasks)
 216         abs_old_rx = abs_old_rx - abs_old_non_dp_rx
 217         abs_old_tx = abs_old_tx - abs_old_non_dp_tx
 218         gensock.start(gencores)
 219         time.sleep(sleep_time)
 220         if sutsock!='none':
 221                 old_sut_rx, old_sut_non_dp_rx, old_sut_tx, old_sut_non_dp_tx, old_sut_drop, old_sut_tx_fail, old_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 222                 old_sut_rx = old_sut_rx - old_sut_non_dp_rx
 223                 old_sut_tx = old_sut_tx - old_sut_non_dp_tx
 224         old_rx, old_non_dp_rx, old_tx, old_non_dp_tx, old_drop, old_tx_fail, old_tsc, tsc_hz = gensock.core_stats(genstatcores,tasks)
 225         old_rx = old_rx - old_non_dp_rx
 226         old_tx = old_tx - old_non_dp_tx
 227         # Measure latency statistics per second
 228         n_loops = 0
 229         lat_min = 0
 230         lat_max = 0
 231         lat_avg = 0
 232         used_avg = 0
 233         while n_loops < float(runtime):
 234                 n_loops +=1
 235                 time.sleep(1)
 236                 lat_min_sample, lat_max_sample, lat_avg_sample, used_sample = gensock.lat_stats(latcores)
 237                 if lat_min > lat_min_sample:
 238                         lat_min = lat_min_sample
 239                 if lat_max < lat_max_sample:
 240                         lat_max = lat_max_sample
 241                 lat_avg = lat_avg + lat_avg_sample
 242                 used_avg = used_avg + used_sample
 243         lat_avg = lat_avg / n_loops
 244         used_avg = used_avg / n_loops
 245         # Get statistics after some execution time
 246         new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = gensock.core_stats(genstatcores,tasks)
 247         new_rx = new_rx - new_non_dp_rx
 248         new_tx = new_tx - new_non_dp_tx
 249         if sutsock!='none':
 250                 new_sut_rx, new_sut_non_dp_rx, new_sut_tx, new_sut_non_dp_tx, new_sut_drop, new_sut_tx_fail, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 251                 new_sut_rx = new_sut_rx - new_sut_non_dp_rx
 252                 new_sut_tx = new_sut_tx - new_sut_non_dp_tx
 253         #Stop generating
 254         gensock.stop(gencores)
 255         time.sleep(sleep_time)
 256         abs_new_rx, abs_new_non_dp_rx, abs_new_tx, abs_new_non_dp_tx, abs_new_drop, abs_new_tx_fail, abs_new_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,tasks)
 257         abs_new_rx = abs_new_rx - abs_new_non_dp_rx
 258         abs_new_tx = abs_new_tx - abs_new_non_dp_tx
 259         drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 260         rx = new_rx - old_rx     # rx is all packets received by the nop task = all packets received in the gen VM
 261         tx = new_tx - old_tx     # tx is all generated packets actually accepted by the interface
 262         abs_dropped = (abs_new_tx - abs_old_tx) - (abs_new_rx - abs_old_rx)
 263         tsc = new_tsc - old_tsc  # time difference between the 2 measurements, expressed in cycles.
 264         pps_req_tx = (tx+drop-rx)*tsc_hz*1.0/(tsc*1000000)
 265         pps_tx = tx*tsc_hz*1.0/(tsc*1000000)
 266         pps_rx = rx*tsc_hz*1.0/(tsc*1000000)
 267         if sutsock!='none':
 268                 sut_rx = new_sut_rx - old_sut_rx
 269                 sut_tx = new_sut_tx - old_sut_tx
 270                 sut_tsc = new_sut_tsc - old_sut_tsc
 271                 pps_sut_tx = sut_tx*sut_tsc_hz*1.0/(sut_tsc*1000000)
 272                 pps_sut_tx_str = '{:>9.3f}'.format(pps_sut_tx)
 273         else:
 274                 pps_sut_tx = 0
 275                 pps_sut_tx_str = 'NO MEAS.'
 276         if (tx == 0):
 277                 log.critical("TX = 0. Test interrupted since no packet has been sent.")
 278                 raise Exception("TX = 0")
 279         return(pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,(abs_new_tx_fail - abs_old_tx_fail),(abs_new_tx - abs_old_tx),lat_min,used_avg)
 280
 281 def new_speed(speed,minspeed,maxspeed,success):
 282         if success:
 283                 minspeed = speed
 284         else:
 285                 maxspeed = speed
 286         newspeed = (maxspeed+minspeed)/2.0
 287         return (newspeed,minspeed,maxspeed)
 288
 289 def get_pps(speed,size):
 290         # speed is given in % of 10Gb/s, returning Mpps
 291         return (speed * 100.0 / (8*(size+24)))
 292
 293 def get_speed(packet_speed,size):
 294         # return speed in Gb/s
 295         return (packet_speed / 1000.0 * (8*(size+24)))
 296
 297
 298 def run_flow_size_test(gensock,sutsock):
 299         gensock.start(latcores)
 300         for size in packet_size_list:
 301                 size = size-4
 302                 gensock.set_size(gencores,0,size) # This is setting the frame size
 303                 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 304                 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 305                 # This will only work when using sending UDP packets. For different protocls and ehternet types, we would need a different calculation
 306                 log.info("+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 307                 log.info("| UDP, "+ '{:>5}'.format(size+4) +" bytes, different number of flows by randomizing SRC & DST UDP port                                                                                           |")
 308                 log.info("+--------+--------------------+----------------+----------------+----------------+------------------------+----------------+----------------+----------------+------------+")
 309                 log.info("| Flows  |  Speed requested   | core generated | Sent by Gen NIC| Forward by SUT |      core received     |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
 310                 log.info("+--------+--------------------+----------------+----------------+----------------+------------------------+----------------+----------------+----------------+------------+")
 311                 for flow_number in flow_size_list:
 312                         attempts = 0
 313                         gensock.reset_stats()
 314                         if sutsock!='none':
 315                                 sutsock.reset_stats()
 316                         source_port,destination_port = flows[flow_number]
 317                         gensock.set_random(gencores,0,34,source_port,2)
 318                         gensock.set_random(gencores,0,36,destination_port,2)
 319                         endpps_sut_tx_str = 'NO_RESULTS'
 320                         maxspeed = speed = STARTSPEED
 321                         minspeed = 0
 322                         while (maxspeed-minspeed > ACCURACY):
 323                                 attempts += 1
 324                                 endwarning =''
 325                                 print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 326                                 sys.stdout.flush()
 327                                 # Start generating packets at requested speed (in % of a 10Gb/s link)
 328                                 gensock.speed(speed / len(gencores), gencores)
 329                                 time.sleep(1)
 330                                 # Get statistics now that the generation is stable and initial ARP messages are dealt with
 331                                 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used = run_iteration(gensock,sutsock)
 332                                 drop_rate = 100.0*abs_dropped/abs_tx
 333                                 if lat_used < 0.95:
 334                                         lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 335                                 else:
 336                                         lat_warning = ''
 337                                 # The following if statement is testing if we pass the success criteria of a certain drop rate, average latenecy and maximum latency below the threshold
 338                                 # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
 339                                 # This can be specified by putting 0 in the .test file
 340                                 if ((drop_rate < DROP_RATE_TRESHOLD) or (abs_dropped==DROP_RATE_TRESHOLD ==0)) and (lat_avg< LAT_AVG_TRESHOLD) and (lat_max < LAT_MAX_TRESHOLD):
 341                                         lat_avg_prefix = bcolors.ENDC
 342                                         lat_max_prefix = bcolors.ENDC
 343                                         abs_drop_rate_prefix = bcolors.ENDC
 344                                         drop_rate_prefix = bcolors.ENDC
 345                                         if ((get_pps(speed,size) - pps_tx)/get_pps(speed,size))>0.01:
 346                                                 speed_prefix = bcolors.WARNING
 347                                                 if abs_tx_fail > 0:
 348                                                         gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
 349                                                 else:
 350                                                         gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(get_pps(speed,size), pps_tx) + bcolors.ENDC
 351                                         else:
 352                                                 speed_prefix = bcolors.ENDC
 353                                                 gen_warning = ''
 354                                         endspeed = speed
 355                                         endpps_req_tx = pps_req_tx
 356                                         endpps_tx = pps_tx
 357                                         endpps_sut_tx_str = pps_sut_tx_str
 358                                         endpps_rx = pps_rx
 359                                         endlat_avg = lat_avg
 360                                         endlat_max = lat_max
 361                                         endabs_dropped = abs_dropped
 362                                         enddrop_rate = drop_rate
 363                                         if lat_warning or gen_warning:
 364                                                 endwarning = '|        | {:167.167} |'.format(lat_warning + gen_warning)
 365                                         success = True
 366                                         success_message='%  | SUCCESS'
 367                                 else:
 368                                         success_message='%  | FAILED'
 369                                         gen_warning = ''
 370                                         abs_drop_rate_prefix = bcolors.ENDC
 371                                         if ((abs_dropped>0) and (DROP_RATE_TRESHOLD ==0)):
 372                                                 abs_drop_rate_prefix = bcolors.FAIL
 373                                         if (drop_rate < DROP_RATE_TRESHOLD):
 374                                                 drop_rate_prefix = bcolors.ENDC
 375                                         else:
 376                                                 drop_rate_prefix = bcolors.FAIL
 377                                         if (lat_avg< LAT_AVG_TRESHOLD):
 378                                                 lat_avg_prefix = bcolors.ENDC
 379                                         else:
 380                                                 lat_avg_prefix = bcolors.FAIL
 381                                         if (lat_max< LAT_MAX_TRESHOLD):
 382                                                 lat_max_prefix = bcolors.ENDC
 383                                         else:
 384                                                 lat_max_prefix = bcolors.FAIL
 385                                         if (((get_pps(speed,size) - pps_tx)/get_pps(speed,size))<0.001):
 386                                                 speed_prefix = bcolors.ENDC
 387                                         else:
 388                                                 speed_prefix = bcolors.FAIL
 389                                         success = False
 390                                 log.debug('|step{:>3}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% '+speed_prefix +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | ' + '{:>9.3f}'.format(pps_tx) +' Mpps | '+ bcolors.ENDC  + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+bcolors.OKBLUE + '{:>4.1f}'.format(get_speed(pps_rx,size)) + 'Gb/s{:>9.3f}'.format(pps_rx)+' Mpps'+bcolors.ENDC+' | '+lat_avg_prefix+ '{:>9.0f}'.format(lat_avg)+' us   | '+lat_max_prefix+ '{:>9.0f}'.format(lat_max)+' us   | '+ abs_drop_rate_prefix + '{:>14d}'.format(abs_dropped)+drop_rate_prefix+ ' |''{:>9.2f}'.format(drop_rate)+bcolors.ENDC+ success_message +lat_warning + gen_warning)
 391                                 speed,minspeed,maxspeed = new_speed(speed,minspeed,maxspeed,success)
 392                         if endpps_sut_tx_str !=  'NO_RESULTS':
 393                                 log.info('|{:>7}'.format(str(flow_number))+" | " + '{:>5.1f}'.format(endspeed) + '% ' + speed_prefix + '{:>6.3f}'.format(get_pps(endspeed,size)) + ' Mpps | '+ '{:>9.3f}'.format(endpps_req_tx)+ ' Mpps | '+ bcolors.ENDC + '{:>9.3f}'.format(endpps_tx) +' Mpps | ' + '{:>9}'.format(endpps_sut_tx_str) +' Mpps | '+bcolors.OKBLUE + '{:>4.1f}'.format(get_speed(pps_rx,size)) + 'Gb/s{:>9.3f}'.format(endpps_rx)+' Mpps'+bcolors.ENDC+' | '+ '{:>9.0f}'.format(endlat_avg)+' us   | '+ '{:>9.0f}'.format(endlat_max)+' us   | '+ '{:>14d}'.format(endabs_dropped)+ ' |'+'{:>9.2f}'.format(enddrop_rate)+ '%  |')
 394                                 if endwarning:
 395                                         log.info (endwarning)
 396                                 log.info("+--------+--------------------+----------------+----------------+----------------+------------------------+----------------+----------------+----------------+------------+")
 397                                 writer.writerow({'flow':flow_number,'size':(size+4),'endspeed':endspeed,'endspeedpps':get_pps(endspeed,size),'endpps_req_tx':endpps_req_tx,'endpps_tx':endpps_tx,'endpps_sut_tx_str':endpps_sut_tx_str,'endpps_rx':endpps_rx,'endlat_avg':endlat_avg,'endlat_max':endlat_max,'endabs_dropped':endabs_dropped,'enddrop_rate':enddrop_rate})
 398                         else:
 399                                 log.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
 400         gensock.stop(latcores)
 401
 402
 403 def run_fixed_rate(gensock,sutsock):
 404         log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------+")
 405         log.info("| UDP, 1 flow, different packet sizes                                                                                                                       |")
 406         log.info("+-----+------------------+-------------+-------------+-------------+-------------+-------------+-------------+-----------+-----------+---------+------------+")
 407         log.info("|Pktsz| Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core| Avg. Latency| Max. Latency|   Sent    |  Received |  Lost   | Total Lost |")
 408         log.info("+-----+------------------+-------------+-------------+-------------+-------------+-------------+-------------+-----------+-----------+---------+------------+")
 409         sleep_time = 3
 410         gensock.start(latcores)
 411         for size in packet_size_list:
 412                 # Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
 413                 time.sleep(sleep_time)
 414                 size = size-4
 415                 gensock.reset_stats()
 416                 if sutsock!='none':
 417                         sutsock.reset_stats()
 418                 gensock.set_size(gencores,0,size) # This is setting the frame size
 419                 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 420                 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 421                 # This will only work when using sending UDP packets. For different protocls and ehternet types, we would need a differnt calculation
 422                 pps_sut_tx_str = 'NO_RESULTS'
 423                 speed = STARTSPEED
 424                 # Start generating packets at requested speed (in % of a 10Gb/s link)
 425                 gensock.speed(speed / len(gencores), gencores)
 426                 duration = float(runtime)
 427                 first = 1
 428                 tot_drop = 0
 429                 if sutsock!='none':
 430                         old_sut_rx, old_sut_non_dp_rx, old_sut_tx, old_sut_non_dp_tx, old_sut_drop, old_sut_tx_fail, old_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 431                         old_sut_rx = old_sut_rx - old_sut_non_dp_rx
 432                         old_sut_tx = old_sut_tx - old_sut_non_dp_tx
 433                 old_rx, old_non_dp_rx, old_tx, old_non_dp_tx, old_drop, old_tx_fail, old_tsc, tsc_hz = gensock.core_stats(genstatcores,tasks)
 434                 old_rx = old_rx - old_non_dp_rx
 435                 old_tx = old_tx - old_non_dp_tx
 436                 gensock.start(gencores)
 437                 while (duration > 0):
 438                         time.sleep(0.5)
 439                         lat_min, lat_max, lat_avg, lat_used = gensock.lat_stats(latcores)
 440                         if lat_used < 0.95:
 441                                 lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 442                         else:
 443                                 lat_warning = ''
 444                         # Get statistics after some execution time
 445                         new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = gensock.core_stats(genstatcores,tasks)
 446                         new_rx = new_rx - new_non_dp_rx
 447                         new_tx = new_tx - new_non_dp_tx
 448                         if sutsock!='none':
 449                                 new_sut_rx, new_sut_non_dp_rx, new_sut_tx, new_sut_non_dp_tx, new_sut_drop, new_sut_tx_fail, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 450                                 new_sut_rx = new_sut_rx - new_sut_non_dp_rx
 451                                 new_sut_tx = new_sut_tx - new_sut_non_dp_tx
 452                                 drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 453                                 rx = new_rx - old_rx     # rx is all packets received by the nop task = all packets received in the gen VM
 454                                 tx = new_tx - old_tx     # tx is all generated packets actually accepted by the interface
 455                                 tsc = new_tsc - old_tsc  # time difference between the 2 measurements, expressed in cycles.
 456                         if tsc == 0 :
 457                                 continue
 458                         if sutsock!='none':
 459                                 sut_rx = new_sut_rx - old_sut_rx
 460                                 sut_tx = new_sut_tx - old_sut_tx
 461                                 sut_tsc = new_sut_tsc - old_sut_tsc
 462                                 if sut_tsc == 0 :
 463                                         continue
 464                         duration = duration - 1
 465                         old_drop = new_drop
 466                         old_rx = new_rx
 467                         old_tx = new_tx
 468                         old_tsc = new_tsc
 469                         pps_req_tx = (tx+drop-rx)*tsc_hz*1.0/(tsc*1000000)
 470                         pps_tx = tx*tsc_hz*1.0/(tsc*1000000)
 471                         pps_rx = rx*tsc_hz*1.0/(tsc*1000000)
 472                         if sutsock!='none':
 473                                 old_sut_tx = new_sut_tx
 474                                 old_sut_rx = new_sut_rx
 475                                 old_sut_tsc= new_sut_tsc
 476                                 pps_sut_tx = sut_tx*sut_tsc_hz*1.0/(sut_tsc*1000000)
 477                                 pps_sut_tx_str = '{:>7.3f}'.format(pps_sut_tx)
 478                         else:
 479                                 pps_sut_tx = 0
 480                                 pps_sut_tx_str = 'NO MEAS.'
 481                         if (tx == 0):
 482                                 log.critical("TX = 0. Test interrupted since no packet has been sent.")
 483                                 raise Exception("TX = 0")
 484                         tot_drop = tot_drop + tx - rx
 485
 486                         if pps_sut_tx_str !=  'NO_RESULTS':
 487                                 # First second mpps are not valid as there is no alignement between time the generator is started and per seconds stats
 488                                 if (first):
 489                                         log.info('|{:>4}'.format(size+4)+" |" + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+'             |' +'             |'  +'             |'+ '             |'+ '{:>8.0f}'.format(lat_avg)+' us  |'+'{:>8.0f}'.format(lat_max)+' us  | ' + '{:>9.0f}'.format(tx) + ' | '+ '{:>9.0f}'.format(rx) + ' | '+ '{:>7.0f}'.format(tx-rx) + ' | '+'{:>7.0f}'.format(tot_drop) +'    |'+lat_warning)
 490                                 else:
 491                                         log.info('|{:>4}'.format(size+4)+" |" + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+ '{:>7.3f}'.format(pps_req_tx)+' Mpps |'+ '{:>7.3f}'.format(pps_tx) +' Mpps |' + '{:>7}'.format(pps_sut_tx_str) +' Mpps |'+ '{:>7.3f}'.format(pps_rx)+' Mpps |'+ '{:>8.0f}'.format(lat_avg)+' us  |'+'{:>8.0f}'.format(lat_max)+' us  | ' + '{:>9.0f}'.format(tx) + ' | '+ '{:>9.0f}'.format(rx) + ' | '+ '{:>7.0f}'.format(tx-rx) + ' | '+ '{:>7.0f}'.format(tot_drop) +'    |'+lat_warning)
 492                         else:
 493                                 log.debug('|{:>7}'.format(str(size))+" | Speed 0 or close to 0")
 494                         first = 0
 495                         if (duration <= 0):
 496                                 #Stop generating
 497                                 gensock.stop(gencores)
 498                                 time.sleep(sleep_time)
 499                                 lat_min, lat_max, lat_avg, lat_used = gensock.lat_stats(latcores)
 500                                 if lat_used < 0.95:
 501                                         lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 502                                 else:
 503                                         lat_warning = ''
 504                                 # Get statistics after some execution time
 505                                 new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = gensock.core_stats(genstatcores,tasks)
 506                                 new_rx = new_rx - new_non_dp_rx
 507                                 new_tx = new_tx - new_non_dp_tx
 508                                 if sutsock!='none':
 509                                         new_sut_rx, new_sut_non_dp_rx, new_sut_tx, new_sut_non_dp_tx, new_sut_drop, new_sut_tx_fail, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 510                                         new_sut_rx = new_sut_rx - new_sut_non_dp_rx
 511                                         new_sut_tx = new_sut_tx - new_sut_non_dp_tx
 512                                 drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 513                                 rx = new_rx - old_rx     # rx is all packets received by the nop task = all packets received in the gen VM
 514                                 tx = new_tx - old_tx     # tx is all generated packets actually accepted by the interface
 515                                 tsc = new_tsc - old_tsc  # time difference between the 2 measurements, expressed in cycles.
 516                                 tot_drop = tot_drop + tx - rx
 517                                 if sutsock!='none':
 518                                         sut_rx = new_sut_rx - old_sut_rx
 519                                         sut_tx = new_sut_tx - old_sut_tx
 520                                         sut_tsc = new_sut_tsc - old_sut_tsc
 521                                 if pps_sut_tx_str !=  'NO_RESULTS':
 522                                         log.info('|{:>4}'.format(size+4)+" |" + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+'             |' +'             |'  +'             |'+ '             |'+ '{:>8.0f}'.format(lat_avg)+' us  |'+'{:>8.0f}'.format(lat_max)+' us  | ' + '{:>9.0f}'.format(tx) + ' | '+ '{:>9.0f}'.format(rx) + ' | '+ '{:>7.0f}'.format(tx-rx) + ' | '+ '{:>7.0f}'.format(tot_drop) +'    |'+lat_warning)
 523                 log.info("+-----+------------------+-------------+-------------+-------------+-------------+-------------+-------------+-----------+-----------+---------+------------+")
 524         gensock.stop(latcores)
 525
 526 def run_measure_swap(sutsock):
 527         log.info("+------------------------------------------------------------------------------------------------------+")
 528         log.info("| Measuring packets on SWAP system                                                                     |")
 529         log.info("+-----------+------------+------------+------------+------------+------------+------------+------------+")
 530         log.info("|    Time   |    RX      |     TX     | non DP RX  | non DP TX  |   TX - RX  | nonDP TX-RX|  DROP TOT  |")
 531         log.info("+-----------+------------+------------+------------+------------+------------+------------+------------+")
 532         sutsock.reset_stats()
 533         duration = float(runtime)
 534         first = 1
 535         tot_drop = 0
 536         old_rx, old_non_dp_rx, old_tx, old_non_dp_tx, old_drop, old_tx_fail, old_tsc, tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 537         while (duration > 0):
 538                 time.sleep(0.5)
 539                 # Get statistics after some execution time
 540                 new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 541                 drop = new_drop-old_drop
 542                 rx = new_rx - old_rx
 543                 tx = new_tx - old_tx
 544                 non_dp_rx = new_non_dp_rx - old_non_dp_rx
 545                 non_dp_tx = new_non_dp_tx - old_non_dp_tx
 546                 tsc = new_tsc - old_tsc
 547                 if tsc == 0 :
 548                         continue
 549                 duration = duration - 1
 550                 old_drop = new_drop
 551                 old_rx = new_rx
 552                 old_tx = new_tx
 553                 old_non_dp_rx = new_non_dp_rx
 554                 old_non_dp_tx = new_non_dp_tx
 555                 old_tsc = new_tsc
 556                 tot_drop = tot_drop + tx - rx
 557
 558                 log.info('|{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(non_dp_rx)+' | '+'{:>10.0f}'.format(non_dp_tx)+' | ' + '{:>10.0f}'.format(tx-rx) + ' | '+ '{:>10.0f}'.format(non_dp_tx-non_dp_rx) + ' | '+'{:>10.0f}'.format(tot_drop) +' |')
 559         log.info("+------------------------------------------------------------------------------------------------------+")
 560
 561
 562 def run_irqtest(sock):
 563         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 564         log.info("| Measuring time probably spent dealing with an interrupt. Interrupting DPDK cores for more than 50us might be problematic   ")
 565         log.info("| and result in packet loss. The first row shows the interrupted time buckets: first number is the bucket between 0us and    ")
 566         log.info("| that number expressed in us and so on. The numbers in the other rows show how many times per second, the program was       ")
 567         log.info("| interrupted for a time as specified by its bucket. '0' is printed when there are no interrupts in this bucket throughout   ")
 568         log.info("| the duration of the test. This is to avoid rounding errors in the case of 0.0                                              ")
 569         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 570         sys.stdout.flush()
 571         buckets=sock.show_irq_buckets(1)
 572         print('Measurement ongoing ... ',end='\r')
 573         sock.stop(irqcores)
 574         old_irq = [[0 for x in range(len(buckets)+1)] for y in range(len(irqcores)+1)]
 575         irq = [[0 for x in range(len(buckets)+1)] for y in range(len(irqcores)+1)]
 576         irq[0][0] = 'bucket us'
 577         for j,bucket in enumerate(buckets,start=1):
 578                 irq[0][j] = '<'+ bucket
 579         irq[0][-1] = '>'+ buckets [-2]
 580         sock.start(irqcores)
 581         time.sleep(2)
 582         for j,bucket in enumerate(buckets,start=1):
 583                 for i,irqcore in enumerate(irqcores,start=1):
 584                         old_irq[i][j] = sock.irq_stats(irqcore,j-1)
 585         time.sleep(float(runtime))
 586         sock.stop(irqcores)
 587         for i,irqcore in enumerate(irqcores,start=1):
 588                 irq[i][0]='core %s '%irqcore
 589                 for j,bucket in enumerate(buckets,start=1):
 590                         diff =  sock.irq_stats(irqcore,j-1) - old_irq[i][j]
 591                         if diff == 0:
 592                                 irq[i][j] = '0'
 593                         else:
 594                                 irq[i][j] = str(round(diff/float(runtime), 2))
 595         for row in irq:
 596                 log.info(''.join(['{:>12}'.format(item) for item in row]))
 597
 598 def run_impairtest(gensock,sutsock):
 599         size=PACKETSIZE-4
 600         log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 601         log.info("| Generator is sending UDP (1 flow) packets ("+ '{:>5}'.format(size+4) +" bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test        |")
 602         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 603         log.info("| Test   |  Speed requested   | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
 604         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 605         attempts = 0
 606         gensock.set_size(gencores,0,size) # This is setting the frame size
 607         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 608         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 609         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 610         gensock.start(latcores)
 611         speed = STARTSPEED
 612         gensock.speed(speed / len(gencores), gencores)
 613         while True:
 614                 attempts += 1
 615                 print('Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 616                 sys.stdout.flush()
 617                 time.sleep(1)
 618                 # Get statistics now that the generation is stable and NO ARP messages any more
 619                 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used = run_iteration(gensock,sutsock)
 620                 drop_rate = 100.0*abs_dropped/abs_tx
 621                 if lat_used < 0.95:
 622                         lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 623                 else:
 624                         lat_warning = ''
 625                 log.info('|{:>7}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | '+ '{:>9.3f}'.format(pps_tx) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9.3f}'.format(pps_rx)+' Mpps | '+ '{:>9.0f}'.format(lat_avg)+' us   | '+ '{:>9.0f}'.format(lat_max)+' us   | '+ '{:>14d}'.format(abs_dropped)+ ' |''{:>9.2f}'.format(drop_rate)+ '%  |'+lat_warning)
 626                 writer.writerow({'flow':'1','size':(size+4),'endspeed':speed,'endspeedpps':get_pps(speed,size),'endpps_req_tx':pps_req_tx,'endpps_tx':pps_tx,'endpps_sut_tx_str':pps_sut_tx_str,'endpps_rx':pps_rx,'endlat_avg':lat_avg,'endlat_max':lat_max,'endabs_dropped':abs_dropped,'enddrop_rate':drop_rate})
 627
 628 def run_warmuptest(gensock):
 629 # Running at low speed to make sure the ARP messages can get through.
 630 # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
 631 # Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
 632 # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
 633         gensock.speed(WARMUPSPEED / len(gencores), gencores)
 634         size=PACKETSIZE-4
 635         gensock.set_size(gencores,0,size) # This is setting the frame size
 636         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 637         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 638         gensock.set_value(gencores,0,56,1,1)
 639         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 640         source_port,destination_port = flows[FLOWSIZE]
 641         gensock.set_random(gencores,0,34,source_port,2)
 642         gensock.set_random(gencores,0,36,destination_port,2)
 643         gensock.start(genstatcores)
 644         time.sleep(WARMUPTIME)
 645         gensock.stop(genstatcores)
 646         gensock.set_value(gencores,0,56,50,1)
 647
 648 global sutstatcores
 649 global genstatcores
 650 global latcores
 651 global gencores
 652 global irqcores
 653 global PACKETSIZE
 654 global packet_size_list
 655 global FLOWSIZE
 656 global flow_size_list
 657 global WARMUPTIME
 658 global WARMUPSPEED
 659 global required_number_of_test_machines
 660 # To generate a desired number of flows, PROX will randomize the bits in source and destination ports, as specified by the bit masks in the flows variable.
 661 flows={\
 662 1:      ['1000000000000000','1000000000000000'],\
 663 2:      ['1000000000000000','100000000000000X'],\
 664 4:      ['100000000000000X','100000000000000X'],\
 665 8:      ['100000000000000X','10000000000000XX'],\
 666 16:     ['10000000000000XX','10000000000000XX'],\
 667 32:     ['10000000000000XX','1000000000000XXX'],\
 668 64:     ['1000000000000XXX','1000000000000XXX'],\
 669 128:    ['1000000000000XXX','100000000000XXXX'],\
 670 256:    ['100000000000XXXX','100000000000XXXX'],\
 671 512:    ['100000000000XXXX','10000000000XXXXX'],\
 672 1024:   ['10000000000XXXXX','10000000000XXXXX'],\
 673 2048:   ['10000000000XXXXX','1000000000XXXXXX'],\
 674 4096:   ['1000000000XXXXXX','1000000000XXXXXX'],\
 675 8192:   ['1000000000XXXXXX','100000000XXXXXXX'],\
 676 16384:  ['100000000XXXXXXX','100000000XXXXXXX'],\
 677 32768:  ['100000000XXXXXXX','10000000XXXXXXXX'],\
 678 65535:  ['10000000XXXXXXXX','10000000XXXXXXXX'],\
 679 131072: ['10000000XXXXXXXX','1000000XXXXXXXXX'],\
 680 262144: ['1000000XXXXXXXXX','1000000XXXXXXXXX'],\
 681 524280: ['1000000XXXXXXXXX','100000XXXXXXXXXX'],\
 682 1048576:['100000XXXXXXXXXX','100000XXXXXXXXXX'],}
 683 clients =[]
 684 socks =[]
 685 socks_control =[]
 686 vmDPIP =[]
 687 vmAdminIP =[]
 688 vmDPmac =[]
 689 hexDPIP =[]
 690 config_file =[]
 691 prox_socket =[]
 692 prox_launch_exit =[]
 693 auto_start =[]
 694 mach_type =[]
 695 sock_type =[]
 696
 697 data_file = 'RUN{}.{}.csv'.format(env,test_file)
 698 data_csv_file = open(data_file,'w')
 699 testconfig = ConfigParser.RawConfigParser()
 700 testconfig.read(test_file)
 701 required_number_of_test_machines = testconfig.get('DEFAULT', 'total_number_of_test_machines')
 702 config = ConfigParser.RawConfigParser()
 703 config.read(env)
 704 machine_map = ConfigParser.RawConfigParser()
 705 machine_map.read(machine_map_file)
 706 key = config.get('ssh', 'key')
 707 total_number_of_machines = config.get('rapid', 'total_number_of_machines')
 708 if int(required_number_of_test_machines) > int(total_number_of_machines):
 709         log.exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
 710         raise Exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
 711 for vm in range(1, int(total_number_of_machines)+1):
 712         vmAdminIP.append(config.get('M%d'%vm, 'admin_ip'))
 713         vmDPmac.append(config.get('M%d'%vm, 'dp_mac'))
 714         vmDPIP.append(config.get('M%d'%vm, 'dp_ip'))
 715         ip = vmDPIP[-1].split('.')
 716         hexDPIP.append(hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2))
 717 machine_index = []
 718 for vm in range(1, int(required_number_of_test_machines)+1):
 719         machine_index.append(int(machine_map.get('TestM%d'%vm, 'machine_index'))-1)
 720         prox_socket.append(testconfig.getboolean('TestM%d'%vm, 'prox_socket'))
 721 for vm in range(1, int(required_number_of_test_machines)+1):
 722         if prox_socket[vm-1]:
 723                 prox_launch_exit.append(testconfig.getboolean('TestM%d'%vm, 'prox_launch_exit'))
 724                 config_file.append(testconfig.get('TestM%d'%vm, 'config_file'))
 725                 with open('{}_{}_parameters{}.lua'.format(env,test_file,vm), "w") as f:
 726                         f.write('name="%s"\n'% testconfig.get('TestM%d'%vm, 'name'))
 727                         f.write('local_ip="%s"\n'% vmDPIP[machine_index[vm-1]])
 728                         f.write('local_hex_ip="%s"\n'% hexDPIP[machine_index[vm-1]])
 729                         if re.match('(l2){0,1}gen(_bare){0,1}\.cfg',config_file[-1]):
 730                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
 731                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
 732                                 genstatcores = gencores + latcores
 733                                 auto_start.append(False)
 734                                 mach_type.append('gen')
 735                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
 736                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
 737                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 738                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 739                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 740                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 741                         elif re.match('(l2){0,1}gen_gw\.cfg',config_file[-1]):
 742                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
 743                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
 744                                 genstatcores = gencores + latcores
 745                                 auto_start.append(False)
 746                                 mach_type.append('gen')
 747                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
 748                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
 749                                 gwVMindex = int(testconfig.get('TestM%d'%vm, 'gw_vm')) -1
 750                                 f.write('gw_ip="%s"\n'% vmDPIP[machine_index[gwVMindex]])
 751                                 f.write('gw_hex_ip="%s"\n'% hexDPIP[machine_index[gwVMindex]])
 752                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 753                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 754                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 755                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 756                         elif  re.match('(l2){0,1}swap.*\.cfg',config_file[-1]):
 757                                 sutstatcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'swapcores'))
 758                                 auto_start.append(True)
 759                                 mach_type.append('sut')
 760                                 f.write('swapcores="%s"\n'% ','.join(map(str, sutstatcores)))
 761                         elif config_file[-1] == 'secgw1.cfg':
 762                                 auto_start.append(True)
 763                                 mach_type.append('none')
 764                                 f.write('secgwcores="%s"\n'% ','.join(map(str, ast.literal_eval(testconfig.get('TestM%d'%vm, 'secgwcores')))))
 765                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 766                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 767                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 768                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 769                         elif config_file[-1] == 'secgw2.cfg':
 770                                 sutstatcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'secgwcores'))
 771                                 auto_start.append(True)
 772                                 mach_type.append('sut')
 773                                 f.write('secgwcores="%s"\n'% ','.join(map(str, sutstatcores)))
 774                         elif config_file[-1] == 'impair.cfg':
 775                                 auto_start.append(True)
 776                                 mach_type.append('none')
 777                                 f.write('impaircores="%s"\n'% ','.join(map(str, ast.literal_eval(testconfig.get('TestM%d'%vm, 'impaircores')))))
 778                         elif config_file[-1] == 'irq.cfg':
 779                                 irqcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'irqcores'))
 780                                 auto_start.append(False)
 781                                 mach_type.append('irq')
 782                                 f.write('irqcores="%s"\n'% ','.join(map(str, irqcores)))
 783                 f.close
 784 #####################################################################################
 785 def exit_handler():
 786         log.debug ('exit cleanup')
 787         for index, sock in enumerate(socks):
 788                 if socks_control[index]:
 789                         sock.quit()
 790         for client in clients:
 791                 client.close()
 792         data_csv_file.close
 793         sys.exit(0)
 794
 795 atexit.register(exit_handler)
 796
 797 for vm in range(0, int(required_number_of_test_machines)):
 798         if prox_socket[vm]:
 799                 clients.append(prox_ctrl(vmAdminIP[machine_index[vm]], key+'.pem','root'))
 800                 connect_client(clients[-1])
 801 # Creating script to bind the right network interface to the poll mode driver
 802                 devbindfile = '{}_{}_devbindvm{}.sh'.format(env,test_file, vm+1)
 803                 with open("devbind.sh") as f:
 804                         newText=f.read().replace('MACADDRESS', vmDPmac[machine_index[vm]])
 805                         with open(devbindfile, "w") as f:
 806                                 f.write(newText)
 807                 st = os.stat(devbindfile)
 808                 os.chmod(devbindfile, st.st_mode | stat.S_IEXEC)
 809                 clients[-1].scp_put('./%s'%devbindfile, '/root/devbind.sh')
 810                 cmd = '/root/devbind.sh'
 811                 clients[-1].run_cmd(cmd)
 812                 log.debug("devbind.sh running on VM%d"%(vm+1))
 813                 clients[-1].scp_put('./%s'%config_file[vm], '/root/%s'%config_file[vm])
 814                 clients[-1].scp_put('./{}_{}_parameters{}.lua'.format(env,test_file, vm+1), '/root/parameters.lua')
 815                 if not configonly:
 816                         if prox_launch_exit[vm]:
 817                                 log.debug("Starting PROX on VM%d"%(vm+1))
 818                                 if auto_start[vm]:
 819                                         cmd = '/root/prox/build/prox -t -o cli -f /root/%s'%config_file[vm]
 820                                 else:
 821                                         cmd = '/root/prox/build/prox -e -t -o cli -f /root/%s'%config_file[vm]
 822                                 clients[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1))
 823                         socks_control.append(prox_launch_exit[vm])
 824                         socks.append(connect_socket(clients[-1]))
 825                         sock_type.append(mach_type[vm])
 826 socks.append('none')
 827 socks_control.append(False)
 828
 829 def get_BinarySearchParams() :
 830         global DROP_RATE_TRESHOLD
 831         global LAT_AVG_TRESHOLD
 832         global LAT_MAX_TRESHOLD
 833         global ACCURACY
 834         global STARTSPEED
 835         DROP_RATE_TRESHOLD = float(testconfig.get('BinarySearchParams', 'drop_rate_threshold'))
 836         LAT_AVG_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_avg_threshold'))
 837         LAT_MAX_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_max_threshold'))
 838         ACCURACY = float(testconfig.get('BinarySearchParams', 'accuracy'))
 839         STARTSPEED = float(testconfig.get('BinarySearchParams', 'startspeed'))
 840
 841 if configonly:
 842         sys.exit()
 843 ####################################################
 844 # Run test cases
 845 # Best to run the flow test at the end since otherwise the tests coming after might be influenced by the big number of entries in the switch flow tables
 846 ####################################################
 847 gensock_index = sock_type.index('gen') if 'gen' in sock_type else -1
 848 sutsock_index = sock_type.index('sut') if 'sut' in sock_type else -1
 849 irqsock_index = sock_type.index('irq') if 'irq' in sock_type else -1
 850 number_of_tests = testconfig.get('DEFAULT', 'number_of_tests')
 851 with data_csv_file:
 852         fieldnames = ['flow','size','endspeed','endspeedpps','endpps_req_tx','endpps_tx','endpps_sut_tx_str','endpps_rx','endlat_avg','endlat_max','endabs_dropped','enddrop_rate']
 853         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 854         writer.writeheader()
 855         for test_nr in range(1, int(number_of_tests)+1):
 856                 test=testconfig.get('test%d'%test_nr,'test')
 857                 tasks= ast.literal_eval(testconfig.get('test%d'%test_nr, 'tasks'))
 858                 log.info(test)
 859                 if test == 'flowsizetest':
 860                         get_BinarySearchParams()
 861                         packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
 862                         flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
 863                         run_flow_size_test(socks[gensock_index],socks[sutsock_index])
 864                 elif test == 'fixed_rate':
 865                         packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
 866                         STARTSPEED = float(testconfig.get('test%d'%test_nr, 'speed'))
 867                         run_fixed_rate(socks[gensock_index],socks[sutsock_index])
 868                 elif test == 'measureswap':
 869                         #packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
 870                         run_measure_swap(socks[sutsock_index])
 871                 elif test == 'impairtest':
 872                         get_BinarySearchParams()
 873                         PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
 874                         run_impairtest(socks[gensock_index],socks[sutsock_index])
 875                 elif test == 'irqtest':
 876                         run_irqtest(socks[irqsock_index])
 877                 elif test == 'warmuptest':
 878                         PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
 879                         FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
 880                         WARMUPSPEED = int(testconfig.get('test%d'%test_nr, 'warmupspeed'))
 881                         WARMUPTIME = int(testconfig.get('test%d'%test_nr, 'warmuptime'))
 882                         run_warmuptest(socks[gensock_index])
 883 ####################################################