VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py

   1 #!/usr/bin/python
   2
   3 ##
   4 ## Copyright (c) 2010-2020 Intel Corporation
   5 ##
   6 ## Licensed under the Apache License, Version 2.0 (the "License");
   7 ## you may not use this file except in compliance with the License.
   8 ## You may obtain a copy of the License at
   9 ##
  10 ##     http://www.apache.org/licenses/LICENSE-2.0
  11 ##
  12 ## Unless required by applicable law or agreed to in writing, software
  13 ## distributed under the License is distributed on an "AS IS" BASIS,
  14 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 ## See the License for the specific language governing permissions and
  16 ## limitations under the License.
  17 ##
  18
  19 from __future__ import print_function
  20
  21 import os
  22 import stat
  23 import sys
  24 import time
  25 import subprocess
  26 import getopt
  27 import re
  28 import logging
  29 from logging.handlers import RotatingFileHandler
  30 from logging import handlers
  31 from prox_ctrl import prox_ctrl
  32 import ConfigParser
  33 import ast
  34 import atexit
  35 import csv
  36 import requests
  37 from numpy import inf
  38 from math import ceil
  39
  40 version="20.01.10"
  41 env = "rapid.env" #Default string for environment
  42 test_file = "basicrapid.test" #Default string for test
  43 machine_map_file = "machine.map" #Default string for machine map file
  44 loglevel="DEBUG" # sets log level for writing to file
  45 screenloglevel="INFO" # sets log level for writing to screen
  46 runtime=10 # time in seconds for 1 test run
  47 configonly = False # IF True, the system will upload all the necessary config fiels to the VMs, but not start PROX and the actual testing
  48 rundir = "/home/centos" # Directory where to find the tools in the machines running PROX
  49
  50 def usage():
  51         print("usage: runrapid    [--version] [-v]")
  52         print("                   [--env ENVIRONMENT_NAME]")
  53         print("                   [--test TEST_NAME]")
  54         print("                   [--map MACHINE_MAP_FILE]")
  55         print("                   [--runtime TIME_FOR_TEST]")
  56         print("                   [--configonly False|True]")
  57         print("                   [--log DEBUG|INFO|WARNING|ERROR|CRITICAL]")
  58         print("                   [-h] [--help]")
  59         print("")
  60         print("Command-line interface to runrapid")
  61         print("")
  62         print("optional arguments:")
  63         print("  -v,  --version                 Show program's version number and exit")
  64         print("  --env ENVIRONMENT_NAME         Parameters will be read from ENVIRONMENT_NAME. Default is %s."%env)
  65         print("  --test TEST_NAME               Test cases will be read from TEST_NAME. Default is %s."%test_file)
  66         print("  --map MACHINE_MAP_FILE Machine mapping will be read from MACHINE_MAP_FILE. Default is %s."%machine_map_file)
  67         print("  --runtime                      Specify time in seconds for 1 test run")
  68         print("  --configonly                   If this option is specified, only upload all config files to the VMs, do not run the tests")
  69         print("  --log                          Specify logging level for log file output, default is DEBUG")
  70         print("  --screenlog                    Specify logging level for screen output, default is INFO")
  71         print("  -h, --help                     Show help message and exit.")
  72         print("")
  73
  74 try:
  75         opts, args = getopt.getopt(sys.argv[1:], "vh", ["version","help", "env=", "test=", "map=", "runtime=","configonly","log=","screenlog="])
  76 except getopt.GetoptError as err:
  77         print("===========================================")
  78         print(str(err))
  79         print("===========================================")
  80         usage()
  81         sys.exit(2)
  82 if args:
  83         usage()
  84         sys.exit(2)
  85 for opt, arg in opts:
  86         if opt in ["-h", "--help"]:
  87                 usage()
  88                 sys.exit()
  89         if opt in ["-v", "--version"]:
  90                 print("Rapid Automated Performance Indication for Dataplane "+version)
  91                 sys.exit()
  92         if opt in ["--env"]:
  93                 env = arg
  94         if opt in ["--test"]:
  95                 test_file = arg
  96         if opt in ["--map"]:
  97                 machine_map_file = arg
  98         if opt in ["--runtime"]:
  99                 runtime = arg
 100         if opt in ["--configonly"]:
 101                 configonly = True
 102                 print('No actual runs, only uploading configuration files')
 103         if opt in ["--log"]:
 104                 loglevel = arg
 105                 print ("Log level: "+ loglevel)
 106         if opt in ["--screenlog"]:
 107                 screenloglevel = arg
 108                 print ("Screen Log level: "+ screenloglevel)
 109
 110 print ("Using '"+env+"' as name for the environment")
 111 print ("Using '"+test_file+"' for test case definition")
 112 print ("Using '"+machine_map_file+"' for machine mapping")
 113 print ("Runtime: "+ str(runtime))
 114
 115 class bcolors:
 116         HEADER = '\033[95m'
 117         OKBLUE = '\033[94m'
 118         OKGREEN = '\033[92m'
 119         WARNING = '\033[93m'
 120         FAIL = '\033[91m'
 121         ENDC = '\033[0m'
 122         BOLD = '\033[1m'
 123         UNDERLINE = '\033[4m'
 124
 125 # create formatters
 126 screen_formatter = logging.Formatter("%(message)s")
 127 file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 128
 129 # get a top-level logger,
 130 # set its log level,
 131 # BUT PREVENT IT from propagating messages to the root logger
 132 #
 133 log = logging.getLogger()
 134 numeric_level = getattr(logging, loglevel.upper(), None)
 135 if not isinstance(numeric_level, int):
 136         raise ValueError('Invalid log level: %s' % loglevel)
 137 log.setLevel(numeric_level)
 138 log.propagate = 0
 139
 140 # create a console handler
 141 # and set its log level to the command-line option
 142 #
 143 console_handler = logging.StreamHandler(sys.stdout)
 144 #console_handler.setLevel(logging.INFO)
 145 numeric_screenlevel = getattr(logging, screenloglevel.upper(), None)
 146 if not isinstance(numeric_screenlevel, int):
 147         raise ValueError('Invalid screenlog level: %s' % screenloglevel)
 148 console_handler.setLevel(numeric_screenlevel)
 149 console_handler.setFormatter(screen_formatter)
 150
 151 # create a file handler
 152 # and set its log level
 153 #
 154 log_file = 'RUN{}.{}.log'.format(env,test_file)
 155 file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10)
 156 #file_handler = log.handlers.TimedRotatingFileHandler(log_file, 'D', 1, 5)
 157 file_handler.setLevel(numeric_level)
 158 file_handler.setFormatter(file_formatter)
 159
 160 # add handlers to the logger
 161 #
 162 log.addHandler(file_handler)
 163 log.addHandler(console_handler)
 164
 165 # Check if log exists and should therefore be rolled
 166 needRoll = os.path.isfile(log_file)
 167
 168
 169 # This is a stale log, so roll it
 170 if needRoll:
 171         # Add timestamp
 172         log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime())
 173
 174         # Roll over on application start
 175         log.handlers[0].doRollover()
 176
 177 # Add timestamp
 178 log.debug('\n---------\nLog started on %s.\n---------\n' % time.asctime())
 179
 180 log.debug("runrapid.py version: "+version)
 181 #========================================================================
 182 def connect_socket(client):
 183         attempts = 1
 184         log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 185         sock = None
 186         while True:
 187                 sock = client.prox_sock()
 188                 if sock is not None:
 189                         break
 190                 attempts += 1
 191                 if attempts > 20:
 192                         log.exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 193                         raise Exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 194                 time.sleep(2)
 195                 log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 196         log.info("Connected to PROX on %s" % client.ip())
 197         return sock
 198
 199 def connect_client(client):
 200         attempts = 1
 201         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 202         while True:
 203                 try:
 204                         client.connect()
 205                         break
 206                 except RuntimeWarning, ex:
 207                         attempts += 1
 208                         if attempts > 20:
 209                                 log.exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 210                                 raise Exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 211                         time.sleep(2)
 212                         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 213         log.debug("Connected to VM on %s" % client.ip())
 214
 215 def report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,tx,rx,tot_drop,elapsed_time,speed_prefix='',lat_avg_prefix='',lat_perc_prefix='',lat_max_prefix='',abs_drop_rate_prefix='',drop_rate_prefix=''):
 216         if flow_number < 0:
 217                 flow_number_str = '| ({:>4}) |'.format(abs(flow_number))
 218         else:
 219                 flow_number_str = '|{:>7} |'.format(flow_number)
 220         if pps_req_tx == None:
 221                 pps_req_tx_str = '{0: >14}'.format('   NA     |')
 222         else:
 223                 pps_req_tx_str = '{:>7.3f} Mpps |'.format(pps_req_tx)
 224         if pps_tx == None:
 225                 pps_tx_str = '{0: >14}'.format('   NA     |')
 226         else:
 227                 pps_tx_str = '{:>7.3f} Mpps |'.format(pps_tx)
 228         if pps_sut_tx == None:
 229                 pps_sut_tx_str = '{0: >14}'.format('   NA     |')
 230         else:
 231                 pps_sut_tx_str = '{:>7.3f} Mpps |'.format(pps_sut_tx)
 232         if pps_rx == None:
 233                 pps_rx_str = '{0: >24|}'.format('NA        ')
 234         else:
 235                 pps_rx_str = bcolors.OKBLUE + '{:>4.1f} Gb/s |{:7.3f} Mpps {}|'.format(get_speed(pps_rx,size),pps_rx,bcolors.ENDC)
 236         if tot_drop == None:
 237                 tot_drop_str = ' |       NA  | '
 238         else:
 239                 tot_drop_str = ' | {:>9.0f} | '.format(tot_drop)
 240         if lat_perc == None:
 241                 lat_perc_str = ' |{:^10.10}|'.format('NA')
 242         elif lat_perc_max == True:
 243                 lat_perc_str = ' |>{}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC)
 244         else:
 245                 lat_perc_str = ' | {}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC)
 246         if elapsed_time == None:
 247                 elapsed_time_str = ' NA |'
 248         else:
 249                 elapsed_time_str = '{:>3.0f} |'.format(elapsed_time)
 250         return(flow_number_str + '{:>5.1f}'.format(speed) + '% '+speed_prefix +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+ pps_req_tx_str + pps_tx_str + bcolors.ENDC + pps_sut_tx_str + pps_rx_str +lat_avg_prefix+ ' {:>5.0f}'.format(lat_avg)+' us'+lat_perc_str+lat_max_prefix+'{:>6.0f}'.format(lat_max)+' us | ' + '{:>9.0f}'.format(tx) + ' | {:>9.0f}'.format(rx) + ' | '+ abs_drop_rate_prefix+ '{:>9.0f}'.format(tx-rx) + tot_drop_str +drop_rate_prefix+ '{:>5.2f}'.format(float(tx-rx)/tx)  +bcolors.ENDC+' |' + elapsed_time_str)
 251
 252 def run_iteration(gensock, sutsock, requested_duration,flow_number,size,speed):
 253         r = 0;
 254         sleep_time = 2
 255         while (r < TST009_MAXr):
 256                 time.sleep(sleep_time)
 257                 # Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
 258                 t1_rx, t1_non_dp_rx, t1_tx, t1_non_dp_tx, t1_drop, t1_tx_fail, t1_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
 259                 t1_dp_rx = t1_rx - t1_non_dp_rx
 260                 t1_dp_tx = t1_tx - t1_non_dp_tx
 261                 gensock.start(gencores)
 262                 time.sleep(2) ## Needs to be 2 seconds since this the time that PROX uses to refresh the stats. Note that this can be changed in PROX!! Don't do it.
 263                 if sutsock!='none':
 264                         t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 265                         ##t2_sut_rx = t2_sut_rx - t2_sut_non_dp_rx
 266                         ##t2_sut_tx = t2_sut_tx - t2_sut_non_dp_tx
 267                 t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 268                 tx = t2_tx - t1_tx
 269                 dp_tx =  tx - (t2_non_dp_tx - t1_non_dp_tx )
 270                 dp_rx =  t2_rx - t1_rx - (t2_non_dp_rx - t1_non_dp_rx)
 271                 tot_dp_drop = dp_tx - dp_rx
 272                 if tx == 0:
 273                         log.critical("TX = 0. Test interrupted since no packet has been sent.")
 274                         raise Exception("TX = 0")
 275                 if dp_tx == 0:
 276                         log.critical("Only non-dataplane packets (e.g. ARP) sent. Test interrupted since no packet has been sent.")
 277                         raise Exception("Only non-dataplane packets (e.g. ARP) sent")
 278                 # Ask PROX to calibrate the bucket size once we have a PROX function to do this.
 279                 # Measure latency statistics per second
 280                 lat_min, lat_max, lat_avg, used_avg, t2_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
 281                 lat_samples = sum(buckets)
 282                 sample_count = 0
 283                 for sample_percentile, bucket in enumerate(buckets,start=1):
 284                         sample_count += bucket
 285                         if sample_count > (lat_samples * LAT_PERCENTILE):
 286                                 break
 287                 if sample_percentile == len(buckets):
 288                         percentile_max = True
 289                 else:
 290                         percentile_max = False
 291                 sample_percentile = sample_percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
 292                 if test == 'fixed_rate':
 293                         log.info(report_result(flow_number,size,speed,None,None,None,None,lat_avg,sample_percentile,percentile_max,lat_max, dp_tx, dp_rx , None, None))
 294                 tot_rx = tot_non_dp_rx = tot_tx = tot_non_dp_tx = tot_drop = 0
 295                 lat_avg = used_avg = 0
 296                 buckets_total = [0] * 128
 297                 tot_lat_samples = 0
 298                 tot_lat_measurement_duration = float(0)
 299                 tot_core_measurement_duration = float(0)
 300                 tot_sut_core_measurement_duration = float(0)
 301                 tot_sut_rx = tot_sut_non_dp_rx = tot_sut_tx = tot_sut_non_dp_tx = tot_sut_drop = tot_sut_tx_fail = tot_sut_tsc = 0
 302                 lat_avail = core_avail = sut_avail = False
 303                 ##while (tot_core_measurement_duration - float(requested_duration) <= 0.1) or (tot_sut_core_measurement_duration - float(requested_duration) <= 0.1) or (tot_lat_measurement_duration - float(requested_duration) <= 0.1):
 304                 while (tot_core_measurement_duration - float(requested_duration) <= 0.1) or (tot_lat_measurement_duration - float(requested_duration) <= 0.1):
 305                         time.sleep(0.5)
 306                         lat_min_sample, lat_max_sample, lat_avg_sample, used_sample, t3_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
 307                         single_lat_measurement_duration = (t3_lat_tsc - t2_lat_tsc) * 1.0 / lat_hz  # time difference between the 2 measurements, expressed in seconds.
 308                         # Get statistics after some execution time
 309                         if single_lat_measurement_duration != 0:
 310                                 # A second has passed in between to lat_stats requests. Hence we need to process the results
 311                                 tot_lat_measurement_duration = tot_lat_measurement_duration + single_lat_measurement_duration
 312                                 if lat_min > lat_min_sample:
 313                                         lat_min = lat_min_sample
 314                                 if lat_max < lat_max_sample:
 315                                         lat_max = lat_max_sample
 316                                 lat_avg = lat_avg + lat_avg_sample * single_lat_measurement_duration # Sometimes, There is more than 1 second between 2 lat_stats. Hence we will take the latest measurement
 317                                 used_avg = used_avg + used_sample * single_lat_measurement_duration  # and give it more weigth.
 318                                 lat_samples = sum(buckets)
 319                                 tot_lat_samples += lat_samples
 320                                 sample_count = 0
 321                                 for sample_percentile, bucket in enumerate(buckets,start=1):
 322                                         sample_count += bucket
 323                                         if sample_count > lat_samples * LAT_PERCENTILE:
 324                                                 break
 325                                 if sample_percentile == len(buckets):
 326                                         percentile_max = True
 327                                 else:
 328                                         percentile_max = False
 329                                 sample_percentile = sample_percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
 330                                 buckets_total = [buckets_total[i] + buckets[i] for i in range(len(buckets_total))]
 331                                 t2_lat_tsc = t3_lat_tsc
 332                                 lat_avail = True
 333                         t3_rx, t3_non_dp_rx, t3_tx, t3_non_dp_tx, t3_drop, t3_tx_fail, t3_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 334                         single_core_measurement_duration = (t3_tsc - t2_tsc) * 1.0 / tsc_hz  # time difference between the 2 measurements, expressed in seconds.
 335                         if single_core_measurement_duration!= 0:
 336                                 stored_single_core_measurement_duration = single_core_measurement_duration
 337                                 tot_core_measurement_duration = tot_core_measurement_duration + single_core_measurement_duration
 338                                 delta_rx = t3_rx - t2_rx
 339                                 tot_rx += delta_rx
 340                                 delta_non_dp_rx = t3_non_dp_rx - t2_non_dp_rx
 341                                 tot_non_dp_rx += delta_non_dp_rx
 342                                 delta_tx = t3_tx - t2_tx
 343                                 tot_tx += delta_tx
 344                                 delta_non_dp_tx = t3_non_dp_tx - t2_non_dp_tx
 345                                 tot_non_dp_tx += delta_non_dp_tx
 346                                 delta_dp_tx = delta_tx -delta_non_dp_tx
 347                                 delta_dp_rx = delta_rx -delta_non_dp_rx
 348                                 delta_dp_drop = delta_dp_tx - delta_dp_rx
 349                                 tot_dp_drop += delta_dp_drop
 350                                 delta_drop = t3_drop - t2_drop
 351                                 tot_drop += delta_drop
 352                                 t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc = t3_rx, t3_non_dp_rx, t3_tx, t3_non_dp_tx, t3_drop, t3_tx_fail, t3_tsc
 353                                 core_avail = True
 354                         if sutsock!='none':
 355                                 t3_sut_rx, t3_sut_non_dp_rx, t3_sut_tx, t3_sut_non_dp_tx, t3_sut_drop, t3_sut_tx_fail, t3_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 356                                 single_sut_core_measurement_duration = (t3_sut_tsc - t2_sut_tsc) * 1.0 / tsc_hz  # time difference between the 2 measurements, expressed in seconds.
 357                                 if single_sut_core_measurement_duration!= 0:
 358                                         stored_single_sut_core_measurement_duration = single_sut_core_measurement_duration
 359                                         tot_sut_core_measurement_duration = tot_sut_core_measurement_duration + single_sut_core_measurement_duration
 360                                         tot_sut_rx += t3_sut_rx - t2_sut_rx
 361                                         tot_sut_non_dp_rx += t3_sut_non_dp_rx - t2_sut_non_dp_rx
 362                                         delta_sut_tx = t3_sut_tx - t2_sut_tx
 363                                         tot_sut_tx += delta_sut_tx
 364                                         delta_sut_non_dp_tx = t3_sut_non_dp_tx - t2_sut_non_dp_tx
 365                                         tot_sut_non_dp_tx += delta_sut_non_dp_tx
 366                                         t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc = t3_sut_rx, t3_sut_non_dp_rx, t3_sut_tx, t3_sut_non_dp_tx, t3_sut_drop, t3_sut_tx_fail, t3_sut_tsc
 367                                         sut_avail = True
 368                         if test == 'fixed_rate':
 369                                 if lat_avail == core_avail == True:
 370                                         lat_avail = core_avail = False
 371                                         pps_req_tx = (delta_tx + delta_drop - delta_rx)/stored_single_core_measurement_duration/1000000
 372                                         pps_tx = delta_tx/stored_single_core_measurement_duration/1000000
 373                                         if sutsock!='none' and sut_avail:
 374                                                 pps_sut_tx = delta_sut_tx/stored_single_sut_core_measurement_duration/1000000
 375                                                 sut_avail = False
 376                                         else:
 377                                                 pps_sut_tx = None
 378                                         pps_rx = delta_rx/stored_single_core_measurement_duration/1000000
 379                                         log.info(report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg_sample,sample_percentile,percentile_max,lat_max_sample,delta_dp_tx,delta_dp_rx,tot_dp_drop,stored_single_core_measurement_duration))
 380                 #Stop generating
 381                 gensock.stop(gencores)
 382                 r += 1
 383                 lat_avg = lat_avg / float(tot_lat_measurement_duration)
 384                 used_avg = used_avg / float(tot_lat_measurement_duration)
 385                 t4_tsc = t2_tsc
 386                 while t4_tsc == t2_tsc:
 387                         t4_rx, t4_non_dp_rx, t4_tx, t4_non_dp_tx, t4_drop, t4_tx_fail, t4_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
 388                 if test == 'fixed_rate':
 389                         t4_lat_tsc = t2_lat_tsc
 390                         while t4_lat_tsc == t2_lat_tsc:
 391                                 lat_min_sample, lat_max_sample, lat_avg_sample, used_sample, t4_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
 392                         sample_count = 0
 393                         lat_samples = sum(buckets)
 394                         for percentile, bucket in enumerate(buckets,start=1):
 395                                 sample_count += bucket
 396                                 if sample_count > lat_samples * LAT_PERCENTILE:
 397                                         break
 398                         if percentile == len(buckets):
 399                                 percentile_max = True
 400                         else:
 401                                 percentile_max = False
 402                         percentile = percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
 403                         lat_max = lat_max_sample
 404                         lat_avg = lat_avg_sample
 405                         delta_rx = t4_rx - t2_rx
 406                         delta_non_dp_rx = t4_non_dp_rx - t2_non_dp_rx
 407                         delta_tx = t4_tx - t2_tx
 408                         delta_non_dp_tx = t4_non_dp_tx - t2_non_dp_tx
 409                         delta_dp_tx = delta_tx -delta_non_dp_tx
 410                         delta_dp_rx = delta_rx -delta_non_dp_rx
 411                         dp_tx = delta_dp_tx
 412                         dp_rx = delta_dp_rx
 413                         tot_dp_drop += delta_dp_tx - delta_dp_rx
 414                         pps_req_tx = None
 415                         pps_tx = None
 416                         pps_sut_tx = None
 417                         pps_rx = None
 418                         drop_rate = 100.0*(dp_tx-dp_rx)/dp_tx
 419                         tot_core_measurement_duration = None
 420                         break ## Not really needed since the while loop will stop when evaluating the value of r
 421                 else:
 422                         sample_count = 0
 423                         for percentile, bucket in enumerate(buckets_total,start=1):
 424                                 sample_count += bucket
 425                                 if sample_count > tot_lat_samples * LAT_PERCENTILE:
 426                                         break
 427                         if percentile == len(buckets):
 428                                 percentile_max = True
 429                         else:
 430                                 percentile_max = False
 431                         percentile = percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
 432                         pps_req_tx = (tot_tx + tot_drop - tot_rx)/tot_core_measurement_duration/1000000.0 # tot_drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 433                         pps_tx = tot_tx/tot_core_measurement_duration/1000000.0 # tot_tx is all generated packets actually accepted by the interface
 434                         pps_rx = tot_rx/tot_core_measurement_duration/1000000.0 # tot_rx is all packets received by the nop task = all packets received in the gen VM
 435                         if sutsock!='none' and sut_avail:
 436                                 pps_sut_tx = tot_sut_tx / tot_sut_core_measurement_duration / 1000000.0
 437                         else:
 438                                 pps_sut_tx = None
 439                         dp_tx = (t4_tx - t1_tx) - (t4_non_dp_tx - t1_non_dp_tx)
 440                         dp_rx = (t4_rx - t1_rx) - (t4_non_dp_rx - t1_non_dp_rx)
 441                         tot_dp_drop = dp_tx - dp_rx
 442                         drop_rate = 100.0*tot_dp_drop/dp_tx
 443                         if ((drop_rate < DROP_RATE_TRESHOLD) or (tot_dp_drop == DROP_RATE_TRESHOLD ==0) or (tot_dp_drop > TST009_MAXz)):
 444                                 break
 445         return(pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,percentile,percentile_max,lat_max,dp_tx,dp_rx,tot_dp_drop,(t4_tx_fail - t1_tx_fail),drop_rate,lat_min,used_avg,r,tot_core_measurement_duration)
 446
 447 def new_speed(speed,size,success):
 448         if test == 'fixed_rate':
 449                 return (STARTSPEED)
 450         elif TST009:
 451                 global TST009_m
 452                 global TST009_L
 453                 global TST009_R
 454                 if success:
 455                         TST009_L = TST009_m + 1
 456                 else:
 457                         TST009_R = max(TST009_m - 1, TST009_L)
 458                 TST009_m = int ((TST009_L + TST009_R)/2)
 459                 return (get_percentageof10Gbs(TST009_S[TST009_m],size))
 460         else:
 461                 global minspeed
 462                 global maxspeed
 463                 if success:
 464                         minspeed = speed
 465                 else:
 466                         maxspeed = speed
 467                 return ((minspeed + maxspeed)/2.0)
 468
 469 def get_start_speed_and_init(size):
 470         if test == 'fixed_rate':
 471                 return (STARTSPEED)
 472         elif TST009:
 473                 global TST009_L
 474                 global TST009_R
 475                 global TST009_m
 476                 TST009_L = 0
 477                 TST009_R = TST009_n - 1
 478                 TST009_m = int((TST009_L + TST009_R) / 2)
 479                 return (get_percentageof10Gbs(TST009_S[TST009_m],size))
 480         else:
 481                 global minspeed
 482                 global maxspeed
 483                 minspeed = 0
 484                 maxspeed = STARTSPEED
 485                 return (STARTSPEED)
 486
 487 def resolution_achieved():
 488         if test == 'fixed_rate':
 489                 return (True)
 490         elif TST009:
 491                 return (TST009_L == TST009_R)
 492         else:
 493                 return ((maxspeed - minspeed) <= ACCURACY)
 494
 495 def get_percentageof10Gbs(pps_speed,size):
 496         # speed is given in pps, returning % of 10Gb/s
 497         return (pps_speed / 1000000.0 * 0.08 * (size+24))
 498
 499 def get_pps(speed,size):
 500         # speed is given in % of 10Gb/s, returning Mpps
 501         return (speed * 100.0 / (8*(size+24)))
 502
 503 def get_speed(packet_speed,size):
 504         # return speed in Gb/s
 505         return (packet_speed / 1000.0 * (8*(size+24)))
 506
 507 def run_flow_size_test(gensock,sutsock):
 508         global fieldnames
 509         global writer
 510         #fieldnames = ['Flows','PacketSize','Gbps','Mpps','AvgLatency','MaxLatency','PacketsDropped','PacketDropRate']
 511         fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Sent','Received','Lost','LostTotal']
 512         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 513         writer.writeheader()
 514         gensock.start(latcores)
 515         for size in packet_size_list:
 516                 size = size-4
 517                 gensock.set_size(gencores,0,size) # This is setting the frame size
 518                 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 519                 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 520                 # This will only work when using sending UDP packets. For different protocls and ethernet types, we would need a different calculation
 521                 log.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 522                 log.info("| UDP, "+ '{:>5}'.format(size+4) +" bytes, different number of flows by randomizing SRC & DST UDP port                                                                                                              |")
 523                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 524                 log.info("| Flows  | Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core           | Avg. Lat.|" + '{:.0f} '.format(LAT_PERCENTILE*100) +"Pcentil| Max. Lat.|   Sent    |  Received |    Lost   | Total Lost|L.Ratio|Time|")
 525                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 526                 for flow_number in flow_size_list:
 527                         attempts = 0
 528                         gensock.reset_stats()
 529                         if sutsock!='none':
 530                                 sutsock.reset_stats()
 531                         source_port,destination_port = flows[flow_number]
 532                         gensock.set_random(gencores,0,34,source_port,2)
 533                         gensock.set_random(gencores,0,36,destination_port,2)
 534                         endspeed = None
 535                         speed = get_start_speed_and_init(size)
 536                         while True:
 537                                 attempts += 1
 538                                 endwarning = False
 539                                 print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 540                                 sys.stdout.flush()
 541                                 # Start generating packets at requested speed (in % of a 10Gb/s link)
 542                                 gensock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
 543                                 ##time.sleep(1)
 544                                 # Get statistics now that the generation is stable and initial ARP messages are dealt with
 545                                 pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc , lat_perc_max, lat_max, abs_tx,abs_rx,abs_dropped, abs_tx_fail, drop_rate, lat_min, lat_used, r, actual_duration = run_iteration(gensock,sutsock,float(runtime),flow_number,size,speed)
 546                                 if r > 1:
 547                                         retry_warning = bcolors.WARNING + ' {:1} retries needed'.format(r) +  bcolors.ENDC
 548                                 else:
 549                                         retry_warning = ''
 550                                 # Drop rate is expressed in percentage. lat_used is a ratio (0 to 1). The sum of these 2 should be 100%.
 551                                 # If the some is lower than 95, it means that more than 5% of the latency measurements where dropped for accuracy reasons.
 552                                 if (drop_rate + lat_used * 100) < 95:
 553                                         lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 554                                 else:
 555                                         lat_warning = ''
 556                                 if test == 'fixed_rate':
 557                                         endspeed = speed
 558                                         endpps_req_tx = None
 559                                         endpps_tx = None
 560                                         endpps_sut_tx = None
 561                                         endpps_rx = None
 562                                         endlat_avg = lat_avg
 563                                         endlat_perc = lat_perc
 564                                         endlat_perc_max = lat_perc_max
 565                                         endlat_max = lat_max
 566                                         endabs_dropped = abs_dropped
 567                                         enddrop_rate = drop_rate
 568                                         endabs_tx = abs_tx
 569                                         endabs_rx = abs_rx
 570                                         if lat_warning or gen_warning or retry_warning:
 571                                                 endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning + gen_warning)
 572                                         success = True
 573                                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 574                                 # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold
 575                                 # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
 576                                 # This can be specified by putting 0 in the .test file
 577                                 elif ((drop_rate < DROP_RATE_TRESHOLD) or (abs_dropped==DROP_RATE_TRESHOLD ==0)) and (lat_avg< LAT_AVG_TRESHOLD) and (lat_perc< LAT_PERC_TRESHOLD) and (lat_max < LAT_MAX_TRESHOLD):
 578                                         lat_avg_prefix = bcolors.ENDC
 579                                         lat_perc_prefix = bcolors.ENDC
 580                                         lat_max_prefix = bcolors.ENDC
 581                                         abs_drop_rate_prefix = bcolors.ENDC
 582                                         drop_rate_prefix = bcolors.ENDC
 583                                         if ((get_pps(speed,size) - pps_tx)/get_pps(speed,size))>0.01:
 584                                                 speed_prefix = bcolors.WARNING
 585                                                 if abs_tx_fail > 0:
 586                                                         gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
 587                                                 else:
 588                                                         gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(get_pps(speed,size), pps_tx) + bcolors.ENDC
 589                                         else:
 590                                                 speed_prefix = bcolors.ENDC
 591                                                 gen_warning = ''
 592                                         endspeed = speed
 593                                         endspeed_prefix = speed_prefix
 594                                         endpps_req_tx = pps_req_tx
 595                                         endpps_tx = pps_tx
 596                                         endpps_sut_tx = pps_sut_tx
 597                                         endpps_rx = pps_rx
 598                                         endlat_avg = lat_avg
 599                                         endlat_perc = lat_perc
 600                                         endlat_perc_max = lat_perc_max
 601                                         endlat_max = lat_max
 602                                         endabs_dropped = None
 603                                         enddrop_rate = drop_rate
 604                                         endabs_tx = abs_tx
 605                                         endabs_rx = abs_rx
 606                                         if lat_warning or gen_warning or retry_warning:
 607                                                 endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning + gen_warning)
 608                                         success = True
 609                                         success_message=' SUCCESS'
 610                                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 611                                         log.debug(report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
 612                                 else:
 613                                         success_message=' FAILED'
 614                                         gen_warning = ''
 615                                         abs_drop_rate_prefix = bcolors.ENDC
 616                                         if ((abs_dropped>0) and (DROP_RATE_TRESHOLD ==0)):
 617                                                 abs_drop_rate_prefix = bcolors.FAIL
 618                                         if (drop_rate < DROP_RATE_TRESHOLD):
 619                                                 drop_rate_prefix = bcolors.ENDC
 620                                         else:
 621                                                 drop_rate_prefix = bcolors.FAIL
 622                                         if (lat_avg< LAT_AVG_TRESHOLD):
 623                                                 lat_avg_prefix = bcolors.ENDC
 624                                         else:
 625                                                 lat_avg_prefix = bcolors.FAIL
 626                                         if (lat_perc< LAT_PERC_TRESHOLD):
 627                                                 lat_perc_prefix = bcolors.ENDC
 628                                         else:
 629                                                 lat_perc_prefix = bcolors.FAIL
 630                                         if (lat_max< LAT_MAX_TRESHOLD):
 631                                                 lat_max_prefix = bcolors.ENDC
 632                                         else:
 633                                                 lat_max_prefix = bcolors.FAIL
 634                                         if (((get_pps(speed,size) - pps_tx)/get_pps(speed,size))<0.001):
 635                                                 speed_prefix = bcolors.ENDC
 636                                         else:
 637                                                 speed_prefix = bcolors.FAIL
 638                                         success = False
 639                                         log.debug(report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
 640                                 speed = new_speed(speed, size, success)
 641                                 if resolution_achieved():
 642                                         break
 643                         if endspeed !=  None:
 644                                 log.info(report_result(flow_number,size,endspeed,endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_perc,endlat_perc_max,endlat_max,endabs_tx,endabs_rx,endabs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix))
 645                                 if endwarning:
 646                                         log.info (endwarning)
 647                                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 648                                 writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':get_pps(endspeed,size),'GeneratedPPS':endpps_req_tx,'SentPPS':endpps_tx,'ForwardedPPS':endpps_sut_tx,'ReceivedPPS':endpps_rx,'AvgLatencyUSEC':endlat_avg,'MaxLatencyUSEC':endlat_max,'Sent':endabs_tx,'Received':endabs_rx,'Lost':endabs_dropped,'LostTotal':endabs_dropped})
 649                                 if PushGateway:
 650                                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
 651                                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nSent {}\nReceived {}\nLost {}\nLostTotal {}\n'.format(flow_number,size+4,get_pps(endspeed,size),endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_max,endabs_tx,endabs_rx,endabs_Dropped,endabs_dropped)
 652                                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 653                                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 654                         else:
 655                                 log.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
 656         gensock.stop(latcores)
 657
 658 def run_core_stats(socks):
 659         fieldnames = ['PROXID','Time','Received','Sent','NonDPReceived','NonDPSent','Delta','NonDPDelta','Dropped']
 660         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 661         writer.writeheader()
 662         log.info("+------------------------------------------------------------------------------------------------------------------+")
 663         log.info("| Measuring core statistics on 1 or more PROX instances                                                            |")
 664         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 665         log.info("| PROX ID   |    Time   |    RX      |     TX     | non DP RX  | non DP TX  |   TX - RX  | nonDP TX-RX|  DROP TOT  |")
 666         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 667         for sock in socks:
 668                 sock.reset_stats()
 669         duration = float(runtime)
 670         tot_drop = []
 671         old_rx = []; old_non_dp_rx = []; old_tx = []; old_non_dp_tx = []; old_drop = []; old_tx_fail = []; old_tsc = []
 672         new_rx = []; new_non_dp_rx = []; new_tx = []; new_non_dp_tx = []; new_drop = []; new_tx_fail = []; new_tsc = []
 673         sockets_to_go = len (socks)
 674         for i,sock in enumerate(socks,start=0):
 675                 tot_drop.append(0)
 676                 old_rx.append(0); old_non_dp_rx.append(0); old_tx.append(0); old_non_dp_tx.append(0); old_drop.append(0); old_tx_fail.append(0); old_tsc.append(0)
 677                 old_rx[-1], old_non_dp_rx[-1], old_tx[-1], old_non_dp_tx[-1], old_drop[-1], old_tx_fail[-1], old_tsc[-1], tsc_hz = sock.core_stats(cores[i],tasks)
 678                 new_rx.append(0); new_non_dp_rx.append(0); new_tx.append(0); new_non_dp_tx.append(0); new_drop.append(0); new_tx_fail.append(0); new_tsc.append(0)
 679         while (duration > 0):
 680                 time.sleep(0.5)
 681                 # Get statistics after some execution time
 682                 for i,sock in enumerate(socks,start=0):
 683                         new_rx[i], new_non_dp_rx[i], new_tx[i], new_non_dp_tx[i], new_drop[i], new_tx_fail[i], new_tsc[i], tsc_hz = sock.core_stats(cores[i],tasks)
 684                         drop = new_drop[i]-old_drop[i]
 685                         rx = new_rx[i] - old_rx[i]
 686                         tx = new_tx[i] - old_tx[i]
 687                         non_dp_rx = new_non_dp_rx[i] - old_non_dp_rx[i]
 688                         non_dp_tx = new_non_dp_tx[i] - old_non_dp_tx[i]
 689                         tsc = new_tsc[i] - old_tsc[i]
 690                         if tsc == 0 :
 691                                 continue
 692                         sockets_to_go -= 1
 693                         old_drop[i] = new_drop[i]
 694                         old_rx[i] = new_rx[i]
 695                         old_tx[i] = new_tx[i]
 696                         old_non_dp_rx[i] = new_non_dp_rx[i]
 697                         old_non_dp_tx[i] = new_non_dp_tx[i]
 698                         old_tsc[i] = new_tsc[i]
 699                         tot_drop[i] = tot_drop[i] + tx - rx
 700                         log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(non_dp_rx)+' | '+'{:>10.0f}'.format(non_dp_tx)+' | ' + '{:>10.0f}'.format(tx-rx) + ' | '+ '{:>10.0f}'.format(non_dp_tx-non_dp_rx) + ' | '+'{:>10.0f}'.format(tot_drop[i]) +' |')
 701                         writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NonDPReceived':non_dp_rx,'NonDPSent':non_dp_tx,'Delta':tx-rx,'NonDPDelta':non_dp_tx-non_dp_rx,'Dropped':tot_drop[i]})
 702                         if PushGateway:
 703                                 URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
 704                                 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNonDPReceived {}\nNonDPSent {}\nDelta {}\nNonDPDelta {}\nDropped {}\n'.format(i,duration,rx,tx,non_dp_rx,non_dp_tx,tx-rx,non_dp_tx-non_dp_rx,tot_drop[i])
 705                                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 706                                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
 707                         if sockets_to_go == 0:
 708                                 duration = duration - 1
 709                                 sockets_to_go = len (socks)
 710         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 711
 712 def run_port_stats(socks):
 713         fieldnames = ['PROXID','Time','Received','Sent','NoMbufs','iErrMiss']
 714         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 715         writer.writeheader()
 716         log.info("+---------------------------------------------------------------------------+")
 717         log.info("| Measuring port statistics on 1 or more PROX instances                     |")
 718         log.info("+-----------+-----------+------------+------------+------------+------------+")
 719         log.info("| PROX ID   |    Time   |    RX      |     TX     | no MBUFS   | ierr&imiss |")
 720         log.info("+-----------+-----------+------------+------------+------------+------------+")
 721         for sock in socks:
 722                 sock.reset_stats()
 723         duration = float(runtime)
 724         old_rx = []; old_tx = []; old_no_mbufs = []; old_errors = []; old_tsc = []
 725         new_rx = []; new_tx = []; new_no_mbufs = []; new_errors = []; new_tsc = []
 726         sockets_to_go = len (socks)
 727         for i,sock in enumerate(socks,start=0):
 728                 old_rx.append(0); old_tx.append(0); old_no_mbufs.append(0); old_errors.append(0); old_tsc.append(0)
 729                 old_rx[-1], old_tx[-1], old_no_mbufs[-1], old_errors[-1], old_tsc[-1] = sock.multi_port_stats(ports[i])
 730                 new_rx.append(0); new_tx.append(0); new_no_mbufs.append(0); new_errors.append(0); new_tsc.append(0)
 731         while (duration > 0):
 732                 time.sleep(0.5)
 733                 # Get statistics after some execution time
 734                 for i,sock in enumerate(socks,start=0):
 735                         new_rx[i], new_tx[i], new_no_mbufs[i], new_errors[i], new_tsc[i] = sock.multi_port_stats(ports[i])
 736                         rx = new_rx[i] - old_rx[i]
 737                         tx = new_tx[i] - old_tx[i]
 738                         no_mbufs = new_no_mbufs[i] - old_no_mbufs[i]
 739                         errors = new_errors[i] - old_errors[i]
 740                         tsc = new_tsc[i] - old_tsc[i]
 741                         if tsc == 0 :
 742                                 continue
 743                         sockets_to_go -= 1
 744                         old_rx[i] = new_rx[i]
 745                         old_tx[i] = new_tx[i]
 746                         old_no_mbufs[i] = new_no_mbufs[i]
 747                         old_errors[i] = new_errors[i]
 748                         old_tsc[i] = new_tsc[i]
 749                         log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(no_mbufs)+' | '+'{:>10.0f}'.format(errors)+' |')
 750                         writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NoMbufs':no_mbufs,'iErrMiss':errors})
 751                         if PushGateway:
 752                                 URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
 753                                 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNoMbufs {}\niErrMiss {}\n'.format(i,duration,rx,tx,no_mbufs,errors)
 754                                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 755                                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
 756                         if sockets_to_go == 0:
 757                                 duration = duration - 1
 758                                 sockets_to_go = len (socks)
 759         log.info("+-----------+-----------+------------+------------+------------+------------+")
 760
 761 def run_irqtest(socks):
 762         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 763         log.info("| Measuring time probably spent dealing with an interrupt. Interrupting DPDK cores for more than 50us might be problematic   ")
 764         log.info("| and result in packet loss. The first row shows the interrupted time buckets: first number is the bucket between 0us and    ")
 765         log.info("| that number expressed in us and so on. The numbers in the other rows show how many times per second, the program was       ")
 766         log.info("| interrupted for a time as specified by its bucket. '0' is printed when there are no interrupts in this bucket throughout   ")
 767         log.info("| the duration of the test. 0.00 means there were interrupts in this bucket but very few. Due to rounding this shows as 0.00 ")
 768         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 769         sys.stdout.flush()
 770         for sock_index,sock in enumerate(socks,start=0):
 771                 buckets=socks[sock_index].show_irq_buckets(1)
 772                 print('Measurement ongoing ... ',end='\r')
 773                 socks[sock_index].stop(cores[sock_index])
 774                 old_irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
 775                 irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
 776                 irq[0][0] = 'bucket us'
 777                 for j,bucket in enumerate(buckets,start=1):
 778                         irq[0][j] = '<'+ bucket
 779                 irq[0][-1] = '>'+ buckets [-2]
 780                 socks[sock_index].start(cores[sock_index])
 781                 time.sleep(2)
 782                 for j,bucket in enumerate(buckets,start=1):
 783                         for i,irqcore in enumerate(cores[sock_index],start=1):
 784                                 old_irq[i][j] = socks[sock_index].irq_stats(irqcore,j-1)
 785                 time.sleep(float(runtime))
 786                 socks[sock_index].stop(cores[sock_index])
 787                 for i,irqcore in enumerate(cores[sock_index],start=1):
 788                         irq[i][0]='core %s '%irqcore
 789                         for j,bucket in enumerate(buckets,start=1):
 790                                 diff =  socks[sock_index].irq_stats(irqcore,j-1) - old_irq[i][j]
 791                                 if diff == 0:
 792                                         irq[i][j] = '0'
 793                                 else:
 794                                         irq[i][j] = str(round(diff/float(runtime), 2))
 795                 log.info('Results for PROX instance %s'%sock_index)
 796                 for row in irq:
 797                         log.info(''.join(['{:>12}'.format(item) for item in row]))
 798
 799 def run_impairtest(gensock,sutsock):
 800         fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Dropped','DropRate']
 801         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 802         writer.writeheader()
 803         size=PACKETSIZE-4
 804         log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 805         log.info("| Generator is sending UDP ("+'{:>5}'.format(FLOWSIZE)+" flow) packets ("+ '{:>5}'.format(size+4) +" bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test    |")
 806         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 807         log.info("| Test   |  Speed requested   | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
 808         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 809         attempts = 0
 810         gensock.set_size(gencores,0,size) # This is setting the frame size
 811         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 812         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 813         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 814         source_port,destination_port = flows[FLOWSIZE]
 815         gensock.set_random(gencores,0,34,source_port,2)
 816         gensock.set_random(gencores,0,36,destination_port,2)
 817         gensock.start(latcores)
 818         speed = STARTSPEED
 819         gensock.speed(speed / len(gencores) / len(gentasks), gencores, gentasks)
 820         while True:
 821                 attempts += 1
 822                 print('Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 823                 sys.stdout.flush()
 824                 time.sleep(1)
 825                 # Get statistics now that the generation is stable and NO ARP messages any more
 826                 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg, lat_perc, lat_perc_max, lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used, r, actual_duration = run_iteration(gensock,sutsock,runtime)
 827                 drop_rate = 100.0*abs_dropped/abs_tx
 828                 if lat_used < 0.95:
 829                         lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 830                 else:
 831                         lat_warning = ''
 832                 log.info('|{:>7}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | '+ '{:>9.3f}'.format(pps_tx) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9.3f}'.format(pps_rx)+' Mpps | '+ '{:>9.0f}'.format(lat_avg)+' us   | '+ '{:>9.0f}'.format(lat_max)+' us   | '+ '{:>14d}'.format(abs_dropped)+ ' |''{:>9.2f}'.format(drop_rate)+ '%  |'+lat_warning)
 833                 writer.writerow({'Flows':FLOWSIZE,'PacketSize':(size+4),'RequestedPPS':get_pps(speed,size),'GeneratedPPS':pps_req_tx,'SentPPS':pps_tx,'ForwardedPPS':pps_sut_tx_str,'ReceivedPPS':pps_rx,'AvgLatencyUSEC':lat_avg,'MaxLatencyUSEC':lat_max,'Dropped':abs_dropped,'DropRate':drop_rate})
 834                 if PushGateway:
 835                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
 836                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nDropped {}\nDropRate {}\n'.format(FLOWSIZE,size+4,get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,drop_rate)
 837                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 838                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 839
 840 def run_warmuptest(gensock):
 841 # Running at low speed to make sure the ARP messages can get through.
 842 # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
 843 # Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
 844 # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
 845         gensock.speed(WARMUPSPEED / len(gencores) /len (gentasks), gencores, gentasks)
 846         size=PACKETSIZE-4
 847         gensock.set_size(gencores,0,size) # This is setting the frame size
 848         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 849         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 850         gensock.set_value(gencores,0,56,1,1)
 851         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 852         source_port,destination_port = flows[FLOWSIZE]
 853         gensock.set_random(gencores,0,34,source_port,2)
 854         gensock.set_random(gencores,0,36,destination_port,2)
 855         gensock.start(genstatcores)
 856         time.sleep(WARMUPTIME)
 857         gensock.stop(genstatcores)
 858         gensock.set_value(gencores,0,56,50,1)
 859         time.sleep(WARMUPTIME)
 860
 861 # To generate a desired number of flows, PROX will randomize the bits in source and destination ports, as specified by the bit masks in the flows variable.
 862 flows={\
 863 1:      ['1000000000000000','1000000000000000'],\
 864 2:      ['1000000000000000','100000000000000X'],\
 865 4:      ['100000000000000X','100000000000000X'],\
 866 8:      ['100000000000000X','10000000000000XX'],\
 867 16:     ['10000000000000XX','10000000000000XX'],\
 868 32:     ['10000000000000XX','1000000000000XXX'],\
 869 64:     ['1000000000000XXX','1000000000000XXX'],\
 870 128:    ['1000000000000XXX','100000000000XXXX'],\
 871 256:    ['100000000000XXXX','100000000000XXXX'],\
 872 512:    ['100000000000XXXX','10000000000XXXXX'],\
 873 1024:   ['10000000000XXXXX','10000000000XXXXX'],\
 874 2048:   ['10000000000XXXXX','1000000000XXXXXX'],\
 875 4096:   ['1000000000XXXXXX','1000000000XXXXXX'],\
 876 8192:   ['1000000000XXXXXX','100000000XXXXXXX'],\
 877 16384:  ['100000000XXXXXXX','100000000XXXXXXX'],\
 878 32768:  ['100000000XXXXXXX','10000000XXXXXXXX'],\
 879 65536:  ['10000000XXXXXXXX','10000000XXXXXXXX'],\
 880 131072: ['10000000XXXXXXXX','1000000XXXXXXXXX'],\
 881 262144: ['1000000XXXXXXXXX','1000000XXXXXXXXX'],\
 882 524288: ['1000000XXXXXXXXX','100000XXXXXXXXXX'],\
 883 1048576:['100000XXXXXXXXXX','100000XXXXXXXXXX'],}
 884 clients =[]
 885 socks =[]
 886 socks_control =[]
 887 vmDPIP =[]
 888 vmAdminIP =[]
 889 vmDPmac =[]
 890 hexDPIP =[]
 891 config_file =[]
 892 prox_socket =[]
 893 prox_launch_exit =[]
 894 auto_start =[]
 895 mach_type =[]
 896 sock_type =[]
 897 cores = []
 898 ports = []
 899 tasks = {}
 900 TST009_S = []
 901
 902 data_file = 'RUN{}.{}.csv'.format(env,test_file)
 903 data_csv_file = open(data_file,'w')
 904 testconfig = ConfigParser.RawConfigParser()
 905 testconfig.read(test_file)
 906 required_number_of_test_machines = testconfig.get('DEFAULT', 'total_number_of_test_machines')
 907 TestName = testconfig.get('DEFAULT', 'name')
 908 if testconfig.has_option('DEFAULT', 'PushGateway'):
 909         PushGateway = testconfig.get('DEFAULT', 'PushGateway')
 910         log.info('Measurements will be pushed to %s'%PushGateway)
 911 else:
 912         PushGateway = None
 913 if testconfig.has_option('DEFAULT', 'lat_percentile'):
 914         LAT_PERCENTILE = float(testconfig.get('DEFAULT', 'lat_percentile')) /100.0
 915 else:
 916         LAT_PERCENTILE = 0.99
 917 log.info('Latency percentile measured at {:.0f}%'.format(LAT_PERCENTILE*100))
 918 config = ConfigParser.RawConfigParser()
 919 config.read(env)
 920 machine_map = ConfigParser.RawConfigParser()
 921 machine_map.read(machine_map_file)
 922 key = config.get('ssh', 'key')
 923 user = config.get('ssh', 'user')
 924 total_number_of_machines = config.get('rapid', 'total_number_of_machines')
 925 if int(required_number_of_test_machines) > int(total_number_of_machines):
 926         log.exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
 927         raise Exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
 928 for vm in range(1, int(total_number_of_machines)+1):
 929         vmAdminIP.append(config.get('M%d'%vm, 'admin_ip'))
 930         vmDPmac.append(config.get('M%d'%vm, 'dp_mac'))
 931         vmDPIP.append(config.get('M%d'%vm, 'dp_ip'))
 932         ip = vmDPIP[-1].split('.')
 933         hexDPIP.append(hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2))
 934 machine_index = []
 935 for vm in range(1, int(required_number_of_test_machines)+1):
 936         machine_index.append(int(machine_map.get('TestM%d'%vm, 'machine_index'))-1)
 937         prox_socket.append(testconfig.getboolean('TestM%d'%vm, 'prox_socket'))
 938 for vm in range(1, int(required_number_of_test_machines)+1):
 939         if prox_socket[vm-1]:
 940                 prox_launch_exit.append(testconfig.getboolean('TestM%d'%vm, 'prox_launch_exit'))
 941                 config_file.append(testconfig.get('TestM%d'%vm, 'config_file'))
 942                 # Looking for all task definitions in the PROX cfg files. Constructing a list of all tasks used
 943                 textfile =  open (config_file[-1], 'r')
 944                 filetext = textfile.read()
 945                 textfile.close()
 946                 tasks_for_this_cfg = set(re.findall("task\s*=\s*(\d+)",filetext))
 947                 with open('{}_{}_parameters{}.lua'.format(env,test_file,vm), "w") as f:
 948                         f.write('name="%s"\n'% testconfig.get('TestM%d'%vm, 'name'))
 949                         f.write('local_ip="%s"\n'% vmDPIP[machine_index[vm-1]])
 950                         f.write('local_hex_ip="%s"\n'% hexDPIP[machine_index[vm-1]])
 951                         if testconfig.has_option('TestM%d'%vm, 'cores'):
 952                                 cores.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'cores')))
 953                                 f.write('cores="%s"\n'% ','.join(map(str, cores[-1])))
 954                         else:
 955                                 cores.append(None)
 956                         if testconfig.has_option('TestM%d'%vm, 'ports'):
 957                                 ports.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'ports')))
 958                                 f.write('ports="%s"\n'% ','.join(map(str, ports[-1])))
 959                         else:
 960                                 ports.append(None)
 961                         if re.match('(l2){0,1}gen(_bare){0,1}.*\.cfg',config_file[-1]):
 962                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
 963                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
 964                                 genstatcores = gencores + latcores
 965                                 gentasks = tasks_for_this_cfg
 966                                 auto_start.append(False)
 967                                 mach_type.append('gen')
 968                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
 969                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
 970                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 971                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 972                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 973                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 974                                 if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'):
 975                                         BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp'))
 976                                 else:
 977                                         BUCKET_SIZE_EXP = 11
 978                                 f.write('bucket_size_exp="%s"\n'% BUCKET_SIZE_EXP)
 979                                 if testconfig.has_option('TestM%d'%vm, 'heartbeat'):
 980                                         heartbeat = int(testconfig.get('TestM%d'%vm, 'heartbeat'))
 981                                 else:
 982                                         heartbeat = 60
 983                                 f.write('heartbeat="%s"\n'% heartbeat)
 984                         elif re.match('(l2){0,1}gen_gw.*\.cfg',config_file[-1]):
 985                                 if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'):
 986                                         BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp'))
 987                                 else:
 988                                         BUCKET_SIZE_EXP = 11
 989                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
 990                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
 991                                 genstatcores = gencores + latcores
 992                                 gentasks = tasks_for_this_cfg
 993                                 auto_start.append(False)
 994                                 mach_type.append('gen')
 995                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
 996                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
 997                                 gwVMindex = int(testconfig.get('TestM%d'%vm, 'gw_vm')) -1
 998                                 f.write('gw_ip="%s"\n'% vmDPIP[machine_index[gwVMindex]])
 999                                 f.write('gw_hex_ip="%s"\n'% hexDPIP[machine_index[gwVMindex]])
1000                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
1001                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
1002                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
1003                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
1004                                 if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'):
1005                                         BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp'))
1006                                 else:
1007                                         BUCKET_SIZE_EXP = 11
1008                                 f.write('bucket_size_exp="%s"\n'% BUCKET_SIZE_EXP)
1009                                 if testconfig.has_option('TestM%d'%vm, 'heartbeat'):
1010                                         heartbeat = int(testconfig.get('TestM%d'%vm, 'heartbeat'))
1011                                 else:
1012                                         heartbeat = 60
1013                                 f.write('heartbeat="%s"\n'% heartbeat)
1014                         elif re.match('(l2){0,1}swap.*\.cfg',config_file[-1]):
1015                                 sutstatcores = cores[-1]
1016                                 auto_start.append(True)
1017                                 mach_type.append('sut')
1018                         elif re.match('secgw1.*\.cfg',config_file[-1]):
1019                                 auto_start.append(True)
1020                                 mach_type.append('none')
1021                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
1022                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
1023                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
1024                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
1025                         elif re.match('secgw2.*\.cfg',config_file[-1]):
1026                                 sutstatcores = cores[-1]
1027                                 auto_start.append(True)
1028                                 mach_type.append('sut')
1029                         else:
1030                                 auto_start.append(True)
1031                                 mach_type.append('none')
1032                 f.close
1033                 tasks = tasks_for_this_cfg.union(tasks)
1034 log.debug("Tasks detected in all PROX config files %r"%tasks)
1035 #####################################################################################
1036 def exit_handler():
1037         log.debug ('exit cleanup')
1038         for index, sock in enumerate(socks):
1039                 if socks_control[index]:
1040                         sock.quit()
1041         for client in clients:
1042                 client.close()
1043         data_csv_file.close
1044         sys.exit(0)
1045
1046 atexit.register(exit_handler)
1047
1048 for vm in range(0, int(required_number_of_test_machines)):
1049         if prox_socket[vm]:
1050                 clients.append(prox_ctrl(vmAdminIP[machine_index[vm]], key,user))
1051                 connect_client(clients[-1])
1052 # Creating script to bind the right network interface to the poll mode driver
1053                 devbindfile = '{}_{}_devbindvm{}.sh'.format(env,test_file, vm+1)
1054                 with open(devbindfile, "w") as f:
1055                         newText= 'link="$(ip -o link | grep '+vmDPmac[machine_index[vm]]+' |cut -d":" -f 2)"\n'
1056                         f.write(newText)
1057                         newText= 'if [ -n "$link" ];\n'
1058                         f.write(newText)
1059                         newText= 'then\n'
1060                         f.write(newText)
1061                         newText= '        echo Need to bind\n'
1062                         f.write(newText)
1063                         newText= '        sudo ' + rundir + '/dpdk/usertools/dpdk-devbind.py --force --bind igb_uio $('+rundir+'/dpdk/usertools/dpdk-devbind.py --status |grep  $link | cut -d" " -f 1)\n'
1064                         f.write(newText)
1065                         newText= 'else\n'
1066                         f.write(newText)
1067                         newText= '       echo Assuming port is already bound to DPDK\n'
1068                         f.write(newText)
1069                         newText= 'fi\n'
1070                         f.write(newText)
1071                         newText= 'exit 0\n'
1072                         f.write(newText)
1073                 st = os.stat(devbindfile)
1074                 os.chmod(devbindfile, st.st_mode | stat.S_IEXEC)
1075                 clients[-1].scp_put('./%s'%devbindfile, rundir+'/devbind.sh')
1076                 cmd = 'sudo ' + rundir+ '/devbind.sh'
1077                 clients[-1].run_cmd(cmd)
1078                 log.debug("devbind.sh running on VM%d"%(vm+1))
1079                 clients[-1].scp_put('./%s'%config_file[vm], rundir+'/%s'%config_file[vm])
1080                 clients[-1].scp_put('./{}_{}_parameters{}.lua'.format(env,test_file, vm+1), rundir + '/parameters.lua')
1081                 if not configonly:
1082                         if prox_launch_exit[vm]:
1083                                 log.debug("Starting PROX on VM%d"%(vm+1))
1084                                 if auto_start[vm]:
1085                                         cmd = 'sudo ' +rundir + '/prox/build/prox -t -o cli -f ' + rundir + '/%s'%config_file[vm]
1086                                 else:
1087                                         cmd = 'sudo ' +rundir + '/prox/build/prox -e -t -o cli -f ' + rundir + '/%s'%config_file[vm]
1088                                 clients[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1))
1089                         socks_control.append(prox_launch_exit[vm])
1090                         socks.append(connect_socket(clients[-1]))
1091                         sock_type.append(mach_type[vm])
1092
1093 def get_BinarySearchParams() :
1094         global  DROP_RATE_TRESHOLD
1095         global  LAT_AVG_TRESHOLD
1096         global  LAT_PERC_TRESHOLD
1097         global  LAT_MAX_TRESHOLD
1098         global  ACCURACY
1099         global  STARTSPEED
1100         global  TST009
1101         global  TST009_MAXr
1102         global  TST009_MAXz
1103         DROP_RATE_TRESHOLD = float(testconfig.get('BinarySearchParams', 'drop_rate_threshold'))
1104         LAT_AVG_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_avg_threshold'))
1105         LAT_PERC_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_perc_threshold'))
1106         LAT_MAX_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_max_threshold'))
1107         ACCURACY = float(testconfig.get('BinarySearchParams', 'accuracy'))
1108         STARTSPEED = float(testconfig.get('BinarySearchParams', 'startspeed'))
1109         TST009_MAXr = 1
1110         TST009_MAXz = inf
1111         TST009 = False
1112
1113 def get_FixedRateParams() :
1114         global  DROP_RATE_TRESHOLD
1115         global  LAT_AVG_TRESHOLD
1116         global  LAT_PERC_TRESHOLD
1117         global  LAT_MAX_TRESHOLD
1118         global  flow_size_list
1119         global  packet_size_list
1120         global  STARTSPEED
1121         global  TST009
1122         global  TST009_MAXr
1123         global  TST009_MAXz
1124         DROP_RATE_TRESHOLD = inf
1125         LAT_AVG_TRESHOLD = inf
1126         LAT_PERC_TRESHOLD = inf
1127         LAT_MAX_TRESHOLD = inf
1128         TST009_MAXr = 1
1129         TST009_MAXz = inf
1130         TST009 = False
1131         packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1132         flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1133         STARTSPEED = float(testconfig.get('test%d'%test_nr, 'speed'))
1134
1135 def get_TST009SearchParams() :
1136         global  DROP_RATE_TRESHOLD
1137         global  LAT_AVG_TRESHOLD
1138         global  LAT_PERC_TRESHOLD
1139         global  LAT_MAX_TRESHOLD
1140         global  TST009
1141         global  TST009_MAXr
1142         global  TST009_MAXz
1143         global  TST009_MAXFramesAllIngress
1144         global  TST009_StepSize
1145         global  TST009_n
1146         global  TST009_L
1147         global  TST009_R
1148         global  TST009_S
1149         if testconfig.has_option('TST009SearchParams', 'drop_rate_threshold'):
1150                 DROP_RATE_TRESHOLD = float(testconfig.get('TST009SearchParams', 'drop_rate_threshold'))
1151         else:
1152                 DROP_RATE_TRESHOLD = 0
1153         LAT_AVG_TRESHOLD = inf
1154         LAT_PERC_TRESHOLD = inf
1155         LAT_MAX_TRESHOLD = inf
1156         TST009_MAXr = float(testconfig.get('TST009SearchParams', 'MAXr'))
1157         TST009_MAXz = float(testconfig.get('TST009SearchParams', 'MAXz'))
1158         TST009_MAXFramesAllIngress = int(testconfig.get('TST009SearchParams', 'MAXFramesPerSecondAllIngress'))
1159         TST009_StepSize = int(testconfig.get('TST009SearchParams', 'StepSize'))
1160         TST009_n = int(ceil(TST009_MAXFramesAllIngress / TST009_StepSize))
1161         TST009 = True
1162         TST009_L = 0
1163         TST009_R = TST009_n - 1
1164         for m in range(0, TST009_n):
1165                 TST009_S.append((m+1) * TST009_StepSize)
1166
1167 if configonly:
1168         sys.exit()
1169 ####################################################
1170 # Run test cases
1171 # Best to run the flow test at the end since otherwise the tests coming after might be influenced by the big number of entries in the switch flow tables
1172 ####################################################
1173 gensock_index = sock_type.index('gen') if 'gen' in sock_type else -1
1174 sutsock_index = sock_type.index('sut') if 'sut' in sock_type else -1
1175 number_of_tests = testconfig.get('DEFAULT', 'number_of_tests')
1176 for test_nr in range(1, int(number_of_tests)+1):
1177         test=testconfig.get('test%d'%test_nr,'test')
1178         log.info(test)
1179         if test == 'flowsizetest':
1180                 get_BinarySearchParams()
1181                 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1182                 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1183                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1184         elif test == 'TST009test':
1185                 get_TST009SearchParams()
1186                 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1187                 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1188                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1189         elif test == 'fixed_rate':
1190                 get_FixedRateParams()
1191                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1192         elif test == 'corestats':
1193                 run_core_stats(socks)
1194         elif test == 'portstats':
1195                 run_port_stats(socks)
1196         elif test == 'impairtest':
1197                 get_BinarySearchParams()
1198                 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1199                 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1200                 run_impairtest(socks[gensock_index],socks[sutsock_index])
1201         elif test == 'irqtest':
1202                 run_irqtest(socks)
1203         elif test == 'warmuptest':
1204                 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1205                 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1206                 WARMUPSPEED = int(testconfig.get('test%d'%test_nr, 'warmupspeed'))
1207                 WARMUPTIME = int(testconfig.get('test%d'%test_nr, 'warmuptime'))
1208                 run_warmuptest(socks[gensock_index])
1209 ####################################################