VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py

   1 #!/usr/bin/python
   2
   3 ##
   4 ## Copyright (c) 2010-2020 Intel Corporation
   5 ##
   6 ## Licensed under the Apache License, Version 2.0 (the "License");
   7 ## you may not use this file except in compliance with the License.
   8 ## You may obtain a copy of the License at
   9 ##
  10 ##     http://www.apache.org/licenses/LICENSE-2.0
  11 ##
  12 ## Unless required by applicable law or agreed to in writing, software
  13 ## distributed under the License is distributed on an "AS IS" BASIS,
  14 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 ## See the License for the specific language governing permissions and
  16 ## limitations under the License.
  17 ##
  18
  19 from __future__ import print_function
  20
  21 import os
  22 import stat
  23 import sys
  24 import time
  25 import subprocess
  26 import getopt
  27 import re
  28 import logging
  29 from logging.handlers import RotatingFileHandler
  30 from logging import handlers
  31 from prox_ctrl import prox_ctrl
  32 import ConfigParser
  33 import ast
  34 import atexit
  35 import csv
  36 import requests
  37 from numpy import inf
  38 from math import ceil
  39
  40 version="20.01.10"
  41 env = "rapid.env" #Default string for environment
  42 test_file = "basicrapid.test" #Default string for test
  43 machine_map_file = "machine.map" #Default string for machine map file
  44 loglevel="DEBUG" # sets log level for writing to file
  45 screenloglevel="INFO" # sets log level for writing to screen
  46 runtime=10 # time in seconds for 1 test run
  47 configonly = False # IF True, the system will upload all the necessary config fiels to the VMs, but not start PROX and the actual testing
  48 rundir = "/home/centos" # Directory where to find the tools in the machines running PROX
  49
  50 def usage():
  51         print("usage: runrapid    [--version] [-v]")
  52         print("                   [--env ENVIRONMENT_NAME]")
  53         print("                   [--test TEST_NAME]")
  54         print("                   [--map MACHINE_MAP_FILE]")
  55         print("                   [--runtime TIME_FOR_TEST]")
  56         print("                   [--configonly False|True]")
  57         print("                   [--log DEBUG|INFO|WARNING|ERROR|CRITICAL]")
  58         print("                   [-h] [--help]")
  59         print("")
  60         print("Command-line interface to runrapid")
  61         print("")
  62         print("optional arguments:")
  63         print("  -v,  --version                 Show program's version number and exit")
  64         print("  --env ENVIRONMENT_NAME         Parameters will be read from ENVIRONMENT_NAME. Default is %s."%env)
  65         print("  --test TEST_NAME               Test cases will be read from TEST_NAME. Default is %s."%test_file)
  66         print("  --map MACHINE_MAP_FILE Machine mapping will be read from MACHINE_MAP_FILE. Default is %s."%machine_map_file)
  67         print("  --runtime                      Specify time in seconds for 1 test run")
  68         print("  --configonly                   If this option is specified, only upload all config files to the VMs, do not run the tests")
  69         print("  --log                          Specify logging level for log file output, default is DEBUG")
  70         print("  --screenlog                    Specify logging level for screen output, default is INFO")
  71         print("  -h, --help                     Show help message and exit.")
  72         print("")
  73
  74 try:
  75         opts, args = getopt.getopt(sys.argv[1:], "vh", ["version","help", "env=", "test=", "map=", "runtime=","configonly","log=","screenlog="])
  76 except getopt.GetoptError as err:
  77         print("===========================================")
  78         print(str(err))
  79         print("===========================================")
  80         usage()
  81         sys.exit(2)
  82 if args:
  83         usage()
  84         sys.exit(2)
  85 for opt, arg in opts:
  86         if opt in ["-h", "--help"]:
  87                 usage()
  88                 sys.exit()
  89         if opt in ["-v", "--version"]:
  90                 print("Rapid Automated Performance Indication for Dataplane "+version)
  91                 sys.exit()
  92         if opt in ["--env"]:
  93                 env = arg
  94         if opt in ["--test"]:
  95                 test_file = arg
  96         if opt in ["--map"]:
  97                 machine_map_file = arg
  98         if opt in ["--runtime"]:
  99                 runtime = arg
 100         if opt in ["--configonly"]:
 101                 configonly = True
 102                 print('No actual runs, only uploading configuration files')
 103         if opt in ["--log"]:
 104                 loglevel = arg
 105                 print ("Log level: "+ loglevel)
 106         if opt in ["--screenlog"]:
 107                 screenloglevel = arg
 108                 print ("Screen Log level: "+ screenloglevel)
 109
 110 print ("Using '"+env+"' as name for the environment")
 111 print ("Using '"+test_file+"' for test case definition")
 112 print ("Using '"+machine_map_file+"' for machine mapping")
 113 print ("Runtime: "+ str(runtime))
 114
 115 class bcolors:
 116         HEADER = '\033[95m'
 117         OKBLUE = '\033[94m'
 118         OKGREEN = '\033[92m'
 119         WARNING = '\033[93m'
 120         FAIL = '\033[91m'
 121         ENDC = '\033[0m'
 122         BOLD = '\033[1m'
 123         UNDERLINE = '\033[4m'
 124
 125 # create formatters
 126 screen_formatter = logging.Formatter("%(message)s")
 127 file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 128
 129 # get a top-level logger,
 130 # set its log level,
 131 # BUT PREVENT IT from propagating messages to the root logger
 132 #
 133 log = logging.getLogger()
 134 numeric_level = getattr(logging, loglevel.upper(), None)
 135 if not isinstance(numeric_level, int):
 136         raise ValueError('Invalid log level: %s' % loglevel)
 137 log.setLevel(numeric_level)
 138 log.propagate = 0
 139
 140 # create a console handler
 141 # and set its log level to the command-line option
 142 #
 143 console_handler = logging.StreamHandler(sys.stdout)
 144 #console_handler.setLevel(logging.INFO)
 145 numeric_screenlevel = getattr(logging, screenloglevel.upper(), None)
 146 if not isinstance(numeric_screenlevel, int):
 147         raise ValueError('Invalid screenlog level: %s' % screenloglevel)
 148 console_handler.setLevel(numeric_screenlevel)
 149 console_handler.setFormatter(screen_formatter)
 150
 151 # create a file handler
 152 # and set its log level
 153 #
 154 log_file = 'RUN{}.{}.log'.format(env,test_file)
 155 file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10)
 156 #file_handler = log.handlers.TimedRotatingFileHandler(log_file, 'D', 1, 5)
 157 file_handler.setLevel(numeric_level)
 158 file_handler.setFormatter(file_formatter)
 159
 160 # add handlers to the logger
 161 #
 162 log.addHandler(file_handler)
 163 log.addHandler(console_handler)
 164
 165 # Check if log exists and should therefore be rolled
 166 needRoll = os.path.isfile(log_file)
 167
 168
 169 # This is a stale log, so roll it
 170 if needRoll:
 171         # Add timestamp
 172         log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime())
 173
 174         # Roll over on application start
 175         log.handlers[0].doRollover()
 176
 177 # Add timestamp
 178 log.debug('\n---------\nLog started on %s.\n---------\n' % time.asctime())
 179
 180 log.debug("runrapid.py version: "+version)
 181 #========================================================================
 182 def connect_socket(client):
 183         attempts = 1
 184         log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 185         sock = None
 186         while True:
 187                 sock = client.prox_sock()
 188                 if sock is not None:
 189                         break
 190                 attempts += 1
 191                 if attempts > 20:
 192                         log.exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 193                         raise Exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 194                 time.sleep(2)
 195                 log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 196         log.info("Connected to PROX on %s" % client.ip())
 197         return sock
 198
 199 def connect_client(client):
 200         attempts = 1
 201         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 202         while True:
 203                 try:
 204                         client.connect()
 205                         break
 206                 except RuntimeWarning, ex:
 207                         attempts += 1
 208                         if attempts > 20:
 209                                 log.exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 210                                 raise Exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 211                         time.sleep(2)
 212                         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 213         log.debug("Connected to VM on %s" % client.ip())
 214
 215 def report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_max,tx,rx,tot_drop,elapsed_time,speed_prefix='',lat_avg_prefix='',lat_perc_prefix='',lat_max_prefix='',abs_drop_rate_prefix='',drop_rate_prefix=''):
 216         if pps_req_tx == None:
 217                 pps_req_tx_str = '{0: >14}'.format('   NA     |')
 218         else:
 219                 pps_req_tx_str = '{:>7.3f} Mpps |'.format(pps_req_tx)
 220         if pps_tx == None:
 221                 pps_tx_str = '{0: >14}'.format('   NA     |')
 222         else:
 223                 pps_tx_str = '{:>7.3f} Mpps |'.format(pps_tx)
 224         if pps_sut_tx == None:
 225                 pps_sut_tx_str = '{0: >14}'.format('   NA     |')
 226         else:
 227                 pps_sut_tx_str = '{:>7.3f} Mpps |'.format(pps_sut_tx)
 228         if pps_rx == None:
 229                 pps_rx_str = '{0: >24}'.format('NA        ')
 230         else:
 231                 pps_rx_str = bcolors.OKBLUE + '{:>4.1f} Gb/s |{:7.3f} Mpps '.format(get_speed(pps_rx,size),pps_rx) + bcolors.ENDC
 232         if tot_drop == None:
 233                 tot_drop_str = ' |       NA  | '
 234         else:
 235                 tot_drop_str = ' | {:>9.0f} | '.format(tot_drop)
 236         if lat_perc == None:
 237                 lat_perc_str = ' |{:^10.10}|'.format('NA')
 238         elif lat_perc == 128:
 239                 lat_perc_str = ' |{:^10.10}|'.format('VERY HIGH')
 240         else:
 241                 lat_perc_str = ' | {}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC)
 242         if elapsed_time == None:
 243                 elapsed_time_str = ' NA |'
 244         else:
 245                 elapsed_time_str = '{:>3.0f} |'.format(elapsed_time)
 246         return('|{:>7}'.format(flow_number)+' |' + '{:>5.1f}'.format(speed) + '% '+speed_prefix +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+ pps_req_tx_str + pps_tx_str + bcolors.ENDC + pps_sut_tx_str + pps_rx_str +lat_avg_prefix+ '| {:>5.0f}'.format(lat_avg)+' us'+lat_perc_str+lat_max_prefix+'{:>6.0f}'.format(lat_max)+' us | ' + '{:>9.0f}'.format(tx) + ' | {:>9.0f}'.format(rx) + ' | '+ abs_drop_rate_prefix+ '{:>9.0f}'.format(tx-rx) + tot_drop_str +drop_rate_prefix+ '{:>5.2f}'.format(float(tx-rx)/tx)  +bcolors.ENDC+' |' + elapsed_time_str)
 247
 248 def run_iteration(gensock, sutsock, requested_duration,flow_number,size,speed):
 249         r = 0;
 250         sleep_time = 2
 251         while (r < TST009_MAXr):
 252                 time.sleep(sleep_time)
 253                 # Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
 254                 t1_rx, t1_non_dp_rx, t1_tx, t1_non_dp_tx, t1_drop, t1_tx_fail, t1_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
 255                 t1_dp_rx = t1_rx - t1_non_dp_rx
 256                 t1_dp_tx = t1_tx - t1_non_dp_tx
 257                 gensock.start(gencores)
 258                 time.sleep(2) ## Needs to be 2 seconds since this the time that PROX uses to refresh the stats. Note that this can be changed in PROX!! Don't do it.
 259                 if sutsock!='none':
 260                         t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 261                         ##t2_sut_rx = t2_sut_rx - t2_sut_non_dp_rx
 262                         ##t2_sut_tx = t2_sut_tx - t2_sut_non_dp_tx
 263                 t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 264                 tx = t2_tx - t1_tx
 265                 dp_tx =  tx - (t2_non_dp_tx - t1_non_dp_tx )
 266                 dp_rx =  t2_rx - t1_rx - (t2_non_dp_rx - t1_non_dp_rx)
 267                 tot_dp_drop = dp_tx - dp_rx
 268                 if tx == 0:
 269                         log.critical("TX = 0. Test interrupted since no packet has been sent.")
 270                         raise Exception("TX = 0")
 271                 if dp_tx == 0:
 272                         log.critical("Only non-dataplane packets (e.g. ARP) sent. Test interrupted since no packet has been sent.")
 273                         raise Exception("Only non-dataplane packets (e.g. ARP) sent")
 274                 # Ask PROX to calibrate the bucket size once we have a PROX function to do this.
 275                 # Measure latency statistics per second
 276                 lat_min, lat_max, lat_avg, used_avg, t2_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
 277                 lat_samples = sum(buckets)
 278                 sample_count = 0
 279                 for sample_percentile, bucket in enumerate(buckets,start=1):
 280                         sample_count += bucket
 281                         if sample_count > (lat_samples * LAT_PERCENTILE):
 282                                 break
 283                 if test == 'fixed_rate':
 284                         log.info(report_result(flow_number,size,speed,None,None,None,None,lat_avg,sample_percentile,lat_max, dp_tx, dp_rx , None, None))
 285                 tot_rx = tot_non_dp_rx = tot_tx = tot_non_dp_tx = tot_drop = 0
 286                 lat_avg = used_avg = 0
 287                 buckets_total = [0] * 128
 288                 tot_lat_samples = 0
 289                 tot_lat_measurement_duration = float(0)
 290                 tot_core_measurement_duration = float(0)
 291                 tot_sut_core_measurement_duration = float(0)
 292                 tot_sut_rx = tot_sut_non_dp_rx = tot_sut_tx = tot_sut_non_dp_tx = tot_sut_drop = tot_sut_tx_fail = tot_sut_tsc = 0
 293                 lat_avail = core_avail = sut_avail = False
 294                 while (tot_core_measurement_duration - float(requested_duration) <= 0.1) or (tot_sut_core_measurement_duration - float(requested_duration) <= 0.1) or (tot_lat_measurement_duration - float(requested_duration) <= 0.1):
 295                         time.sleep(0.5)
 296                         lat_min_sample, lat_max_sample, lat_avg_sample, used_sample, t3_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
 297                         single_lat_measurement_duration = (t3_lat_tsc - t2_lat_tsc) * 1.0 / lat_hz  # time difference between the 2 measurements, expressed in seconds.
 298                         # Get statistics after some execution time
 299                         if single_lat_measurement_duration != 0:
 300                                 # A second has passed in between to lat_stats requests. Hence we need to process the results
 301                                 tot_lat_measurement_duration = tot_lat_measurement_duration + single_lat_measurement_duration
 302                                 if lat_min > lat_min_sample:
 303                                         lat_min = lat_min_sample
 304                                 if lat_max < lat_max_sample:
 305                                         lat_max = lat_max_sample
 306                                 lat_avg = lat_avg + lat_avg_sample * single_lat_measurement_duration # Sometimes, There is more than 1 second between 2 lat_stats. Hence we will take the latest measurement
 307                                 used_avg = used_avg + used_sample * single_lat_measurement_duration  # and give it more weigth.
 308                                 lat_samples = sum(buckets)
 309                                 tot_lat_samples += lat_samples
 310                                 sample_count = 0
 311                                 for sample_percentile, bucket in enumerate(buckets,start=1):
 312                                         sample_count += bucket
 313                                         if sample_count > lat_samples * LAT_PERCENTILE:
 314                                                 break
 315                                 buckets_total = [buckets_total[i] + buckets[i] for i in range(len(buckets_total))]
 316                                 t2_lat_tsc = t3_lat_tsc
 317                                 lat_avail = True
 318                         t3_rx, t3_non_dp_rx, t3_tx, t3_non_dp_tx, t3_drop, t3_tx_fail, t3_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 319                         single_core_measurement_duration = (t3_tsc - t2_tsc) * 1.0 / tsc_hz  # time difference between the 2 measurements, expressed in seconds.
 320                         if single_core_measurement_duration!= 0:
 321                                 stored_single_core_measurement_duration = single_core_measurement_duration
 322                                 tot_core_measurement_duration = tot_core_measurement_duration + single_core_measurement_duration
 323                                 delta_rx = t3_rx - t2_rx
 324                                 tot_rx += delta_rx
 325                                 delta_non_dp_rx = t3_non_dp_rx - t2_non_dp_rx
 326                                 tot_non_dp_rx += delta_non_dp_rx
 327                                 delta_tx = t3_tx - t2_tx
 328                                 tot_tx += delta_tx
 329                                 delta_non_dp_tx = t3_non_dp_tx - t2_non_dp_tx
 330                                 tot_non_dp_tx += delta_non_dp_tx
 331                                 delta_dp_tx = delta_tx -delta_non_dp_tx
 332                                 delta_dp_rx = delta_rx -delta_non_dp_rx
 333                                 delta_dp_drop = delta_dp_tx - delta_dp_rx
 334                                 tot_dp_drop += delta_dp_drop
 335                                 delta_drop = t3_drop - t2_drop
 336                                 tot_drop += delta_drop
 337                                 t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc = t3_rx, t3_non_dp_rx, t3_tx, t3_non_dp_tx, t3_drop, t3_tx_fail, t3_tsc
 338                                 core_avail = True
 339                         if sutsock!='none':
 340                                 t3_sut_rx, t3_sut_non_dp_rx, t3_sut_tx, t3_sut_non_dp_tx, t3_sut_drop, t3_sut_tx_fail, t3_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 341                                 single_sut_core_measurement_duration = (t3_sut_tsc - t2_sut_tsc) * 1.0 / tsc_hz  # time difference between the 2 measurements, expressed in seconds.
 342                                 if single_sut_core_measurement_duration!= 0:
 343                                         stored_single_sut_core_measurement_duration = single_sut_core_measurement_duration
 344                                         tot_sut_core_measurement_duration = tot_sut_core_measurement_duration + single_sut_core_measurement_duration
 345                                         tot_sut_rx += t3_sut_rx - t2_sut_rx
 346                                         tot_sut_non_dp_rx += t3_sut_non_dp_rx - t2_sut_non_dp_rx
 347                                         delta_sut_tx = t3_sut_tx - t2_sut_tx
 348                                         tot_sut_tx += delta_sut_tx
 349                                         delta_sut_non_dp_tx = t3_sut_non_dp_tx - t2_sut_non_dp_tx
 350                                         tot_sut_non_dp_tx += delta_sut_non_dp_tx
 351                                         t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc = t3_sut_rx, t3_sut_non_dp_rx, t3_sut_tx, t3_sut_non_dp_tx, t3_sut_drop, t3_sut_tx_fail, t3_sut_tsc
 352                                         sut_avail = True
 353                         if test == 'fixed_rate':
 354                                 if lat_avail == core_avail == sut_avail == True:
 355                                         lat_avail = core_avail = sut_avail = False
 356                                         pps_req_tx = (delta_tx + delta_drop - delta_rx)/stored_single_core_measurement_duration/1000000
 357                                         pps_tx = delta_tx/stored_single_core_measurement_duration/1000000
 358                                         if sutsock!='none':
 359                                                 pps_sut_tx = delta_sut_tx/stored_single_sut_core_measurement_duration/1000000
 360                                         else:
 361                                                 pps_sut_tx = None
 362                                         pps_rx = delta_rx/stored_single_core_measurement_duration/1000000
 363                                         log.info(report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg_sample,sample_percentile,lat_max_sample,delta_dp_tx,delta_dp_rx,tot_dp_drop,stored_single_core_measurement_duration))
 364                 #Stop generating
 365                 gensock.stop(gencores)
 366                 r += 1
 367                 lat_avg = lat_avg / float(tot_lat_measurement_duration)
 368                 used_avg = used_avg / float(tot_lat_measurement_duration)
 369                 t4_tsc = t2_tsc
 370                 while t4_tsc == t2_tsc:
 371                         t4_rx, t4_non_dp_rx, t4_tx, t4_non_dp_tx, t4_drop, t4_tx_fail, t4_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
 372                 if test == 'fixed_rate':
 373                         t4_lat_tsc = t2_lat_tsc
 374                         while t4_lat_tsc == t2_lat_tsc:
 375                                 lat_min_sample, lat_max_sample, lat_avg_sample, used_sample, t4_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
 376                         sample_count = 0
 377                         lat_samples = sum(buckets)
 378                         for percentile, bucket in enumerate(buckets,start=1):
 379                                 sample_count += bucket
 380                                 if sample_count > lat_samples * LAT_PERCENTILE:
 381                                         break
 382                         lat_max = lat_max_sample
 383                         lat_avg = lat_avg_sample
 384                         delta_rx = t4_rx - t2_rx
 385                         delta_non_dp_rx = t4_non_dp_rx - t2_non_dp_rx
 386                         delta_tx = t4_tx - t2_tx
 387                         delta_non_dp_tx = t4_non_dp_tx - t2_non_dp_tx
 388                         delta_dp_tx = delta_tx -delta_non_dp_tx
 389                         delta_dp_rx = delta_rx -delta_non_dp_rx
 390                         dp_tx = delta_dp_tx
 391                         dp_rx = delta_dp_rx
 392                         tot_dp_drop += delta_dp_tx - delta_dp_rx
 393                         pps_req_tx = None
 394                         pps_tx = None
 395                         pps_sut_tx = None
 396                         pps_rx = None
 397                         drop_rate = 100.0*(dp_tx-dp_rx)/dp_tx
 398                         tot_core_measurement_duration = None
 399                         break ## Not really needed since the while loop will stop when evaluating the value of r
 400                 else:
 401                         sample_count = 0
 402                         for percentile, bucket in enumerate(buckets_total,start=1):
 403                                 sample_count += bucket
 404                                 if sample_count > tot_lat_samples * LAT_PERCENTILE:
 405                                         break
 406                         pps_req_tx = (tot_tx + tot_drop - tot_rx)/tot_core_measurement_duration/1000000.0 # tot_drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 407                         pps_tx = tot_tx/tot_core_measurement_duration/1000000.0 # tot_tx is all generated packets actually accepted by the interface
 408                         pps_rx = tot_rx/tot_core_measurement_duration/1000000.0 # tot_rx is all packets received by the nop task = all packets received in the gen VM
 409                         if sutsock!='none':
 410                                 pps_sut_tx = tot_sut_tx / tot_sut_core_measurement_duration / 1000000.0
 411                         else:
 412                                 pps_sut_tx = None
 413                         dp_tx = (t4_tx - t1_tx) - (t4_non_dp_tx - t1_non_dp_tx)
 414                         dp_rx = (t4_rx - t1_rx) - (t4_non_dp_rx - t1_non_dp_rx)
 415                         tot_dp_drop = dp_tx - dp_rx
 416                         drop_rate = 100.0*tot_dp_drop/dp_tx
 417                         if ((drop_rate < DROP_RATE_TRESHOLD) or (tot_dp_drop == DROP_RATE_TRESHOLD ==0) or (tot_dp_drop > TST009_MAXz)):
 418                                 break
 419         return(pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,percentile,lat_max,dp_tx,dp_rx,tot_dp_drop,(t4_tx_fail - t1_tx_fail),drop_rate,lat_min,used_avg,r,tot_core_measurement_duration)
 420
 421 def new_speed(speed,size,success):
 422         if test == 'fixed_rate':
 423                 return (STARTSPEED)
 424         elif TST009:
 425                 global TST009_m
 426                 global TST009_L
 427                 global TST009_R
 428                 if success:
 429                         TST009_L = TST009_m + 1
 430                 else:
 431                         TST009_R = max(TST009_m - 1, TST009_L)
 432                 TST009_m = int ((TST009_L + TST009_R)/2)
 433                 return (get_percentageof10Gbs(TST009_S[TST009_m],size))
 434         else:
 435                 global minspeed
 436                 global maxspeed
 437                 if success:
 438                         minspeed = speed
 439                 else:
 440                         maxspeed = speed
 441                 return ((minspeed + maxspeed)/2.0)
 442
 443 def get_start_speed_and_init(size):
 444         if test == 'fixed_rate':
 445                 return (STARTSPEED)
 446         elif TST009:
 447                 global TST009_L
 448                 global TST009_R
 449                 global TST009_m
 450                 TST009_L = 0
 451                 TST009_R = TST009_n - 1
 452                 TST009_m = int((TST009_L + TST009_R) / 2)
 453                 return (get_percentageof10Gbs(TST009_S[TST009_m],size))
 454         else:
 455                 global minspeed
 456                 global maxspeed
 457                 minspeed = 0
 458                 maxspeed = STARTSPEED
 459                 return (STARTSPEED)
 460
 461 def resolution_achieved():
 462         if test == 'fixed_rate':
 463                 return (True)
 464         elif TST009:
 465                 return (TST009_L == TST009_R)
 466         else:
 467                 return ((maxspeed - minspeed) <= ACCURACY)
 468
 469 def get_percentageof10Gbs(pps_speed,size):
 470         # speed is given in pps, returning % of 10Gb/s
 471         return (pps_speed / 1000000.0 * 0.08 * (size+24))
 472
 473 def get_pps(speed,size):
 474         # speed is given in % of 10Gb/s, returning Mpps
 475         return (speed * 100.0 / (8*(size+24)))
 476
 477 def get_speed(packet_speed,size):
 478         # return speed in Gb/s
 479         return (packet_speed / 1000.0 * (8*(size+24)))
 480
 481 def run_flow_size_test(gensock,sutsock):
 482         global fieldnames
 483         global writer
 484         #fieldnames = ['Flows','PacketSize','Gbps','Mpps','AvgLatency','MaxLatency','PacketsDropped','PacketDropRate']
 485         fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Sent','Received','Lost','LostTotal']
 486         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 487         writer.writeheader()
 488         gensock.start(latcores)
 489         for size in packet_size_list:
 490                 size = size-4
 491                 gensock.set_size(gencores,0,size) # This is setting the frame size
 492                 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 493                 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 494                 # This will only work when using sending UDP packets. For different protocls and ethernet types, we would need a different calculation
 495                 log.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 496                 log.info("| UDP, "+ '{:>5}'.format(size+4) +" bytes, different number of flows by randomizing SRC & DST UDP port                                                                                                              |")
 497                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 498                 log.info("| Flows  | Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core           | Avg. Lat.|" + '{:.0f} '.format(LAT_PERCENTILE*100) +"Pcentil| Max. Lat.|   Sent    |  Received |    Lost   | Total Lost|L.Ratio|Time|")
 499                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 500                 for flow_number in flow_size_list:
 501                         attempts = 0
 502                         gensock.reset_stats()
 503                         if sutsock!='none':
 504                                 sutsock.reset_stats()
 505                         source_port,destination_port = flows[flow_number]
 506                         gensock.set_random(gencores,0,34,source_port,2)
 507                         gensock.set_random(gencores,0,36,destination_port,2)
 508                         endspeed = None
 509                         speed = get_start_speed_and_init(size)
 510                         while True:
 511                                 attempts += 1
 512                                 endwarning = False
 513                                 print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 514                                 sys.stdout.flush()
 515                                 # Start generating packets at requested speed (in % of a 10Gb/s link)
 516                                 gensock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
 517                                 ##time.sleep(1)
 518                                 # Get statistics now that the generation is stable and initial ARP messages are dealt with
 519                                 pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc ,lat_max, abs_tx,abs_rx,abs_dropped, abs_tx_fail, drop_rate, lat_min, lat_used, r, actual_duration = run_iteration(gensock,sutsock,float(runtime),flow_number,size,speed)
 520                                 if r > 1:
 521                                         retry_warning = bcolors.WARNING + ' {:1} retries needed'.format(r) +  bcolors.ENDC
 522                                 else:
 523                                         retry_warning = ''
 524                                 # Drop rate is expressed in percentage. lat_used is a ratio (0 to 1). The sum of these 2 should be 100%.
 525                                 # If the some is lower than 95, it means that more than 5% of the latency measurements where dropped for accuracy reasons.
 526                                 if (drop_rate + lat_used * 100) < 95:
 527                                         lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 528                                 else:
 529                                         lat_warning = ''
 530                                 if test == 'fixed_rate':
 531                                         endspeed = speed
 532                                         endpps_req_tx = None
 533                                         endpps_tx = None
 534                                         endpps_sut_tx = None
 535                                         endpps_rx = None
 536                                         endlat_avg = lat_avg
 537                                         endlat_perc = lat_perc
 538                                         endlat_max = lat_max
 539                                         endabs_dropped = abs_dropped
 540                                         enddrop_rate = drop_rate
 541                                         endabs_tx = abs_tx
 542                                         endabs_rx = abs_rx
 543                                         success = True
 544                                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 545                                 # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold
 546                                 # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
 547                                 # This can be specified by putting 0 in the .test file
 548                                 elif ((drop_rate < DROP_RATE_TRESHOLD) or (abs_dropped==DROP_RATE_TRESHOLD ==0)) and (lat_avg< LAT_AVG_TRESHOLD) and (lat_perc< LAT_PERC_TRESHOLD) and (lat_max < LAT_MAX_TRESHOLD):
 549                                         lat_avg_prefix = bcolors.ENDC
 550                                         lat_perc_prefix = bcolors.ENDC
 551                                         lat_max_prefix = bcolors.ENDC
 552                                         abs_drop_rate_prefix = bcolors.ENDC
 553                                         drop_rate_prefix = bcolors.ENDC
 554                                         if ((get_pps(speed,size) - pps_tx)/get_pps(speed,size))>0.01:
 555                                                 speed_prefix = bcolors.WARNING
 556                                                 if abs_tx_fail > 0:
 557                                                         gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
 558                                                 else:
 559                                                         gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(get_pps(speed,size), pps_tx) + bcolors.ENDC
 560                                         else:
 561                                                 speed_prefix = bcolors.ENDC
 562                                                 gen_warning = ''
 563                                         endspeed = speed
 564                                         endspeed_prefix = speed_prefix
 565                                         endpps_req_tx = pps_req_tx
 566                                         endpps_tx = pps_tx
 567                                         endpps_sut_tx = pps_sut_tx
 568                                         endpps_rx = pps_rx
 569                                         endlat_avg = lat_avg
 570                                         endlat_perc = lat_perc
 571                                         endlat_max = lat_max
 572                                         endabs_dropped = None
 573                                         enddrop_rate = drop_rate
 574                                         endabs_tx = abs_tx
 575                                         endabs_rx = abs_rx
 576                                         if lat_warning or gen_warning or retry_warning:
 577                                                 endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning + gen_warning)
 578                                         success = True
 579                                         success_message=' SUCCESS'
 580                                         log.debug(report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
 581                                 else:
 582                                         success_message=' FAILED'
 583                                         gen_warning = ''
 584                                         abs_drop_rate_prefix = bcolors.ENDC
 585                                         if ((abs_dropped>0) and (DROP_RATE_TRESHOLD ==0)):
 586                                                 abs_drop_rate_prefix = bcolors.FAIL
 587                                         if (drop_rate < DROP_RATE_TRESHOLD):
 588                                                 drop_rate_prefix = bcolors.ENDC
 589                                         else:
 590                                                 drop_rate_prefix = bcolors.FAIL
 591                                         if (lat_avg< LAT_AVG_TRESHOLD):
 592                                                 lat_avg_prefix = bcolors.ENDC
 593                                         else:
 594                                                 lat_avg_prefix = bcolors.FAIL
 595                                         if (lat_perc< LAT_PERC_TRESHOLD):
 596                                                 lat_perc_prefix = bcolors.ENDC
 597                                         else:
 598                                                 lat_perc_prefix = bcolors.FAIL
 599                                         if (lat_max< LAT_MAX_TRESHOLD):
 600                                                 lat_max_prefix = bcolors.ENDC
 601                                         else:
 602                                                 lat_max_prefix = bcolors.FAIL
 603                                         if (((get_pps(speed,size) - pps_tx)/get_pps(speed,size))<0.001):
 604                                                 speed_prefix = bcolors.ENDC
 605                                         else:
 606                                                 speed_prefix = bcolors.FAIL
 607                                         success = False
 608                                         log.debug(report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
 609                                 speed = new_speed(speed, size, success)
 610                                 if resolution_achieved():
 611                                         break
 612                         if endspeed !=  None:
 613                                 log.info(report_result(flow_number,size,endspeed,endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_perc,endlat_max,endabs_tx,endabs_rx,endabs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix))
 614                                 if endwarning:
 615                                         log.info (endwarning)
 616                                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 617                                 writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':get_pps(endspeed,size),'GeneratedPPS':endpps_req_tx,'SentPPS':endpps_tx,'ForwardedPPS':endpps_sut_tx,'ReceivedPPS':endpps_rx,'AvgLatencyUSEC':endlat_avg,'MaxLatencyUSEC':endlat_max,'Sent':endabs_tx,'Received':endabs_rx,'Lost':endabs_dropped,'LostTotal':endabs_dropped})
 618                                 if PushGateway:
 619                                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
 620                                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nSent {}\nReceived {}\nLost {}\nLostTotal {}\n'.format(flow_number,size+4,get_pps(endspeed,size),endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_max,endabs_tx,endabs_rx,endabs_Dropped,endabs_dropped)
 621                                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 622                                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 623                         else:
 624                                 log.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
 625         gensock.stop(latcores)
 626
 627 def run_core_stats(socks):
 628         fieldnames = ['PROXID','Time','Received','Sent','NonDPReceived','NonDPSent','Delta','NonDPDelta','Dropped']
 629         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 630         writer.writeheader()
 631         log.info("+------------------------------------------------------------------------------------------------------------------+")
 632         log.info("| Measuring core statistics on 1 or more PROX instances                                                            |")
 633         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 634         log.info("| PROX ID   |    Time   |    RX      |     TX     | non DP RX  | non DP TX  |   TX - RX  | nonDP TX-RX|  DROP TOT  |")
 635         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 636         for sock in socks:
 637                 sock.reset_stats()
 638         duration = float(runtime)
 639         tot_drop = []
 640         old_rx = []; old_non_dp_rx = []; old_tx = []; old_non_dp_tx = []; old_drop = []; old_tx_fail = []; old_tsc = []
 641         new_rx = []; new_non_dp_rx = []; new_tx = []; new_non_dp_tx = []; new_drop = []; new_tx_fail = []; new_tsc = []
 642         sockets_to_go = len (socks)
 643         for i,sock in enumerate(socks,start=0):
 644                 tot_drop.append(0)
 645                 old_rx.append(0); old_non_dp_rx.append(0); old_tx.append(0); old_non_dp_tx.append(0); old_drop.append(0); old_tx_fail.append(0); old_tsc.append(0)
 646                 old_rx[-1], old_non_dp_rx[-1], old_tx[-1], old_non_dp_tx[-1], old_drop[-1], old_tx_fail[-1], old_tsc[-1], tsc_hz = sock.core_stats(cores[i],tasks)
 647                 new_rx.append(0); new_non_dp_rx.append(0); new_tx.append(0); new_non_dp_tx.append(0); new_drop.append(0); new_tx_fail.append(0); new_tsc.append(0)
 648         while (duration > 0):
 649                 time.sleep(0.5)
 650                 # Get statistics after some execution time
 651                 for i,sock in enumerate(socks,start=0):
 652                         new_rx[i], new_non_dp_rx[i], new_tx[i], new_non_dp_tx[i], new_drop[i], new_tx_fail[i], new_tsc[i], tsc_hz = sock.core_stats(cores[i],tasks)
 653                         drop = new_drop[i]-old_drop[i]
 654                         rx = new_rx[i] - old_rx[i]
 655                         tx = new_tx[i] - old_tx[i]
 656                         non_dp_rx = new_non_dp_rx[i] - old_non_dp_rx[i]
 657                         non_dp_tx = new_non_dp_tx[i] - old_non_dp_tx[i]
 658                         tsc = new_tsc[i] - old_tsc[i]
 659                         if tsc == 0 :
 660                                 continue
 661                         sockets_to_go -= 1
 662                         old_drop[i] = new_drop[i]
 663                         old_rx[i] = new_rx[i]
 664                         old_tx[i] = new_tx[i]
 665                         old_non_dp_rx[i] = new_non_dp_rx[i]
 666                         old_non_dp_tx[i] = new_non_dp_tx[i]
 667                         old_tsc[i] = new_tsc[i]
 668                         tot_drop[i] = tot_drop[i] + tx - rx
 669                         log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(non_dp_rx)+' | '+'{:>10.0f}'.format(non_dp_tx)+' | ' + '{:>10.0f}'.format(tx-rx) + ' | '+ '{:>10.0f}'.format(non_dp_tx-non_dp_rx) + ' | '+'{:>10.0f}'.format(tot_drop[i]) +' |')
 670                         writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NonDPReceived':non_dp_rx,'NonDPSent':non_dp_tx,'Delta':tx-rx,'NonDPDelta':non_dp_tx-non_dp_rx,'Dropped':tot_drop[i]})
 671                         if PushGateway:
 672                                 URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
 673                                 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNonDPReceived {}\nNonDPSent {}\nDelta {}\nNonDPDelta {}\nDropped {}\n'.format(i,duration,rx,tx,non_dp_rx,non_dp_tx,tx-rx,non_dp_tx-non_dp_rx,tot_drop[i])
 674                                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 675                                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
 676                         if sockets_to_go == 0:
 677                                 duration = duration - 1
 678                                 sockets_to_go = len (socks)
 679         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 680
 681 def run_port_stats(socks):
 682         fieldnames = ['PROXID','Time','Received','Sent','NoMbufs','iErrMiss']
 683         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 684         writer.writeheader()
 685         log.info("+---------------------------------------------------------------------------+")
 686         log.info("| Measuring port statistics on 1 or more PROX instances                     |")
 687         log.info("+-----------+-----------+------------+------------+------------+------------+")
 688         log.info("| PROX ID   |    Time   |    RX      |     TX     | no MBUFS   | ierr&imiss |")
 689         log.info("+-----------+-----------+------------+------------+------------+------------+")
 690         for sock in socks:
 691                 sock.reset_stats()
 692         duration = float(runtime)
 693         old_rx = []; old_tx = []; old_no_mbufs = []; old_errors = []; old_tsc = []
 694         new_rx = []; new_tx = []; new_no_mbufs = []; new_errors = []; new_tsc = []
 695         sockets_to_go = len (socks)
 696         for i,sock in enumerate(socks,start=0):
 697                 old_rx.append(0); old_tx.append(0); old_no_mbufs.append(0); old_errors.append(0); old_tsc.append(0)
 698                 old_rx[-1], old_tx[-1], old_no_mbufs[-1], old_errors[-1], old_tsc[-1] = sock.multi_port_stats(ports[i])
 699                 new_rx.append(0); new_tx.append(0); new_no_mbufs.append(0); new_errors.append(0); new_tsc.append(0)
 700         while (duration > 0):
 701                 time.sleep(0.5)
 702                 # Get statistics after some execution time
 703                 for i,sock in enumerate(socks,start=0):
 704                         new_rx[i], new_tx[i], new_no_mbufs[i], new_errors[i], new_tsc[i] = sock.multi_port_stats(ports[i])
 705                         rx = new_rx[i] - old_rx[i]
 706                         tx = new_tx[i] - old_tx[i]
 707                         no_mbufs = new_no_mbufs[i] - old_no_mbufs[i]
 708                         errors = new_errors[i] - old_errors[i]
 709                         tsc = new_tsc[i] - old_tsc[i]
 710                         if tsc == 0 :
 711                                 continue
 712                         sockets_to_go -= 1
 713                         old_rx[i] = new_rx[i]
 714                         old_tx[i] = new_tx[i]
 715                         old_no_mbufs[i] = new_no_mbufs[i]
 716                         old_errors[i] = new_errors[i]
 717                         old_tsc[i] = new_tsc[i]
 718                         log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(no_mbufs)+' | '+'{:>10.0f}'.format(errors)+' |')
 719                         writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NoMbufs':no_mbufs,'iErrMiss':errors})
 720                         if PushGateway:
 721                                 URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
 722                                 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNoMbufs {}\niErrMiss {}\n'.format(i,duration,rx,tx,no_mbufs,errors)
 723                                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 724                                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
 725                         if sockets_to_go == 0:
 726                                 duration = duration - 1
 727                                 sockets_to_go = len (socks)
 728         log.info("+-----------+-----------+------------+------------+------------+------------+")
 729
 730 def run_irqtest(socks):
 731         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 732         log.info("| Measuring time probably spent dealing with an interrupt. Interrupting DPDK cores for more than 50us might be problematic   ")
 733         log.info("| and result in packet loss. The first row shows the interrupted time buckets: first number is the bucket between 0us and    ")
 734         log.info("| that number expressed in us and so on. The numbers in the other rows show how many times per second, the program was       ")
 735         log.info("| interrupted for a time as specified by its bucket. '0' is printed when there are no interrupts in this bucket throughout   ")
 736         log.info("| the duration of the test. 0.00 means there were interrupts in this bucket but very few. Due to rounding this shows as 0.00 ")
 737         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 738         sys.stdout.flush()
 739         for sock_index,sock in enumerate(socks,start=0):
 740                 buckets=socks[sock_index].show_irq_buckets(1)
 741                 print('Measurement ongoing ... ',end='\r')
 742                 socks[sock_index].stop(cores[sock_index])
 743                 old_irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
 744                 irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
 745                 irq[0][0] = 'bucket us'
 746                 for j,bucket in enumerate(buckets,start=1):
 747                         irq[0][j] = '<'+ bucket
 748                 irq[0][-1] = '>'+ buckets [-2]
 749                 socks[sock_index].start(cores[sock_index])
 750                 time.sleep(2)
 751                 for j,bucket in enumerate(buckets,start=1):
 752                         for i,irqcore in enumerate(cores[sock_index],start=1):
 753                                 old_irq[i][j] = socks[sock_index].irq_stats(irqcore,j-1)
 754                 time.sleep(float(runtime))
 755                 socks[sock_index].stop(cores[sock_index])
 756                 for i,irqcore in enumerate(cores[sock_index],start=1):
 757                         irq[i][0]='core %s '%irqcore
 758                         for j,bucket in enumerate(buckets,start=1):
 759                                 diff =  socks[sock_index].irq_stats(irqcore,j-1) - old_irq[i][j]
 760                                 if diff == 0:
 761                                         irq[i][j] = '0'
 762                                 else:
 763                                         irq[i][j] = str(round(diff/float(runtime), 2))
 764                 log.info('Results for PROX instance %s'%sock_index)
 765                 for row in irq:
 766                         log.info(''.join(['{:>12}'.format(item) for item in row]))
 767
 768 def run_impairtest(gensock,sutsock):
 769         fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Dropped','DropRate']
 770         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 771         writer.writeheader()
 772         size=PACKETSIZE-4
 773         log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 774         log.info("| Generator is sending UDP ("+'{:>5}'.format(FLOWSIZE)+" flow) packets ("+ '{:>5}'.format(size+4) +" bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test    |")
 775         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 776         log.info("| Test   |  Speed requested   | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
 777         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 778         attempts = 0
 779         gensock.set_size(gencores,0,size) # This is setting the frame size
 780         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 781         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 782         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 783         source_port,destination_port = flows[FLOWSIZE]
 784         gensock.set_random(gencores,0,34,source_port,2)
 785         gensock.set_random(gencores,0,36,destination_port,2)
 786         gensock.start(latcores)
 787         speed = STARTSPEED
 788         gensock.speed(speed / len(gencores) / len(gentasks), gencores, gentasks)
 789         while True:
 790                 attempts += 1
 791                 print('Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 792                 sys.stdout.flush()
 793                 time.sleep(1)
 794                 # Get statistics now that the generation is stable and NO ARP messages any more
 795                 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg, lat_perc, lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used, r, actual_duration = run_iteration(gensock,sutsock,runtime)
 796                 drop_rate = 100.0*abs_dropped/abs_tx
 797                 if lat_used < 0.95:
 798                         lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 799                 else:
 800                         lat_warning = ''
 801                 log.info('|{:>7}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | '+ '{:>9.3f}'.format(pps_tx) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9.3f}'.format(pps_rx)+' Mpps | '+ '{:>9.0f}'.format(lat_avg)+' us   | '+ '{:>9.0f}'.format(lat_max)+' us   | '+ '{:>14d}'.format(abs_dropped)+ ' |''{:>9.2f}'.format(drop_rate)+ '%  |'+lat_warning)
 802                 writer.writerow({'Flows':FLOWSIZE,'PacketSize':(size+4),'RequestedPPS':get_pps(speed,size),'GeneratedPPS':pps_req_tx,'SentPPS':pps_tx,'ForwardedPPS':pps_sut_tx_str,'ReceivedPPS':pps_rx,'AvgLatencyUSEC':lat_avg,'MaxLatencyUSEC':lat_max,'Dropped':abs_dropped,'DropRate':drop_rate})
 803                 if PushGateway:
 804                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
 805                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nDropped {}\nDropRate {}\n'.format(FLOWSIZE,size+4,get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,drop_rate)
 806                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 807                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 808
 809 def run_warmuptest(gensock):
 810 # Running at low speed to make sure the ARP messages can get through.
 811 # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
 812 # Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
 813 # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
 814         gensock.speed(WARMUPSPEED / len(gencores) /len (gentasks), gencores, gentasks)
 815         size=PACKETSIZE-4
 816         gensock.set_size(gencores,0,size) # This is setting the frame size
 817         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 818         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 819         gensock.set_value(gencores,0,56,1,1)
 820         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 821         source_port,destination_port = flows[FLOWSIZE]
 822         gensock.set_random(gencores,0,34,source_port,2)
 823         gensock.set_random(gencores,0,36,destination_port,2)
 824         gensock.start(genstatcores)
 825         time.sleep(WARMUPTIME)
 826         gensock.stop(genstatcores)
 827         gensock.set_value(gencores,0,56,50,1)
 828         time.sleep(WARMUPTIME)
 829
 830 # To generate a desired number of flows, PROX will randomize the bits in source and destination ports, as specified by the bit masks in the flows variable.
 831 flows={\
 832 1:      ['1000000000000000','1000000000000000'],\
 833 2:      ['1000000000000000','100000000000000X'],\
 834 4:      ['100000000000000X','100000000000000X'],\
 835 8:      ['100000000000000X','10000000000000XX'],\
 836 16:     ['10000000000000XX','10000000000000XX'],\
 837 32:     ['10000000000000XX','1000000000000XXX'],\
 838 64:     ['1000000000000XXX','1000000000000XXX'],\
 839 128:    ['1000000000000XXX','100000000000XXXX'],\
 840 256:    ['100000000000XXXX','100000000000XXXX'],\
 841 512:    ['100000000000XXXX','10000000000XXXXX'],\
 842 1024:   ['10000000000XXXXX','10000000000XXXXX'],\
 843 2048:   ['10000000000XXXXX','1000000000XXXXXX'],\
 844 4096:   ['1000000000XXXXXX','1000000000XXXXXX'],\
 845 8192:   ['1000000000XXXXXX','100000000XXXXXXX'],\
 846 16384:  ['100000000XXXXXXX','100000000XXXXXXX'],\
 847 32768:  ['100000000XXXXXXX','10000000XXXXXXXX'],\
 848 65536:  ['10000000XXXXXXXX','10000000XXXXXXXX'],\
 849 131072: ['10000000XXXXXXXX','1000000XXXXXXXXX'],\
 850 262144: ['1000000XXXXXXXXX','1000000XXXXXXXXX'],\
 851 524288: ['1000000XXXXXXXXX','100000XXXXXXXXXX'],\
 852 1048576:['100000XXXXXXXXXX','100000XXXXXXXXXX'],}
 853 clients =[]
 854 socks =[]
 855 socks_control =[]
 856 vmDPIP =[]
 857 vmAdminIP =[]
 858 vmDPmac =[]
 859 hexDPIP =[]
 860 config_file =[]
 861 prox_socket =[]
 862 prox_launch_exit =[]
 863 auto_start =[]
 864 mach_type =[]
 865 sock_type =[]
 866 cores = []
 867 ports = []
 868 tasks = {}
 869 TST009_S = []
 870
 871 data_file = 'RUN{}.{}.csv'.format(env,test_file)
 872 data_csv_file = open(data_file,'w')
 873 testconfig = ConfigParser.RawConfigParser()
 874 testconfig.read(test_file)
 875 required_number_of_test_machines = testconfig.get('DEFAULT', 'total_number_of_test_machines')
 876 TestName = testconfig.get('DEFAULT', 'name')
 877 if testconfig.has_option('DEFAULT', 'PushGateway'):
 878         PushGateway = testconfig.get('DEFAULT', 'PushGateway')
 879         log.info('Measurements will be pushed to %s'%PushGateway)
 880 else:
 881         PushGateway = None
 882 if testconfig.has_option('DEFAULT', 'lat_percentile'):
 883         LAT_PERCENTILE = float(testconfig.get('DEFAULT', 'lat_percentile')) /100.0
 884 else:
 885         LAT_PERCENTILE = 0.99
 886 log.info('Latency percentile measured at {:.0f}%'.format(LAT_PERCENTILE*100))
 887 config = ConfigParser.RawConfigParser()
 888 config.read(env)
 889 machine_map = ConfigParser.RawConfigParser()
 890 machine_map.read(machine_map_file)
 891 key = config.get('ssh', 'key')
 892 user = config.get('ssh', 'user')
 893 total_number_of_machines = config.get('rapid', 'total_number_of_machines')
 894 if int(required_number_of_test_machines) > int(total_number_of_machines):
 895         log.exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
 896         raise Exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
 897 for vm in range(1, int(total_number_of_machines)+1):
 898         vmAdminIP.append(config.get('M%d'%vm, 'admin_ip'))
 899         vmDPmac.append(config.get('M%d'%vm, 'dp_mac'))
 900         vmDPIP.append(config.get('M%d'%vm, 'dp_ip'))
 901         ip = vmDPIP[-1].split('.')
 902         hexDPIP.append(hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2))
 903 machine_index = []
 904 for vm in range(1, int(required_number_of_test_machines)+1):
 905         machine_index.append(int(machine_map.get('TestM%d'%vm, 'machine_index'))-1)
 906         prox_socket.append(testconfig.getboolean('TestM%d'%vm, 'prox_socket'))
 907 for vm in range(1, int(required_number_of_test_machines)+1):
 908         if prox_socket[vm-1]:
 909                 prox_launch_exit.append(testconfig.getboolean('TestM%d'%vm, 'prox_launch_exit'))
 910                 config_file.append(testconfig.get('TestM%d'%vm, 'config_file'))
 911                 # Looking for all task definitions in the PROX cfg files. Constructing a list of all tasks used
 912                 textfile =  open (config_file[-1], 'r')
 913                 filetext = textfile.read()
 914                 textfile.close()
 915                 tasks_for_this_cfg = set(re.findall("task\s*=\s*(\d+)",filetext))
 916                 with open('{}_{}_parameters{}.lua'.format(env,test_file,vm), "w") as f:
 917                         f.write('name="%s"\n'% testconfig.get('TestM%d'%vm, 'name'))
 918                         f.write('local_ip="%s"\n'% vmDPIP[machine_index[vm-1]])
 919                         f.write('local_hex_ip="%s"\n'% hexDPIP[machine_index[vm-1]])
 920                         if testconfig.has_option('TestM%d'%vm, 'cores'):
 921                                 cores.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'cores')))
 922                                 f.write('cores="%s"\n'% ','.join(map(str, cores[-1])))
 923                         else:
 924                                 cores.append(None)
 925                         if testconfig.has_option('TestM%d'%vm, 'ports'):
 926                                 ports.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'ports')))
 927                                 f.write('ports="%s"\n'% ','.join(map(str, ports[-1])))
 928                         else:
 929                                 ports.append(None)
 930                         if re.match('(l2){0,1}gen(_bare){0,1}.*\.cfg',config_file[-1]):
 931                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
 932                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
 933                                 genstatcores = gencores + latcores
 934                                 gentasks = tasks_for_this_cfg
 935                                 auto_start.append(False)
 936                                 mach_type.append('gen')
 937                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
 938                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
 939                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 940                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 941                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 942                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 943                         elif re.match('(l2){0,1}gen_gw.*\.cfg',config_file[-1]):
 944                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
 945                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
 946                                 genstatcores = gencores + latcores
 947                                 gentasks = tasks_for_this_cfg
 948                                 auto_start.append(False)
 949                                 mach_type.append('gen')
 950                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
 951                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
 952                                 gwVMindex = int(testconfig.get('TestM%d'%vm, 'gw_vm')) -1
 953                                 f.write('gw_ip="%s"\n'% vmDPIP[machine_index[gwVMindex]])
 954                                 f.write('gw_hex_ip="%s"\n'% hexDPIP[machine_index[gwVMindex]])
 955                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 956                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 957                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 958                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 959                         elif re.match('(l2){0,1}swap.*\.cfg',config_file[-1]):
 960                                 sutstatcores = cores[-1]
 961                                 auto_start.append(True)
 962                                 mach_type.append('sut')
 963                         elif re.match('secgw1.*\.cfg',config_file[-1]):
 964                                 auto_start.append(True)
 965                                 mach_type.append('none')
 966                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 967                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 968                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 969                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 970                         elif re.match('secgw2.*\.cfg',config_file[-1]):
 971                                 sutstatcores = cores[-1]
 972                                 auto_start.append(True)
 973                                 mach_type.append('sut')
 974                         else:
 975                                 auto_start.append(True)
 976                                 mach_type.append('none')
 977                 f.close
 978                 tasks = tasks_for_this_cfg.union(tasks)
 979 log.debug("Tasks detected in all PROX config files %r"%tasks)
 980 #####################################################################################
 981 def exit_handler():
 982         log.debug ('exit cleanup')
 983         for index, sock in enumerate(socks):
 984                 if socks_control[index]:
 985                         sock.quit()
 986         for client in clients:
 987                 client.close()
 988         data_csv_file.close
 989         sys.exit(0)
 990
 991 atexit.register(exit_handler)
 992
 993 for vm in range(0, int(required_number_of_test_machines)):
 994         if prox_socket[vm]:
 995                 clients.append(prox_ctrl(vmAdminIP[machine_index[vm]], key,user))
 996                 connect_client(clients[-1])
 997 # Creating script to bind the right network interface to the poll mode driver
 998                 devbindfile = '{}_{}_devbindvm{}.sh'.format(env,test_file, vm+1)
 999                 with open(devbindfile, "w") as f:
1000                         newText= 'link="$(ip -o link | grep '+vmDPmac[machine_index[vm]]+' |cut -d":" -f 2)"\n'
1001                         f.write(newText)
1002                         newText= 'if [ -n "$link" ];\n'
1003                         f.write(newText)
1004                         newText= 'then\n'
1005                         f.write(newText)
1006                         newText= '        echo Need to bind\n'
1007                         f.write(newText)
1008                         newText= '        sudo ' + rundir + '/dpdk/usertools/dpdk-devbind.py --force --bind igb_uio $('+rundir+'/dpdk/usertools/dpdk-devbind.py --status |grep  $link | cut -d" " -f 1)\n'
1009                         f.write(newText)
1010                         newText= 'else\n'
1011                         f.write(newText)
1012                         newText= '       echo Assuming port is already bound to DPDK\n'
1013                         f.write(newText)
1014                         newText= 'fi\n'
1015                         f.write(newText)
1016                         newText= 'exit 0\n'
1017                         f.write(newText)
1018                 st = os.stat(devbindfile)
1019                 os.chmod(devbindfile, st.st_mode | stat.S_IEXEC)
1020                 clients[-1].scp_put('./%s'%devbindfile, rundir+'/devbind.sh')
1021                 cmd = 'sudo ' + rundir+ '/devbind.sh'
1022                 clients[-1].run_cmd(cmd)
1023                 log.debug("devbind.sh running on VM%d"%(vm+1))
1024                 clients[-1].scp_put('./%s'%config_file[vm], rundir+'/%s'%config_file[vm])
1025                 clients[-1].scp_put('./{}_{}_parameters{}.lua'.format(env,test_file, vm+1), rundir + '/parameters.lua')
1026                 if not configonly:
1027                         if prox_launch_exit[vm]:
1028                                 log.debug("Starting PROX on VM%d"%(vm+1))
1029                                 if auto_start[vm]:
1030                                         cmd = 'sudo ' +rundir + '/prox/build/prox -t -o cli -f ' + rundir + '/%s'%config_file[vm]
1031                                 else:
1032                                         cmd = 'sudo ' +rundir + '/prox/build/prox -e -t -o cli -f ' + rundir + '/%s'%config_file[vm]
1033                                 clients[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1))
1034                         socks_control.append(prox_launch_exit[vm])
1035                         socks.append(connect_socket(clients[-1]))
1036                         sock_type.append(mach_type[vm])
1037
1038 def get_BinarySearchParams() :
1039         global  DROP_RATE_TRESHOLD
1040         global  LAT_AVG_TRESHOLD
1041         global  LAT_PERC_TRESHOLD
1042         global  LAT_MAX_TRESHOLD
1043         global  ACCURACY
1044         global  STARTSPEED
1045         global  TST009
1046         global  TST009_MAXr
1047         global  TST009_MAXz
1048         DROP_RATE_TRESHOLD = float(testconfig.get('BinarySearchParams', 'drop_rate_threshold'))
1049         LAT_AVG_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_avg_threshold'))
1050         LAT_PERC_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_perc_threshold'))
1051         LAT_MAX_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_max_threshold'))
1052         ACCURACY = float(testconfig.get('BinarySearchParams', 'accuracy'))
1053         STARTSPEED = float(testconfig.get('BinarySearchParams', 'startspeed'))
1054         TST009_MAXr = 1
1055         TST009_MAXz = inf
1056         TST009 = False
1057
1058 def get_FixedRateParams() :
1059         global  DROP_RATE_TRESHOLD
1060         global  LAT_AVG_TRESHOLD
1061         global  LAT_PERC_TRESHOLD
1062         global  LAT_MAX_TRESHOLD
1063         global  flow_size_list
1064         global  packet_size_list
1065         global  STARTSPEED
1066         global  TST009
1067         global  TST009_MAXr
1068         global  TST009_MAXz
1069         DROP_RATE_TRESHOLD = inf
1070         LAT_AVG_TRESHOLD = inf
1071         LAT_PERC_TRESHOLD = inf
1072         LAT_MAX_TRESHOLD = inf
1073         TST009_MAXr = 1
1074         TST009_MAXz = inf
1075         TST009 = False
1076         packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1077         flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1078         STARTSPEED = float(testconfig.get('test%d'%test_nr, 'speed'))
1079
1080 def get_TST009SearchParams() :
1081         global  DROP_RATE_TRESHOLD
1082         global  LAT_AVG_TRESHOLD
1083         global  LAT_PERC_TRESHOLD
1084         global  LAT_MAX_TRESHOLD
1085         global  TST009
1086         global  TST009_MAXr
1087         global  TST009_MAXz
1088         global  TST009_MAXFramesAllIngress
1089         global  TST009_StepSize
1090         global  TST009_n
1091         global  TST009_L
1092         global  TST009_R
1093         global  TST009_S
1094         if testconfig.has_option('TST009SearchParams', 'drop_rate_threshold'):
1095                 DROP_RATE_TRESHOLD = float(testconfig.get('TST009SearchParams', 'drop_rate_threshold'))
1096         else:
1097                 DROP_RATE_TRESHOLD = 0
1098         LAT_AVG_TRESHOLD = inf
1099         LAT_PERC_TRESHOLD = inf
1100         LAT_MAX_TRESHOLD = inf
1101         TST009_MAXr = float(testconfig.get('TST009SearchParams', 'MAXr'))
1102         TST009_MAXz = float(testconfig.get('TST009SearchParams', 'MAXz'))
1103         TST009_MAXFramesAllIngress = int(testconfig.get('TST009SearchParams', 'MAXFramesPerSecondAllIngress'))
1104         TST009_StepSize = int(testconfig.get('TST009SearchParams', 'StepSize'))
1105         TST009_n = int(ceil(TST009_MAXFramesAllIngress / TST009_StepSize))
1106         TST009 = True
1107         TST009_L = 0
1108         TST009_R = TST009_n - 1
1109         for m in range(0, TST009_n):
1110                 TST009_S.append((m+1) * TST009_StepSize)
1111
1112 if configonly:
1113         sys.exit()
1114 ####################################################
1115 # Run test cases
1116 # Best to run the flow test at the end since otherwise the tests coming after might be influenced by the big number of entries in the switch flow tables
1117 ####################################################
1118 gensock_index = sock_type.index('gen') if 'gen' in sock_type else -1
1119 sutsock_index = sock_type.index('sut') if 'sut' in sock_type else -1
1120 number_of_tests = testconfig.get('DEFAULT', 'number_of_tests')
1121 for test_nr in range(1, int(number_of_tests)+1):
1122         test=testconfig.get('test%d'%test_nr,'test')
1123         log.info(test)
1124         if test == 'flowsizetest':
1125                 get_BinarySearchParams()
1126                 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1127                 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1128                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1129         elif test == 'TST009test':
1130                 get_TST009SearchParams()
1131                 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1132                 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1133                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1134         elif test == 'fixed_rate':
1135                 get_FixedRateParams()
1136                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1137         elif test == 'corestats':
1138                 run_core_stats(socks)
1139         elif test == 'portstats':
1140                 run_port_stats(socks)
1141         elif test == 'impairtest':
1142                 get_BinarySearchParams()
1143                 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1144                 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1145                 run_impairtest(socks[gensock_index],socks[sutsock_index])
1146         elif test == 'irqtest':
1147                 run_irqtest(socks)
1148         elif test == 'warmuptest':
1149                 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1150                 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1151                 WARMUPSPEED = int(testconfig.get('test%d'%test_nr, 'warmupspeed'))
1152                 WARMUPTIME = int(testconfig.get('test%d'%test_nr, 'warmuptime'))
1153                 run_warmuptest(socks[gensock_index])
1154 ####################################################