VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py

   1 #!/usr/bin/python
   2
   3 ##
   4 ## Copyright (c) 2010-2020 Intel Corporation
   5 ##
   6 ## Licensed under the Apache License, Version 2.0 (the "License");
   7 ## you may not use this file except in compliance with the License.
   8 ## You may obtain a copy of the License at
   9 ##
  10 ##     http://www.apache.org/licenses/LICENSE-2.0
  11 ##
  12 ## Unless required by applicable law or agreed to in writing, software
  13 ## distributed under the License is distributed on an "AS IS" BASIS,
  14 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 ## See the License for the specific language governing permissions and
  16 ## limitations under the License.
  17 ##
  18
  19 from __future__ import print_function
  20
  21 import os
  22 import stat
  23 import sys
  24 import time
  25 import subprocess
  26 import getopt
  27 import re
  28 import logging
  29 from logging.handlers import RotatingFileHandler
  30 from logging import handlers
  31 from prox_ctrl import prox_ctrl
  32 import ConfigParser
  33 import ast
  34 import atexit
  35 import csv
  36 import requests
  37 from math import ceil
  38 inf = float("inf")
  39
  40 version="20.01.10"
  41 env = "rapid.env" #Default string for environment
  42 test_file = "basicrapid.test" #Default string for test
  43 machine_map_file = "machine.map" #Default string for machine map file
  44 loglevel="DEBUG" # sets log level for writing to file
  45 screenloglevel="INFO" # sets log level for writing to screen
  46 runtime=10 # time in seconds for 1 test run
  47 configonly = False # IF True, the system will upload all the necessary config fiels to the VMs, but not start PROX and the actual testing
  48 rundir = "/home/centos" # Directory where to find the tools in the machines running PROX
  49
  50 def usage():
  51         print("usage: runrapid    [--version] [-v]")
  52         print("                   [--env ENVIRONMENT_NAME]")
  53         print("                   [--test TEST_NAME]")
  54         print("                   [--map MACHINE_MAP_FILE]")
  55         print("                   [--runtime TIME_FOR_TEST]")
  56         print("                   [--configonly False|True]")
  57         print("                   [--log DEBUG|INFO|WARNING|ERROR|CRITICAL]")
  58         print("                   [-h] [--help]")
  59         print("")
  60         print("Command-line interface to runrapid")
  61         print("")
  62         print("optional arguments:")
  63         print("  -v,  --version                 Show program's version number and exit")
  64         print("  --env ENVIRONMENT_NAME         Parameters will be read from ENVIRONMENT_NAME. Default is %s."%env)
  65         print("  --test TEST_NAME               Test cases will be read from TEST_NAME. Default is %s."%test_file)
  66         print("  --map MACHINE_MAP_FILE Machine mapping will be read from MACHINE_MAP_FILE. Default is %s."%machine_map_file)
  67         print("  --runtime                      Specify time in seconds for 1 test run")
  68         print("  --configonly                   If this option is specified, only upload all config files to the VMs, do not run the tests")
  69         print("  --log                          Specify logging level for log file output, default is DEBUG")
  70         print("  --screenlog                    Specify logging level for screen output, default is INFO")
  71         print("  -h, --help                     Show help message and exit.")
  72         print("")
  73
  74 try:
  75         opts, args = getopt.getopt(sys.argv[1:], "vh", ["version","help", "env=", "test=", "map=", "runtime=","configonly","log=","screenlog="])
  76 except getopt.GetoptError as err:
  77         print("===========================================")
  78         print(str(err))
  79         print("===========================================")
  80         usage()
  81         sys.exit(2)
  82 if args:
  83         usage()
  84         sys.exit(2)
  85 for opt, arg in opts:
  86         if opt in ["-h", "--help"]:
  87                 usage()
  88                 sys.exit()
  89         if opt in ["-v", "--version"]:
  90                 print("Rapid Automated Performance Indication for Dataplane "+version)
  91                 sys.exit()
  92         if opt in ["--env"]:
  93                 env = arg
  94         if opt in ["--test"]:
  95                 test_file = arg
  96         if opt in ["--map"]:
  97                 machine_map_file = arg
  98         if opt in ["--runtime"]:
  99                 runtime = arg
 100         if opt in ["--configonly"]:
 101                 configonly = True
 102                 print('No actual runs, only uploading configuration files')
 103         if opt in ["--log"]:
 104                 loglevel = arg
 105                 print ("Log level: "+ loglevel)
 106         if opt in ["--screenlog"]:
 107                 screenloglevel = arg
 108                 print ("Screen Log level: "+ screenloglevel)
 109
 110 print ("Using '"+env+"' as name for the environment")
 111 print ("Using '"+test_file+"' for test case definition")
 112 print ("Using '"+machine_map_file+"' for machine mapping")
 113 print ("Runtime: "+ str(runtime))
 114
 115 class bcolors:
 116         HEADER = '\033[95m'
 117         OKBLUE = '\033[94m'
 118         OKGREEN = '\033[92m'
 119         WARNING = '\033[93m'
 120         FAIL = '\033[91m'
 121         ENDC = '\033[0m'
 122         BOLD = '\033[1m'
 123         UNDERLINE = '\033[4m'
 124         FLASH = '\033[5m'
 125
 126 # create formatters
 127 screen_formatter = logging.Formatter("%(message)s")
 128 file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 129
 130 # get a top-level logger,
 131 # set its log level,
 132 # BUT PREVENT IT from propagating messages to the root logger
 133 #
 134 log = logging.getLogger()
 135 numeric_level = getattr(logging, loglevel.upper(), None)
 136 if not isinstance(numeric_level, int):
 137         raise ValueError('Invalid log level: %s' % loglevel)
 138 log.setLevel(numeric_level)
 139 log.propagate = 0
 140
 141 # create a console handler
 142 # and set its log level to the command-line option
 143 #
 144 console_handler = logging.StreamHandler(sys.stdout)
 145 #console_handler.setLevel(logging.INFO)
 146 numeric_screenlevel = getattr(logging, screenloglevel.upper(), None)
 147 if not isinstance(numeric_screenlevel, int):
 148         raise ValueError('Invalid screenlog level: %s' % screenloglevel)
 149 console_handler.setLevel(numeric_screenlevel)
 150 console_handler.setFormatter(screen_formatter)
 151
 152 # create a file handler
 153 # and set its log level
 154 #
 155 log_file = 'RUN{}.{}.log'.format(env,test_file)
 156 file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10)
 157 #file_handler = log.handlers.TimedRotatingFileHandler(log_file, 'D', 1, 5)
 158 file_handler.setLevel(numeric_level)
 159 file_handler.setFormatter(file_formatter)
 160
 161 # add handlers to the logger
 162 #
 163 log.addHandler(file_handler)
 164 log.addHandler(console_handler)
 165
 166 # Check if log exists and should therefore be rolled
 167 needRoll = os.path.isfile(log_file)
 168
 169
 170 # This is a stale log, so roll it
 171 if needRoll:
 172         # Add timestamp
 173         log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime())
 174
 175         # Roll over on application start
 176         log.handlers[0].doRollover()
 177
 178 # Add timestamp
 179 log.debug('\n---------\nLog started on %s.\n---------\n' % time.asctime())
 180
 181 log.debug("runrapid.py version: "+version)
 182 #========================================================================
 183 def connect_socket(client):
 184         attempts = 1
 185         log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 186         sock = None
 187         while True:
 188                 sock = client.prox_sock()
 189                 if sock is not None:
 190                         break
 191                 attempts += 1
 192                 if attempts > 20:
 193                         log.exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 194                         raise Exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 195                 time.sleep(2)
 196                 log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 197         log.info("Connected to PROX on %s" % client.ip())
 198         return sock
 199
 200 def connect_client(client):
 201         attempts = 1
 202         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 203         while True:
 204                 try:
 205                         client.connect()
 206                         break
 207                 except RuntimeWarning, ex:
 208                         attempts += 1
 209                         if attempts > 20:
 210                                 log.exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 211                                 raise Exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 212                         time.sleep(2)
 213                         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 214         log.debug("Connected to VM on %s" % client.ip())
 215
 216 def report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,tx,rx,tot_drop,elapsed_time,speed_prefix='',lat_avg_prefix='',lat_perc_prefix='',lat_max_prefix='',abs_drop_rate_prefix='',drop_rate_prefix=''):
 217         if flow_number < 0:
 218                 flow_number_str = '| ({:>4}) |'.format(abs(flow_number))
 219         else:
 220                 flow_number_str = '|{:>7} |'.format(flow_number)
 221         if pps_req_tx is None:
 222                 pps_req_tx_str = '{0: >14}'.format('   NA     |')
 223         else:
 224                 pps_req_tx_str = '{:>7.3f} Mpps |'.format(pps_req_tx)
 225         if pps_tx is None:
 226                 pps_tx_str = '{0: >14}'.format('   NA     |')
 227         else:
 228                 pps_tx_str = '{:>7.3f} Mpps |'.format(pps_tx)
 229         if pps_sut_tx is None:
 230                 pps_sut_tx_str = '{0: >14}'.format('   NA     |')
 231         else:
 232                 pps_sut_tx_str = '{:>7.3f} Mpps |'.format(pps_sut_tx)
 233         if pps_rx is None:
 234                 pps_rx_str = '{0: >24|}'.format('NA        ')
 235         else:
 236                 pps_rx_str = bcolors.OKBLUE + '{:>4.1f} Gb/s |{:7.3f} Mpps {}|'.format(get_speed(pps_rx,size),pps_rx,bcolors.ENDC)
 237         if tot_drop is None:
 238                 tot_drop_str = ' |       NA  | '
 239         else:
 240                 tot_drop_str = ' | {:>9.0f} | '.format(tot_drop)
 241         if lat_perc is None:
 242                 lat_perc_str = ' |{:^10.10}|'.format('NA')
 243         elif lat_perc_max == True:
 244                 lat_perc_str = ' |>{}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC)
 245         else:
 246                 lat_perc_str = ' | {}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC)
 247         if elapsed_time is None:
 248                 elapsed_time_str = ' NA |'
 249         else:
 250                 elapsed_time_str = '{:>3.0f} |'.format(elapsed_time)
 251         return(flow_number_str + '{:>5.1f}'.format(speed) + '% '+speed_prefix +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+ pps_req_tx_str + pps_tx_str + bcolors.ENDC + pps_sut_tx_str + pps_rx_str +lat_avg_prefix+ ' {:>5.0f}'.format(lat_avg)+' us'+lat_perc_str+lat_max_prefix+'{:>6.0f}'.format(lat_max)+' us | ' + '{:>9.0f}'.format(tx) + ' | {:>9.0f}'.format(rx) + ' | '+ abs_drop_rate_prefix+ '{:>9.0f}'.format(tx-rx) + tot_drop_str +drop_rate_prefix+ '{:>5.2f}'.format(float(tx-rx)/tx)  +bcolors.ENDC+' |' + elapsed_time_str)
 252
 253 def run_iteration(gensock, sutsock, requested_duration,flow_number,size,speed):
 254         r = 0;
 255         sleep_time = 2
 256         while (r < TST009_MAXr):
 257                 time.sleep(sleep_time)
 258                 # Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
 259                 t1_rx, t1_non_dp_rx, t1_tx, t1_non_dp_tx, t1_drop, t1_tx_fail, t1_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
 260                 t1_dp_rx = t1_rx - t1_non_dp_rx
 261                 t1_dp_tx = t1_tx - t1_non_dp_tx
 262                 gensock.start(gencores)
 263                 time.sleep(2) ## Needs to be 2 seconds since this the time that PROX uses to refresh the stats. Note that this can be changed in PROX!! Don't do it.
 264                 if sutsock!='none':
 265                         t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 266                         ##t2_sut_rx = t2_sut_rx - t2_sut_non_dp_rx
 267                         ##t2_sut_tx = t2_sut_tx - t2_sut_non_dp_tx
 268                 t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 269                 tx = t2_tx - t1_tx
 270                 dp_tx =  tx - (t2_non_dp_tx - t1_non_dp_tx )
 271                 dp_rx =  t2_rx - t1_rx - (t2_non_dp_rx - t1_non_dp_rx)
 272                 tot_dp_drop = dp_tx - dp_rx
 273                 if tx == 0:
 274                         log.critical("TX = 0. Test interrupted since no packet has been sent.")
 275                         raise Exception("TX = 0")
 276                 if dp_tx == 0:
 277                         log.critical("Only non-dataplane packets (e.g. ARP) sent. Test interrupted since no packet has been sent.")
 278                         raise Exception("Only non-dataplane packets (e.g. ARP) sent")
 279                 # Ask PROX to calibrate the bucket size once we have a PROX function to do this.
 280                 # Measure latency statistics per second
 281                 lat_min, lat_max, lat_avg, used_avg, t2_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
 282                 lat_samples = sum(buckets)
 283                 sample_count = 0
 284                 for sample_percentile, bucket in enumerate(buckets,start=1):
 285                         sample_count += bucket
 286                         if sample_count > (lat_samples * LAT_PERCENTILE):
 287                                 break
 288                 percentile_max = (sample_percentile == len(buckets))
 289                 sample_percentile = sample_percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
 290                 if test == 'fixed_rate':
 291                         log.info(report_result(flow_number,size,speed,None,None,None,None,lat_avg,sample_percentile,percentile_max,lat_max, dp_tx, dp_rx , None, None))
 292                 tot_rx = tot_non_dp_rx = tot_tx = tot_non_dp_tx = tot_drop = 0
 293                 lat_avg = used_avg = 0
 294                 buckets_total = [0] * 128
 295                 tot_lat_samples = 0
 296                 tot_lat_measurement_duration = float(0)
 297                 tot_core_measurement_duration = float(0)
 298                 tot_sut_core_measurement_duration = float(0)
 299                 tot_sut_rx = tot_sut_non_dp_rx = tot_sut_tx = tot_sut_non_dp_tx = tot_sut_drop = tot_sut_tx_fail = tot_sut_tsc = 0
 300                 lat_avail = core_avail = sut_avail = False
 301                 while (tot_core_measurement_duration - float(requested_duration) <= 0.1) or (tot_lat_measurement_duration - float(requested_duration) <= 0.1):
 302                         time.sleep(0.5)
 303                         lat_min_sample, lat_max_sample, lat_avg_sample, used_sample, t3_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
 304                         # Get statistics after some execution time
 305                         if t3_lat_tsc != t2_lat_tsc:
 306                                 single_lat_measurement_duration = (t3_lat_tsc - t2_lat_tsc) * 1.0 / lat_hz  # time difference between the 2 measurements, expressed in seconds.
 307                                 # A second has passed in between to lat_stats requests. Hence we need to process the results
 308                                 tot_lat_measurement_duration = tot_lat_measurement_duration + single_lat_measurement_duration
 309                                 if lat_min > lat_min_sample:
 310                                         lat_min = lat_min_sample
 311                                 if lat_max < lat_max_sample:
 312                                         lat_max = lat_max_sample
 313                                 lat_avg = lat_avg + lat_avg_sample * single_lat_measurement_duration # Sometimes, There is more than 1 second between 2 lat_stats. Hence we will take the latest measurement
 314                                 used_avg = used_avg + used_sample * single_lat_measurement_duration  # and give it more weigth.
 315                                 lat_samples = sum(buckets)
 316                                 tot_lat_samples += lat_samples
 317                                 sample_count = 0
 318                                 for sample_percentile, bucket in enumerate(buckets,start=1):
 319                                         sample_count += bucket
 320                                         if sample_count > lat_samples * LAT_PERCENTILE:
 321                                                 break
 322                                 percentile_max = (sample_percentile == len(buckets))
 323                                 sample_percentile = sample_percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
 324                                 buckets_total = [buckets_total[i] + buckets[i] for i in range(len(buckets_total))]
 325                                 t2_lat_tsc = t3_lat_tsc
 326                                 lat_avail = True
 327                         t3_rx, t3_non_dp_rx, t3_tx, t3_non_dp_tx, t3_drop, t3_tx_fail, t3_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 328                         if t3_tsc != t2_tsc:
 329                                 single_core_measurement_duration = (t3_tsc - t2_tsc) * 1.0 / tsc_hz  # time difference between the 2 measurements, expressed in seconds.
 330                                 tot_core_measurement_duration = tot_core_measurement_duration + single_core_measurement_duration
 331                                 delta_rx = t3_rx - t2_rx
 332                                 tot_rx += delta_rx
 333                                 delta_non_dp_rx = t3_non_dp_rx - t2_non_dp_rx
 334                                 tot_non_dp_rx += delta_non_dp_rx
 335                                 delta_tx = t3_tx - t2_tx
 336                                 tot_tx += delta_tx
 337                                 delta_non_dp_tx = t3_non_dp_tx - t2_non_dp_tx
 338                                 tot_non_dp_tx += delta_non_dp_tx
 339                                 delta_dp_tx = delta_tx -delta_non_dp_tx
 340                                 delta_dp_rx = delta_rx -delta_non_dp_rx
 341                                 delta_dp_drop = delta_dp_tx - delta_dp_rx
 342                                 tot_dp_drop += delta_dp_drop
 343                                 delta_drop = t3_drop - t2_drop
 344                                 tot_drop += delta_drop
 345                                 t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc = t3_rx, t3_non_dp_rx, t3_tx, t3_non_dp_tx, t3_drop, t3_tx_fail, t3_tsc
 346                                 core_avail = True
 347                         if sutsock!='none':
 348                                 t3_sut_rx, t3_sut_non_dp_rx, t3_sut_tx, t3_sut_non_dp_tx, t3_sut_drop, t3_sut_tx_fail, t3_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 349                                 if t3_sut_tsc != t2_sut_tsc:
 350                                         single_sut_core_measurement_duration = (t3_sut_tsc - t2_sut_tsc) * 1.0 / tsc_hz  # time difference between the 2 measurements, expressed in seconds.
 351                                         tot_sut_core_measurement_duration = tot_sut_core_measurement_duration + single_sut_core_measurement_duration
 352                                         tot_sut_rx += t3_sut_rx - t2_sut_rx
 353                                         tot_sut_non_dp_rx += t3_sut_non_dp_rx - t2_sut_non_dp_rx
 354                                         delta_sut_tx = t3_sut_tx - t2_sut_tx
 355                                         tot_sut_tx += delta_sut_tx
 356                                         delta_sut_non_dp_tx = t3_sut_non_dp_tx - t2_sut_non_dp_tx
 357                                         tot_sut_non_dp_tx += delta_sut_non_dp_tx
 358                                         t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc = t3_sut_rx, t3_sut_non_dp_rx, t3_sut_tx, t3_sut_non_dp_tx, t3_sut_drop, t3_sut_tx_fail, t3_sut_tsc
 359                                         sut_avail = True
 360                         if test == 'fixed_rate':
 361                                 if lat_avail == core_avail == True:
 362                                         lat_avail = core_avail = False
 363                                         pps_req_tx = (delta_tx + delta_drop - delta_rx)/single_core_measurement_duration/1000000
 364                                         pps_tx = delta_tx/single_core_measurement_duration/1000000
 365                                         if sutsock!='none' and sut_avail:
 366                                                 pps_sut_tx = delta_sut_tx/single_sut_core_measurement_duration/1000000
 367                                                 sut_avail = False
 368                                         else:
 369                                                 pps_sut_tx = None
 370                                         pps_rx = delta_rx/single_core_measurement_duration/1000000
 371                                         log.info(report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg_sample,sample_percentile,percentile_max,lat_max_sample,delta_dp_tx,delta_dp_rx,tot_dp_drop,single_core_measurement_duration))
 372                 #Stop generating
 373                 gensock.stop(gencores)
 374                 r += 1
 375                 lat_avg = lat_avg / float(tot_lat_measurement_duration)
 376                 used_avg = used_avg / float(tot_lat_measurement_duration)
 377                 t4_tsc = t2_tsc
 378                 while t4_tsc == t2_tsc:
 379                         t4_rx, t4_non_dp_rx, t4_tx, t4_non_dp_tx, t4_drop, t4_tx_fail, t4_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
 380                 if test == 'fixed_rate':
 381                         t4_lat_tsc = t2_lat_tsc
 382                         while t4_lat_tsc == t2_lat_tsc:
 383                                 lat_min_sample, lat_max_sample, lat_avg_sample, used_sample, t4_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
 384                         sample_count = 0
 385                         lat_samples = sum(buckets)
 386                         for percentile, bucket in enumerate(buckets,start=1):
 387                                 sample_count += bucket
 388                                 if sample_count > lat_samples * LAT_PERCENTILE:
 389                                         break
 390                         percentile_max = (percentile == len(buckets))
 391                         percentile = percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
 392                         lat_max = lat_max_sample
 393                         lat_avg = lat_avg_sample
 394                         delta_rx = t4_rx - t2_rx
 395                         delta_non_dp_rx = t4_non_dp_rx - t2_non_dp_rx
 396                         delta_tx = t4_tx - t2_tx
 397                         delta_non_dp_tx = t4_non_dp_tx - t2_non_dp_tx
 398                         delta_dp_tx = delta_tx -delta_non_dp_tx
 399                         delta_dp_rx = delta_rx -delta_non_dp_rx
 400                         dp_tx = delta_dp_tx
 401                         dp_rx = delta_dp_rx
 402                         tot_dp_drop += delta_dp_tx - delta_dp_rx
 403                         pps_req_tx = None
 404                         pps_tx = None
 405                         pps_sut_tx = None
 406                         pps_rx = None
 407                         drop_rate = 100.0*(dp_tx-dp_rx)/dp_tx
 408                         tot_core_measurement_duration = None
 409                         break ## Not really needed since the while loop will stop when evaluating the value of r
 410                 else:
 411                         sample_count = 0
 412                         for percentile, bucket in enumerate(buckets_total,start=1):
 413                                 sample_count += bucket
 414                                 if sample_count > tot_lat_samples * LAT_PERCENTILE:
 415                                         break
 416                         percentile_max = (percentile == len(buckets_total))
 417                         percentile = percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
 418                         pps_req_tx = (tot_tx + tot_drop - tot_rx)/tot_core_measurement_duration/1000000.0 # tot_drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 419                         pps_tx = tot_tx/tot_core_measurement_duration/1000000.0 # tot_tx is all generated packets actually accepted by the interface
 420                         pps_rx = tot_rx/tot_core_measurement_duration/1000000.0 # tot_rx is all packets received by the nop task = all packets received in the gen VM
 421                         if sutsock!='none' and sut_avail:
 422                                 pps_sut_tx = tot_sut_tx / tot_sut_core_measurement_duration / 1000000.0
 423                         else:
 424                                 pps_sut_tx = None
 425                         dp_tx = (t4_tx - t1_tx) - (t4_non_dp_tx - t1_non_dp_tx)
 426                         dp_rx = (t4_rx - t1_rx) - (t4_non_dp_rx - t1_non_dp_rx)
 427                         tot_dp_drop = dp_tx - dp_rx
 428                         drop_rate = 100.0*tot_dp_drop/dp_tx
 429                         if ((drop_rate < DROP_RATE_TRESHOLD) or (tot_dp_drop == DROP_RATE_TRESHOLD ==0) or (tot_dp_drop > TST009_MAXz)):
 430                                 break
 431         return(pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,percentile,percentile_max,lat_max,dp_tx,dp_rx,tot_dp_drop,(t4_tx_fail - t1_tx_fail),drop_rate,lat_min,used_avg,r,tot_core_measurement_duration)
 432
 433 def new_speed(speed,size,success):
 434         if test == 'fixed_rate':
 435                 return (STARTSPEED)
 436         elif TST009:
 437                 global TST009_m
 438                 global TST009_L
 439                 global TST009_R
 440                 if success:
 441                         TST009_L = TST009_m + 1
 442                 else:
 443                         TST009_R = max(TST009_m - 1, TST009_L)
 444                 TST009_m = int ((TST009_L + TST009_R)/2)
 445                 return (get_percentageof10Gbps(TST009_S[TST009_m],size))
 446         else:
 447                 global minspeed
 448                 global maxspeed
 449                 if success:
 450                         minspeed = speed
 451                 else:
 452                         maxspeed = speed
 453                 return ((minspeed + maxspeed)/2.0)
 454
 455 def get_start_speed_and_init(size):
 456         if test == 'fixed_rate':
 457                 return (STARTSPEED)
 458         elif TST009:
 459                 global TST009_L
 460                 global TST009_R
 461                 global TST009_m
 462                 TST009_L = 0
 463                 TST009_R = TST009_n - 1
 464                 TST009_m = int((TST009_L + TST009_R) / 2)
 465                 return (get_percentageof10Gbps(TST009_S[TST009_m],size))
 466         else:
 467                 global minspeed
 468                 global maxspeed
 469                 minspeed = 0
 470                 maxspeed = STARTSPEED
 471                 return (STARTSPEED)
 472
 473 def resolution_achieved():
 474         if test == 'fixed_rate':
 475                 return (True)
 476         elif TST009:
 477                 return (TST009_L == TST009_R)
 478         else:
 479                 return ((maxspeed - minspeed) <= ACCURACY)
 480
 481 def get_percentageof10Gbps(pps_speed,size):
 482         # speed is given in pps, returning % of 10Gb/s
 483         return (pps_speed / 1000000.0 * 0.08 * (size+24))
 484
 485 def get_pps(speed,size):
 486         # speed is given in % of 10Gb/s, returning Mpps
 487         return (speed * 100.0 / (8*(size+24)))
 488
 489 def get_speed(packet_speed,size):
 490         # return speed in Gb/s
 491         return (packet_speed / 1000.0 * (8*(size+24)))
 492
 493 def set_background_flows(source_port,destination_port):
 494         for sock in background_gen_socks:
 495                 sock.set_random(gencores,0,34,source_port,2)
 496                 sock.set_random(gencores,0,36,destination_port,2)
 497
 498 def set_background_speed(speed):
 499         for sock in background_gen_socks:
 500                 sock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
 501
 502 def start_background_traffic():
 503         for sock in background_gen_socks:
 504                 sock.start(gencores+latcores)
 505
 506 def stop_background_traffic():
 507         for sock in background_gen_socks:
 508                 sock.stop(gencores+latcores)
 509
 510 def run_flow_size_test(gensock,sutsock):
 511         global fieldnames
 512         global writer
 513         #fieldnames = ['Flows','PacketSize','Gbps','Mpps','AvgLatency','MaxLatency','PacketsDropped','PacketDropRate']
 514         fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Sent','Received','Lost','LostTotal']
 515         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 516         writer.writeheader()
 517         gensock.start(latcores)
 518         for size in packet_size_list:
 519                 size = size-4
 520                 gensock.set_size(gencores,0,size) # This is setting the frame size
 521                 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 522                 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 523                 # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 524                 if background_gen_socks:
 525                         backgroundinfo = '{}Running {} x background traffic not represented in the table{}'.format(bcolors.FLASH,len(background_gen_socks),bcolors.ENDC)
 526                 else:
 527                         backgroundinfo = '{}{}'.format(bcolors.FLASH,bcolors.ENDC)
 528                 log.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 529                 log.info('| UDP, {:>5} bytes, different number of flows by randomizing SRC & DST UDP port. {:116.116}|'.format((size+4),backgroundinfo))
 530                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 531                 log.info('| Flows  | Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core           | Avg. Lat.|{:.0f} Pcentil| Max. Lat.|   Sent    |  Received |    Lost   | Total Lost|L.Ratio|Time|'.format(LAT_PERCENTILE*100))
 532                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 533                 for flow_number in flow_size_list:
 534                         attempts = 0
 535                         gensock.reset_stats()
 536                         if sutsock!='none':
 537                                 sutsock.reset_stats()
 538                         source_port,destination_port = flows[flow_number]
 539                         gensock.set_random(gencores,0,34,source_port,2)
 540                         gensock.set_random(gencores,0,36,destination_port,2)
 541                         set_background_flows(source_port,destination_port)
 542                         endspeed = None
 543                         speed = get_start_speed_and_init(size)
 544                         while True:
 545                                 attempts += 1
 546                                 endwarning = False
 547                                 print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 548                                 sys.stdout.flush()
 549                                 # Start generating packets at requested speed (in % of a 10Gb/s link)
 550                                 gensock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
 551                                 set_background_speed(speed)
 552                                 start_background_traffic()
 553                                 ##time.sleep(1)
 554                                 # Get statistics now that the generation is stable and initial ARP messages are dealt with
 555                                 pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc , lat_perc_max, lat_max, abs_tx,abs_rx,abs_dropped, abs_tx_fail, drop_rate, lat_min, lat_used, r, actual_duration = run_iteration(gensock,sutsock,float(runtime),flow_number,size,speed)
 556                                 stop_background_traffic()
 557                                 if r > 1:
 558                                         retry_warning = bcolors.WARNING + ' {:1} retries needed'.format(r) +  bcolors.ENDC
 559                                 else:
 560                                         retry_warning = ''
 561                                 # Drop rate is expressed in percentage. lat_used is a ratio (0 to 1). The sum of these 2 should be 100%.
 562                                 # If the sum is lower than 95, it means that more than 5% of the latency measurements where dropped for accuracy reasons.
 563                                 if (drop_rate + lat_used * 100) < 95:
 564                                         lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 565                                 else:
 566                                         lat_warning = ''
 567                                 if test == 'fixed_rate':
 568                                         endspeed = speed
 569                                         endpps_req_tx = None
 570                                         endpps_tx = None
 571                                         endpps_sut_tx = None
 572                                         endpps_rx = None
 573                                         endlat_avg = lat_avg
 574                                         endlat_perc = lat_perc
 575                                         endlat_perc_max = lat_perc_max
 576                                         endlat_max = lat_max
 577                                         endabs_dropped = abs_dropped
 578                                         enddrop_rate = drop_rate
 579                                         endabs_tx = abs_tx
 580                                         endabs_rx = abs_rx
 581                                         if lat_warning or gen_warning or retry_warning:
 582                                                 endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning + gen_warning)
 583                                         success = True
 584                                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 585                                 # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold
 586                                 # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
 587                                 # This can be specified by putting 0 in the .test file
 588                                 elif ((drop_rate < DROP_RATE_TRESHOLD) or (abs_dropped==DROP_RATE_TRESHOLD==0)) and (lat_avg< LAT_AVG_TRESHOLD) and (lat_perc< LAT_PERC_TRESHOLD) and (lat_max < LAT_MAX_TRESHOLD):
 589                                         if ((get_pps(speed,size) - pps_tx)/get_pps(speed,size))>0.01:
 590                                                 speed_prefix = bcolors.WARNING
 591                                                 if abs_tx_fail > 0:
 592                                                         gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
 593                                                 else:
 594                                                         gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(get_pps(speed,size), pps_tx) + bcolors.ENDC
 595                                         else:
 596                                                 speed_prefix = bcolors.ENDC
 597                                                 gen_warning = ''
 598                                         endspeed = speed
 599                                         endspeed_prefix = speed_prefix
 600                                         endpps_req_tx = pps_req_tx
 601                                         endpps_tx = pps_tx
 602                                         endpps_sut_tx = pps_sut_tx
 603                                         endpps_rx = pps_rx
 604                                         endlat_avg = lat_avg
 605                                         endlat_perc = lat_perc
 606                                         endlat_perc_max = lat_perc_max
 607                                         endlat_max = lat_max
 608                                         endabs_dropped = None
 609                                         enddrop_rate = drop_rate
 610                                         endabs_tx = abs_tx
 611                                         endabs_rx = abs_rx
 612                                         if lat_warning or gen_warning or retry_warning:
 613                                                 endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning + gen_warning)
 614                                         success = True
 615                                         success_message=' SUCCESS'
 616                                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 617                                         log.debug(report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
 618                                 else:
 619                                         success_message=' FAILED'
 620                                         gen_warning = ''
 621                                         abs_drop_rate_prefix = bcolors.ENDC
 622                                         if ((abs_dropped>0) and (DROP_RATE_TRESHOLD ==0)):
 623                                                 abs_drop_rate_prefix = bcolors.FAIL
 624                                         if (drop_rate < DROP_RATE_TRESHOLD):
 625                                                 drop_rate_prefix = bcolors.ENDC
 626                                         else:
 627                                                 drop_rate_prefix = bcolors.FAIL
 628                                         if (lat_avg< LAT_AVG_TRESHOLD):
 629                                                 lat_avg_prefix = bcolors.ENDC
 630                                         else:
 631                                                 lat_avg_prefix = bcolors.FAIL
 632                                         if (lat_perc< LAT_PERC_TRESHOLD):
 633                                                 lat_perc_prefix = bcolors.ENDC
 634                                         else:
 635                                                 lat_perc_prefix = bcolors.FAIL
 636                                         if (lat_max< LAT_MAX_TRESHOLD):
 637                                                 lat_max_prefix = bcolors.ENDC
 638                                         else:
 639                                                 lat_max_prefix = bcolors.FAIL
 640                                         if (((get_pps(speed,size) - pps_tx)/get_pps(speed,size))<0.001):
 641                                                 speed_prefix = bcolors.ENDC
 642                                         else:
 643                                                 speed_prefix = bcolors.FAIL
 644                                         success = False
 645                                         log.debug(report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
 646                                 speed = new_speed(speed, size, success)
 647                                 if resolution_achieved():
 648                                         break
 649                         if endspeed is not  None:
 650                                 speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 651                                 log.info(report_result(flow_number,size,endspeed,endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_perc,endlat_perc_max,endlat_max,endabs_tx,endabs_rx,endabs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix))
 652                                 if endwarning:
 653                                         log.info (endwarning)
 654                                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 655                                 writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':get_pps(endspeed,size),'GeneratedPPS':endpps_req_tx,'SentPPS':endpps_tx,'ForwardedPPS':endpps_sut_tx,'ReceivedPPS':endpps_rx,'AvgLatencyUSEC':endlat_avg,'MaxLatencyUSEC':endlat_max,'Sent':endabs_tx,'Received':endabs_rx,'Lost':endabs_dropped,'LostTotal':endabs_dropped})
 656                                 if PushGateway:
 657                                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
 658                                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nSent {}\nReceived {}\nLost {}\nLostTotal {}\n'.format(flow_number,size+4,get_pps(endspeed,size),endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_max,endabs_tx,endabs_rx,endabs_Dropped,endabs_dropped)
 659                                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 660                                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 661                         else:
 662                                 log.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
 663         gensock.stop(latcores)
 664
 665 def run_core_stats(socks):
 666         fieldnames = ['PROXID','Time','Received','Sent','NonDPReceived','NonDPSent','Delta','NonDPDelta','Dropped']
 667         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 668         writer.writeheader()
 669         log.info("+------------------------------------------------------------------------------------------------------------------+")
 670         log.info("| Measuring core statistics on 1 or more PROX instances                                                            |")
 671         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 672         log.info("| PROX ID   |    Time   |    RX      |     TX     | non DP RX  | non DP TX  |   TX - RX  | nonDP TX-RX|  DROP TOT  |")
 673         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 674         for sock in socks:
 675                 sock.reset_stats()
 676         duration = float(runtime)
 677         tot_drop = []
 678         old_rx = []; old_non_dp_rx = []; old_tx = []; old_non_dp_tx = []; old_drop = []; old_tx_fail = []; old_tsc = []
 679         new_rx = []; new_non_dp_rx = []; new_tx = []; new_non_dp_tx = []; new_drop = []; new_tx_fail = []; new_tsc = []
 680         sockets_to_go = len (socks)
 681         for i,sock in enumerate(socks,start=0):
 682                 tot_drop.append(0)
 683                 old_rx.append(0); old_non_dp_rx.append(0); old_tx.append(0); old_non_dp_tx.append(0); old_drop.append(0); old_tx_fail.append(0); old_tsc.append(0)
 684                 old_rx[-1], old_non_dp_rx[-1], old_tx[-1], old_non_dp_tx[-1], old_drop[-1], old_tx_fail[-1], old_tsc[-1], tsc_hz = sock.core_stats(cores[i],tasks)
 685                 new_rx.append(0); new_non_dp_rx.append(0); new_tx.append(0); new_non_dp_tx.append(0); new_drop.append(0); new_tx_fail.append(0); new_tsc.append(0)
 686         while (duration > 0):
 687                 time.sleep(0.5)
 688                 # Get statistics after some execution time
 689                 for i,sock in enumerate(socks,start=0):
 690                         new_rx[i], new_non_dp_rx[i], new_tx[i], new_non_dp_tx[i], new_drop[i], new_tx_fail[i], new_tsc[i], tsc_hz = sock.core_stats(cores[i],tasks)
 691                         drop = new_drop[i]-old_drop[i]
 692                         rx = new_rx[i] - old_rx[i]
 693                         tx = new_tx[i] - old_tx[i]
 694                         non_dp_rx = new_non_dp_rx[i] - old_non_dp_rx[i]
 695                         non_dp_tx = new_non_dp_tx[i] - old_non_dp_tx[i]
 696                         tsc = new_tsc[i] - old_tsc[i]
 697                         if tsc == 0 :
 698                                 continue
 699                         sockets_to_go -= 1
 700                         old_drop[i] = new_drop[i]
 701                         old_rx[i] = new_rx[i]
 702                         old_tx[i] = new_tx[i]
 703                         old_non_dp_rx[i] = new_non_dp_rx[i]
 704                         old_non_dp_tx[i] = new_non_dp_tx[i]
 705                         old_tsc[i] = new_tsc[i]
 706                         tot_drop[i] = tot_drop[i] + tx - rx
 707                         log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(non_dp_rx)+' | '+'{:>10.0f}'.format(non_dp_tx)+' | ' + '{:>10.0f}'.format(tx-rx) + ' | '+ '{:>10.0f}'.format(non_dp_tx-non_dp_rx) + ' | '+'{:>10.0f}'.format(tot_drop[i]) +' |')
 708                         writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NonDPReceived':non_dp_rx,'NonDPSent':non_dp_tx,'Delta':tx-rx,'NonDPDelta':non_dp_tx-non_dp_rx,'Dropped':tot_drop[i]})
 709                         if PushGateway:
 710                                 URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
 711                                 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNonDPReceived {}\nNonDPSent {}\nDelta {}\nNonDPDelta {}\nDropped {}\n'.format(i,duration,rx,tx,non_dp_rx,non_dp_tx,tx-rx,non_dp_tx-non_dp_rx,tot_drop[i])
 712                                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 713                                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
 714                         if sockets_to_go == 0:
 715                                 duration = duration - 1
 716                                 sockets_to_go = len (socks)
 717         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 718
 719 def run_port_stats(socks):
 720         fieldnames = ['PROXID','Time','Received','Sent','NoMbufs','iErrMiss']
 721         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 722         writer.writeheader()
 723         log.info("+---------------------------------------------------------------------------+")
 724         log.info("| Measuring port statistics on 1 or more PROX instances                     |")
 725         log.info("+-----------+-----------+------------+------------+------------+------------+")
 726         log.info("| PROX ID   |    Time   |    RX      |     TX     | no MBUFS   | ierr&imiss |")
 727         log.info("+-----------+-----------+------------+------------+------------+------------+")
 728         for sock in socks:
 729                 sock.reset_stats()
 730         duration = float(runtime)
 731         old_rx = []; old_tx = []; old_no_mbufs = []; old_errors = []; old_tsc = []
 732         new_rx = []; new_tx = []; new_no_mbufs = []; new_errors = []; new_tsc = []
 733         sockets_to_go = len (socks)
 734         for i,sock in enumerate(socks,start=0):
 735                 old_rx.append(0); old_tx.append(0); old_no_mbufs.append(0); old_errors.append(0); old_tsc.append(0)
 736                 old_rx[-1], old_tx[-1], old_no_mbufs[-1], old_errors[-1], old_tsc[-1] = sock.multi_port_stats(ports[i])
 737                 new_rx.append(0); new_tx.append(0); new_no_mbufs.append(0); new_errors.append(0); new_tsc.append(0)
 738         while (duration > 0):
 739                 time.sleep(0.5)
 740                 # Get statistics after some execution time
 741                 for i,sock in enumerate(socks,start=0):
 742                         new_rx[i], new_tx[i], new_no_mbufs[i], new_errors[i], new_tsc[i] = sock.multi_port_stats(ports[i])
 743                         rx = new_rx[i] - old_rx[i]
 744                         tx = new_tx[i] - old_tx[i]
 745                         no_mbufs = new_no_mbufs[i] - old_no_mbufs[i]
 746                         errors = new_errors[i] - old_errors[i]
 747                         tsc = new_tsc[i] - old_tsc[i]
 748                         if tsc == 0 :
 749                                 continue
 750                         sockets_to_go -= 1
 751                         old_rx[i] = new_rx[i]
 752                         old_tx[i] = new_tx[i]
 753                         old_no_mbufs[i] = new_no_mbufs[i]
 754                         old_errors[i] = new_errors[i]
 755                         old_tsc[i] = new_tsc[i]
 756                         log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(no_mbufs)+' | '+'{:>10.0f}'.format(errors)+' |')
 757                         writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NoMbufs':no_mbufs,'iErrMiss':errors})
 758                         if PushGateway:
 759                                 URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
 760                                 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNoMbufs {}\niErrMiss {}\n'.format(i,duration,rx,tx,no_mbufs,errors)
 761                                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 762                                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
 763                         if sockets_to_go == 0:
 764                                 duration = duration - 1
 765                                 sockets_to_go = len (socks)
 766         log.info("+-----------+-----------+------------+------------+------------+------------+")
 767
 768 def run_irqtest(socks):
 769         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 770         log.info("| Measuring time probably spent dealing with an interrupt. Interrupting DPDK cores for more than 50us might be problematic   ")
 771         log.info("| and result in packet loss. The first row shows the interrupted time buckets: first number is the bucket between 0us and    ")
 772         log.info("| that number expressed in us and so on. The numbers in the other rows show how many times per second, the program was       ")
 773         log.info("| interrupted for a time as specified by its bucket. '0' is printed when there are no interrupts in this bucket throughout   ")
 774         log.info("| the duration of the test. 0.00 means there were interrupts in this bucket but very few. Due to rounding this shows as 0.00 ")
 775         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 776         sys.stdout.flush()
 777         for sock_index,sock in enumerate(socks,start=0):
 778                 buckets=socks[sock_index].show_irq_buckets(1)
 779                 print('Measurement ongoing ... ',end='\r')
 780                 socks[sock_index].stop(cores[sock_index])
 781                 old_irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
 782                 irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
 783                 irq[0][0] = 'bucket us'
 784                 for j,bucket in enumerate(buckets,start=1):
 785                         irq[0][j] = '<'+ bucket
 786                 irq[0][-1] = '>'+ buckets [-2]
 787                 socks[sock_index].start(cores[sock_index])
 788                 time.sleep(2)
 789                 for j,bucket in enumerate(buckets,start=1):
 790                         for i,irqcore in enumerate(cores[sock_index],start=1):
 791                                 old_irq[i][j] = socks[sock_index].irq_stats(irqcore,j-1)
 792                 time.sleep(float(runtime))
 793                 socks[sock_index].stop(cores[sock_index])
 794                 for i,irqcore in enumerate(cores[sock_index],start=1):
 795                         irq[i][0]='core %s '%irqcore
 796                         for j,bucket in enumerate(buckets,start=1):
 797                                 diff =  socks[sock_index].irq_stats(irqcore,j-1) - old_irq[i][j]
 798                                 if diff == 0:
 799                                         irq[i][j] = '0'
 800                                 else:
 801                                         irq[i][j] = str(round(diff/float(runtime), 2))
 802                 log.info('Results for PROX instance %s'%sock_index)
 803                 for row in irq:
 804                         log.info(''.join(['{:>12}'.format(item) for item in row]))
 805
 806 def run_impairtest(gensock,sutsock):
 807         fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Dropped','DropRate']
 808         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 809         writer.writeheader()
 810         size=PACKETSIZE-4
 811         log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 812         log.info("| Generator is sending UDP ("+'{:>5}'.format(FLOWSIZE)+" flow) packets ("+ '{:>5}'.format(size+4) +" bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test    |")
 813         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 814         log.info("| Test   |  Speed requested   | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
 815         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 816         attempts = 0
 817         gensock.set_size(gencores,0,size) # This is setting the frame size
 818         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 819         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 820         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 821         source_port,destination_port = flows[FLOWSIZE]
 822         gensock.set_random(gencores,0,34,source_port,2)
 823         gensock.set_random(gencores,0,36,destination_port,2)
 824         gensock.start(latcores)
 825         speed = STARTSPEED
 826         gensock.speed(speed / len(gencores) / len(gentasks), gencores, gentasks)
 827         while True:
 828                 attempts += 1
 829                 print('Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 830                 sys.stdout.flush()
 831                 time.sleep(1)
 832                 # Get statistics now that the generation is stable and NO ARP messages any more
 833                 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg, lat_perc, lat_perc_max, lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used, r, actual_duration = run_iteration(gensock,sutsock,runtime)
 834                 drop_rate = 100.0*abs_dropped/abs_tx
 835                 if lat_used < 0.95:
 836                         lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 837                 else:
 838                         lat_warning = ''
 839                 log.info('|{:>7}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | '+ '{:>9.3f}'.format(pps_tx) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9.3f}'.format(pps_rx)+' Mpps | '+ '{:>9.0f}'.format(lat_avg)+' us   | '+ '{:>9.0f}'.format(lat_max)+' us   | '+ '{:>14d}'.format(abs_dropped)+ ' |''{:>9.2f}'.format(drop_rate)+ '%  |'+lat_warning)
 840                 writer.writerow({'Flows':FLOWSIZE,'PacketSize':(size+4),'RequestedPPS':get_pps(speed,size),'GeneratedPPS':pps_req_tx,'SentPPS':pps_tx,'ForwardedPPS':pps_sut_tx_str,'ReceivedPPS':pps_rx,'AvgLatencyUSEC':lat_avg,'MaxLatencyUSEC':lat_max,'Dropped':abs_dropped,'DropRate':drop_rate})
 841                 if PushGateway:
 842                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
 843                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nDropped {}\nDropRate {}\n'.format(FLOWSIZE,size+4,get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,drop_rate)
 844                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 845                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 846
 847 def run_warmuptest(gensock):
 848 # Running at low speed to make sure the ARP messages can get through.
 849 # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
 850 # Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
 851 # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
 852         gensock.speed(WARMUPSPEED / len(gencores) /len (gentasks), gencores, gentasks)
 853         size=PACKETSIZE-4
 854         gensock.set_size(gencores,0,size) # This is setting the frame size
 855         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 856         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 857         gensock.set_value(gencores,0,56,1,1)
 858         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 859         source_port,destination_port = flows[FLOWSIZE]
 860         gensock.set_random(gencores,0,34,source_port,2)
 861         gensock.set_random(gencores,0,36,destination_port,2)
 862         gensock.start(genstatcores)
 863         time.sleep(WARMUPTIME)
 864         gensock.stop(genstatcores)
 865         gensock.set_value(gencores,0,56,50,1)
 866         time.sleep(WARMUPTIME)
 867
 868 def exit_handler():
 869         log.debug ('exit cleanup')
 870         for index, sock in enumerate(socks):
 871                 if socks_control[index]:
 872                         sock.quit()
 873         for client in clients:
 874                 client.close()
 875         data_csv_file.close
 876         sys.exit(0)
 877
 878 def get_BinarySearchParams() :
 879         global  DROP_RATE_TRESHOLD
 880         global  LAT_AVG_TRESHOLD
 881         global  LAT_PERC_TRESHOLD
 882         global  LAT_MAX_TRESHOLD
 883         global  ACCURACY
 884         global  STARTSPEED
 885         global  TST009
 886         global  TST009_MAXr
 887         global  TST009_MAXz
 888         DROP_RATE_TRESHOLD = float(testconfig.get('BinarySearchParams', 'drop_rate_threshold'))
 889         LAT_AVG_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_avg_threshold'))
 890         LAT_PERC_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_perc_threshold'))
 891         LAT_MAX_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_max_threshold'))
 892         ACCURACY = float(testconfig.get('BinarySearchParams', 'accuracy'))
 893         STARTSPEED = float(testconfig.get('BinarySearchParams', 'startspeed'))
 894         TST009_MAXr = 1
 895         TST009_MAXz = inf
 896         TST009 = False
 897
 898 def get_FixedRateParams() :
 899         global  DROP_RATE_TRESHOLD
 900         global  LAT_AVG_TRESHOLD
 901         global  LAT_PERC_TRESHOLD
 902         global  LAT_MAX_TRESHOLD
 903         global  flow_size_list
 904         global  packet_size_list
 905         global  STARTSPEED
 906         global  TST009
 907         global  TST009_MAXr
 908         global  TST009_MAXz
 909         DROP_RATE_TRESHOLD = inf
 910         LAT_AVG_TRESHOLD = inf
 911         LAT_PERC_TRESHOLD = inf
 912         LAT_MAX_TRESHOLD = inf
 913         TST009_MAXr = 1
 914         TST009_MAXz = inf
 915         TST009 = False
 916         packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
 917         flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
 918         STARTSPEED = float(testconfig.get('test%d'%test_nr, 'speed'))
 919
 920 def get_TST009SearchParams() :
 921         global  DROP_RATE_TRESHOLD
 922         global  LAT_AVG_TRESHOLD
 923         global  LAT_PERC_TRESHOLD
 924         global  LAT_MAX_TRESHOLD
 925         global  TST009
 926         global  TST009_MAXr
 927         global  TST009_MAXz
 928         global  TST009_MAXFramesAllIngress
 929         global  TST009_StepSize
 930         global  TST009_n
 931         global  TST009_L
 932         global  TST009_R
 933         global  TST009_S
 934         if testconfig.has_option('TST009SearchParams', 'drop_rate_threshold'):
 935                 DROP_RATE_TRESHOLD = float(testconfig.get('TST009SearchParams', 'drop_rate_threshold'))
 936         else:
 937                 DROP_RATE_TRESHOLD = 0
 938         LAT_AVG_TRESHOLD = inf
 939         LAT_PERC_TRESHOLD = inf
 940         LAT_MAX_TRESHOLD = inf
 941         TST009_MAXr = float(testconfig.get('TST009SearchParams', 'MAXr'))
 942         TST009_MAXz = float(testconfig.get('TST009SearchParams', 'MAXz'))
 943         TST009_MAXFramesAllIngress = int(testconfig.get('TST009SearchParams', 'MAXFramesPerSecondAllIngress'))
 944         TST009_StepSize = int(testconfig.get('TST009SearchParams', 'StepSize'))
 945         TST009_n = int(ceil(TST009_MAXFramesAllIngress / TST009_StepSize))
 946         TST009 = True
 947         TST009_L = 0
 948         TST009_R = TST009_n - 1
 949         for m in range(0, TST009_n):
 950                 TST009_S.append((m+1) * TST009_StepSize)
 951 # To generate a desired number of flows, PROX will randomize the bits in source and destination ports, as specified by the bit masks in the flows variable.
 952 flows={\
 953 1:      ['1000000000000000','1000000000000000'],\
 954 2:      ['1000000000000000','100000000000000X'],\
 955 4:      ['100000000000000X','100000000000000X'],\
 956 8:      ['100000000000000X','10000000000000XX'],\
 957 16:     ['10000000000000XX','10000000000000XX'],\
 958 32:     ['10000000000000XX','1000000000000XXX'],\
 959 64:     ['1000000000000XXX','1000000000000XXX'],\
 960 128:    ['1000000000000XXX','100000000000XXXX'],\
 961 256:    ['100000000000XXXX','100000000000XXXX'],\
 962 512:    ['100000000000XXXX','10000000000XXXXX'],\
 963 1024:   ['10000000000XXXXX','10000000000XXXXX'],\
 964 2048:   ['10000000000XXXXX','1000000000XXXXXX'],\
 965 4096:   ['1000000000XXXXXX','1000000000XXXXXX'],\
 966 8192:   ['1000000000XXXXXX','100000000XXXXXXX'],\
 967 16384:  ['100000000XXXXXXX','100000000XXXXXXX'],\
 968 32768:  ['100000000XXXXXXX','10000000XXXXXXXX'],\
 969 65536:  ['10000000XXXXXXXX','10000000XXXXXXXX'],\
 970 131072: ['10000000XXXXXXXX','1000000XXXXXXXXX'],\
 971 262144: ['1000000XXXXXXXXX','1000000XXXXXXXXX'],\
 972 524288: ['1000000XXXXXXXXX','100000XXXXXXXXXX'],\
 973 1048576:['100000XXXXXXXXXX','100000XXXXXXXXXX'],}
 974 clients =[]
 975 socks =[]
 976 socks_control =[]
 977 vmDPIP =[]
 978 vmAdminIP =[]
 979 vmDPmac =[]
 980 hexDPIP =[]
 981 config_file =[]
 982 prox_socket =[]
 983 prox_launch_exit =[]
 984 auto_start =[]
 985 mach_type =[]
 986 sock_type =[]
 987 monitor =[]
 988 sock_monitor =[]
 989 cores = []
 990 ports = []
 991 tasks = {}
 992 TST009_S = []
 993
 994 data_file = 'RUN{}.{}.csv'.format(env,test_file)
 995 data_csv_file = open(data_file,'w')
 996 testconfig = ConfigParser.RawConfigParser()
 997 testconfig.read(test_file)
 998 required_number_of_test_machines = testconfig.get('DEFAULT', 'total_number_of_test_machines')
 999 TestName = testconfig.get('DEFAULT', 'name')
1000 if testconfig.has_option('DEFAULT', 'PushGateway'):
1001         PushGateway = testconfig.get('DEFAULT', 'PushGateway')
1002         log.info('Measurements will be pushed to %s'%PushGateway)
1003 else:
1004         PushGateway = None
1005 if testconfig.has_option('DEFAULT', 'lat_percentile'):
1006         LAT_PERCENTILE = float(testconfig.get('DEFAULT', 'lat_percentile')) /100.0
1007 else:
1008         LAT_PERCENTILE = 0.99
1009 log.info('Latency percentile measured at {:.0f}%'.format(LAT_PERCENTILE*100))
1010 config = ConfigParser.RawConfigParser()
1011 config.read(env)
1012 machine_map = ConfigParser.RawConfigParser()
1013 machine_map.read(machine_map_file)
1014 key = config.get('ssh', 'key')
1015 user = config.get('ssh', 'user')
1016 total_number_of_machines = config.get('rapid', 'total_number_of_machines')
1017 if int(required_number_of_test_machines) > int(total_number_of_machines):
1018         log.exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
1019         raise Exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
1020 for vm in range(1, int(total_number_of_machines)+1):
1021         vmAdminIP.append(config.get('M%d'%vm, 'admin_ip'))
1022         vmDPmac.append(config.get('M%d'%vm, 'dp_mac'))
1023         vmDPIP.append(config.get('M%d'%vm, 'dp_ip'))
1024         ip = vmDPIP[-1].split('.')
1025         hexDPIP.append(hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2))
1026 machine_index = []
1027 for vm in range(1, int(required_number_of_test_machines)+1):
1028         machine_index.append(int(machine_map.get('TestM%d'%vm, 'machine_index'))-1)
1029         prox_socket.append(testconfig.getboolean('TestM%d'%vm, 'prox_socket'))
1030 for vm in range(1, int(required_number_of_test_machines)+1):
1031         if prox_socket[vm-1]:
1032                 prox_launch_exit.append(testconfig.getboolean('TestM%d'%vm, 'prox_launch_exit'))
1033                 config_file.append(testconfig.get('TestM%d'%vm, 'config_file'))
1034                 # Looking for all task definitions in the PROX cfg files. Constructing a list of all tasks used
1035                 textfile =  open (config_file[-1], 'r')
1036                 filetext = textfile.read()
1037                 textfile.close()
1038                 tasks_for_this_cfg = set(re.findall("task\s*=\s*(\d+)",filetext))
1039                 with open('{}_{}_parameters{}.lua'.format(env,test_file,vm), "w") as f:
1040                         f.write('name="%s"\n'% testconfig.get('TestM%d'%vm, 'name'))
1041                         f.write('local_ip="%s"\n'% vmDPIP[machine_index[vm-1]])
1042                         f.write('local_hex_ip="%s"\n'% hexDPIP[machine_index[vm-1]])
1043                         if testconfig.has_option('TestM%d'%vm, 'cores'):
1044                                 cores.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'cores')))
1045                                 f.write('cores="%s"\n'% ','.join(map(str, cores[-1])))
1046                         else:
1047                                 cores.append(None)
1048                         if testconfig.has_option('TestM%d'%vm, 'ports'):
1049                                 ports.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'ports')))
1050                                 f.write('ports="%s"\n'% ','.join(map(str, ports[-1])))
1051                         else:
1052                                 ports.append(None)
1053                         if testconfig.has_option('TestM%d'%vm, 'monitor'):
1054                                 monitor.append(testconfig.getboolean('TestM%d'%vm, 'monitor'))
1055                         else:
1056                                 monitor.append(True)
1057                         if re.match('(l2){0,1}gen(_bare){0,1}.*\.cfg',config_file[-1]):
1058                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
1059                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
1060                                 genstatcores = gencores + latcores
1061                                 gentasks = tasks_for_this_cfg
1062                                 auto_start.append(False)
1063                                 mach_type.append('gen')
1064                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
1065                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
1066                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
1067                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
1068                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
1069                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
1070                                 if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'):
1071                                         BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp'))
1072                                 else:
1073                                         BUCKET_SIZE_EXP = 11
1074                                 f.write('bucket_size_exp="%s"\n'% BUCKET_SIZE_EXP)
1075                                 if testconfig.has_option('TestM%d'%vm, 'heartbeat'):
1076                                         heartbeat = int(testconfig.get('TestM%d'%vm, 'heartbeat'))
1077                                 else:
1078                                         heartbeat = 60
1079                                 f.write('heartbeat="%s"\n'% heartbeat)
1080                         elif re.match('(l2){0,1}gen_gw.*\.cfg',config_file[-1]):
1081                                 if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'):
1082                                         BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp'))
1083                                 else:
1084                                         BUCKET_SIZE_EXP = 11
1085                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
1086                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
1087                                 genstatcores = gencores + latcores
1088                                 gentasks = tasks_for_this_cfg
1089                                 auto_start.append(False)
1090                                 mach_type.append('gen')
1091                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
1092                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
1093                                 gwVMindex = int(testconfig.get('TestM%d'%vm, 'gw_vm')) -1
1094                                 f.write('gw_ip="%s"\n'% vmDPIP[machine_index[gwVMindex]])
1095                                 f.write('gw_hex_ip="%s"\n'% hexDPIP[machine_index[gwVMindex]])
1096                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
1097                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
1098                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
1099                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
1100                                 if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'):
1101                                         BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp'))
1102                                 else:
1103                                         BUCKET_SIZE_EXP = 11
1104                                 f.write('bucket_size_exp="%s"\n'% BUCKET_SIZE_EXP)
1105                                 if testconfig.has_option('TestM%d'%vm, 'heartbeat'):
1106                                         heartbeat = int(testconfig.get('TestM%d'%vm, 'heartbeat'))
1107                                 else:
1108                                         heartbeat = 60
1109                                 f.write('heartbeat="%s"\n'% heartbeat)
1110                         elif re.match('(l2){0,1}swap.*\.cfg',config_file[-1]):
1111                                 sutstatcores = cores[-1]
1112                                 auto_start.append(True)
1113                                 mach_type.append('sut')
1114                         elif re.match('secgw1.*\.cfg',config_file[-1]):
1115                                 auto_start.append(True)
1116                                 mach_type.append('none')
1117                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
1118                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
1119                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
1120                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
1121                         elif re.match('secgw2.*\.cfg',config_file[-1]):
1122                                 sutstatcores = cores[-1]
1123                                 auto_start.append(True)
1124                                 mach_type.append('sut')
1125                         else:
1126                                 auto_start.append(True)
1127                                 mach_type.append('none')
1128                 f.close
1129                 tasks = tasks_for_this_cfg.union(tasks)
1130 log.debug("Tasks detected in all PROX config files %r"%tasks)
1131 #####################################################################################
1132 for vm in range(0, int(required_number_of_test_machines)):
1133         if prox_socket[vm]:
1134                 clients.append(prox_ctrl(vmAdminIP[machine_index[vm]], key,user))
1135                 connect_client(clients[-1])
1136 # Creating script to bind the right network interface to the poll mode driver
1137                 devbindfile = '{}_{}_devbindvm{}.sh'.format(env,test_file, vm+1)
1138                 with open(devbindfile, "w") as f:
1139                         newText= 'link="$(ip -o link | grep '+vmDPmac[machine_index[vm]]+' |cut -d":" -f 2)"\n'
1140                         f.write(newText)
1141                         newText= 'if [ -n "$link" ];\n'
1142                         f.write(newText)
1143                         newText= 'then\n'
1144                         f.write(newText)
1145                         newText= '        echo Need to bind\n'
1146                         f.write(newText)
1147                         newText= '        sudo ' + rundir + '/dpdk/usertools/dpdk-devbind.py --force --bind igb_uio $('+rundir+'/dpdk/usertools/dpdk-devbind.py --status |grep  $link | cut -d" " -f 1)\n'
1148                         f.write(newText)
1149                         newText= 'else\n'
1150                         f.write(newText)
1151                         newText= '       echo Assuming port is already bound to DPDK\n'
1152                         f.write(newText)
1153                         newText= 'fi\n'
1154                         f.write(newText)
1155                         newText= 'exit 0\n'
1156                         f.write(newText)
1157                 st = os.stat(devbindfile)
1158                 os.chmod(devbindfile, st.st_mode | stat.S_IEXEC)
1159                 clients[-1].scp_put('./%s'%devbindfile, rundir+'/devbind.sh')
1160                 cmd = 'sudo ' + rundir+ '/devbind.sh'
1161                 clients[-1].run_cmd(cmd)
1162                 log.debug("devbind.sh running on VM%d"%(vm+1))
1163                 clients[-1].scp_put('./%s'%config_file[vm], rundir+'/%s'%config_file[vm])
1164                 clients[-1].scp_put('./{}_{}_parameters{}.lua'.format(env,test_file, vm+1), rundir + '/parameters.lua')
1165                 if not configonly:
1166                         if prox_launch_exit[vm]:
1167                                 log.debug("Starting PROX on VM%d"%(vm+1))
1168                                 if auto_start[vm]:
1169                                         cmd = 'sudo ' +rundir + '/prox/build/prox -t -o cli -f ' + rundir + '/%s'%config_file[vm]
1170                                 else:
1171                                         cmd = 'sudo ' +rundir + '/prox/build/prox -e -t -o cli -f ' + rundir + '/%s'%config_file[vm]
1172                                 clients[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1))
1173                         socks_control.append(prox_launch_exit[vm])
1174                         socks.append(connect_socket(clients[-1]))
1175                         sock_type.append(mach_type[vm])
1176                         sock_monitor.append(monitor[vm])
1177 atexit.register(exit_handler)
1178 monitor_gen = monitor_sut = False
1179 background_gen_socks =[]
1180 for index, sock in enumerate(socks, start=0):
1181         if sock_type[index] == 'gen':
1182                 if sock_monitor[index]:
1183                         if monitor_gen:
1184                                 log.exception("Can only monitor 1 generator")
1185                                 raise Exception("Can only monitor 1 generator")
1186                         else:
1187                                 monitor_gen = True
1188                                 gensock_index = index
1189                 else:
1190                         background_gen_socks.append(sock)
1191         elif sock_type[index] == 'sut' and sock_monitor[index]:
1192                 if monitor_sut:
1193                         log.exception("Can only monitor 1 sut")
1194                         raise Exception("Can only monitor 1 sut")
1195                 else:
1196                         monitor_sut = True
1197                         sutsock_index = index
1198 if not(monitor_gen and monitor_sut):
1199         log.exception("No generator and/or sut to monitor")
1200         raise Exception("No generator and/or sut to monitor")
1201 if configonly:
1202         sys.exit()
1203 ####################################################
1204 # Run test cases
1205 # Best to run the flow test at the end since otherwise the tests coming after might be influenced by the big number of entries in the switch flow tables
1206 ####################################################
1207 number_of_tests = testconfig.get('DEFAULT', 'number_of_tests')
1208 for test_nr in range(1, int(number_of_tests)+1):
1209         test=testconfig.get('test%d'%test_nr,'test')
1210         log.info(test)
1211         if test == 'flowsizetest':
1212                 get_BinarySearchParams()
1213                 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1214                 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1215                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1216         elif test == 'TST009test':
1217                 # This test implements some of the testing as defined in https://docbox.etsi.org/ISG/NFV/open/Publications_pdf/Specs-Reports/NFV-TST%20009v3.2.1%20-%20GS%20-%20NFVI_Benchmarks.pdf
1218                 get_TST009SearchParams()
1219                 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1220                 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1221                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1222         elif test == 'fixed_rate':
1223                 get_FixedRateParams()
1224                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1225         elif test == 'corestats':
1226                 run_core_stats(socks)
1227         elif test == 'portstats':
1228                 run_port_stats(socks)
1229         elif test == 'impairtest':
1230                 get_BinarySearchParams()
1231                 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1232                 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1233                 run_impairtest(socks[gensock_index],socks[sutsock_index])
1234         elif test == 'irqtest':
1235                 run_irqtest(socks)
1236         elif test == 'warmuptest':
1237                 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1238                 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1239                 WARMUPSPEED = int(testconfig.get('test%d'%test_nr, 'warmupspeed'))
1240                 WARMUPTIME = int(testconfig.get('test%d'%test_nr, 'warmuptime'))
1241                 run_warmuptest(socks[gensock_index])
1242 ####################################################