VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py

   1 #!/usr/bin/python
   2
   3 ##
   4 ## Copyright (c) 2010-2020 Intel Corporation
   5 ##
   6 ## Licensed under the Apache License, Version 2.0 (the "License");
   7 ## you may not use this file except in compliance with the License.
   8 ## You may obtain a copy of the License at
   9 ##
  10 ##     http://www.apache.org/licenses/LICENSE-2.0
  11 ##
  12 ## Unless required by applicable law or agreed to in writing, software
  13 ## distributed under the License is distributed on an "AS IS" BASIS,
  14 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 ## See the License for the specific language governing permissions and
  16 ## limitations under the License.
  17 ##
  18
  19 from __future__ import print_function
  20
  21 import os
  22 import stat
  23 import sys
  24 import time
  25 import subprocess
  26 import getopt
  27 import re
  28 import logging
  29 from logging.handlers import RotatingFileHandler
  30 from logging import handlers
  31 from prox_ctrl import prox_ctrl
  32 import ConfigParser
  33 import ast
  34 import atexit
  35 import csv
  36 import requests
  37 from math import ceil
  38 inf = float("inf")
  39
  40 version="20.01.10"
  41 env = "rapid.env" #Default string for environment
  42 test_file = "basicrapid.test" #Default string for test
  43 machine_map_file = "machine.map" #Default string for machine map file
  44 loglevel="DEBUG" # sets log level for writing to file
  45 screenloglevel="INFO" # sets log level for writing to screen
  46 runtime=10 # time in seconds for 1 test run
  47 configonly = False # IF True, the system will upload all the necessary config fiels to the VMs, but not start PROX and the actual testing
  48 rundir = "/home/centos" # Directory where to find the tools in the machines running PROX
  49
  50 def usage():
  51         print("usage: runrapid    [--version] [-v]")
  52         print("                   [--env ENVIRONMENT_NAME]")
  53         print("                   [--test TEST_NAME]")
  54         print("                   [--map MACHINE_MAP_FILE]")
  55         print("                   [--runtime TIME_FOR_TEST]")
  56         print("                   [--configonly False|True]")
  57         print("                   [--log DEBUG|INFO|WARNING|ERROR|CRITICAL]")
  58         print("                   [-h] [--help]")
  59         print("")
  60         print("Command-line interface to runrapid")
  61         print("")
  62         print("optional arguments:")
  63         print("  -v,  --version                 Show program's version number and exit")
  64         print("  --env ENVIRONMENT_NAME         Parameters will be read from ENVIRONMENT_NAME. Default is %s."%env)
  65         print("  --test TEST_NAME               Test cases will be read from TEST_NAME. Default is %s."%test_file)
  66         print("  --map MACHINE_MAP_FILE Machine mapping will be read from MACHINE_MAP_FILE. Default is %s."%machine_map_file)
  67         print("  --runtime                      Specify time in seconds for 1 test run")
  68         print("  --configonly                   If this option is specified, only upload all config files to the VMs, do not run the tests")
  69         print("  --log                          Specify logging level for log file output, default is DEBUG")
  70         print("  --screenlog                    Specify logging level for screen output, default is INFO")
  71         print("  -h, --help                     Show help message and exit.")
  72         print("")
  73
  74 try:
  75         opts, args = getopt.getopt(sys.argv[1:], "vh", ["version","help", "env=", "test=", "map=", "runtime=","configonly","log=","screenlog="])
  76 except getopt.GetoptError as err:
  77         print("===========================================")
  78         print(str(err))
  79         print("===========================================")
  80         usage()
  81         sys.exit(2)
  82 if args:
  83         usage()
  84         sys.exit(2)
  85 for opt, arg in opts:
  86         if opt in ["-h", "--help"]:
  87                 usage()
  88                 sys.exit()
  89         if opt in ["-v", "--version"]:
  90                 print("Rapid Automated Performance Indication for Dataplane "+version)
  91                 sys.exit()
  92         if opt in ["--env"]:
  93                 env = arg
  94         if opt in ["--test"]:
  95                 test_file = arg
  96         if opt in ["--map"]:
  97                 machine_map_file = arg
  98         if opt in ["--runtime"]:
  99                 runtime = arg
 100         if opt in ["--configonly"]:
 101                 configonly = True
 102                 print('No actual runs, only uploading configuration files')
 103         if opt in ["--log"]:
 104                 loglevel = arg
 105                 print ("Log level: "+ loglevel)
 106         if opt in ["--screenlog"]:
 107                 screenloglevel = arg
 108                 print ("Screen Log level: "+ screenloglevel)
 109
 110 print ("Using '"+env+"' as name for the environment")
 111 print ("Using '"+test_file+"' for test case definition")
 112 print ("Using '"+machine_map_file+"' for machine mapping")
 113 print ("Runtime: "+ str(runtime))
 114
 115 class bcolors:
 116         HEADER = '\033[95m'
 117         OKBLUE = '\033[94m'
 118         OKGREEN = '\033[92m'
 119         WARNING = '\033[93m'
 120         FAIL = '\033[91m'
 121         ENDC = '\033[0m'
 122         BOLD = '\033[1m'
 123         UNDERLINE = '\033[4m'
 124         FLASH = '\033[5m'
 125
 126 # create formatters
 127 screen_formatter = logging.Formatter("%(message)s")
 128 file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 129
 130 # get a top-level logger,
 131 # set its log level,
 132 # BUT PREVENT IT from propagating messages to the root logger
 133 #
 134 log = logging.getLogger()
 135 numeric_level = getattr(logging, loglevel.upper(), None)
 136 if not isinstance(numeric_level, int):
 137         raise ValueError('Invalid log level: %s' % loglevel)
 138 log.setLevel(numeric_level)
 139 log.propagate = 0
 140
 141 # create a console handler
 142 # and set its log level to the command-line option
 143 #
 144 console_handler = logging.StreamHandler(sys.stdout)
 145 #console_handler.setLevel(logging.INFO)
 146 numeric_screenlevel = getattr(logging, screenloglevel.upper(), None)
 147 if not isinstance(numeric_screenlevel, int):
 148         raise ValueError('Invalid screenlog level: %s' % screenloglevel)
 149 console_handler.setLevel(numeric_screenlevel)
 150 console_handler.setFormatter(screen_formatter)
 151
 152 # create a file handler
 153 # and set its log level
 154 #
 155 log_file = 'RUN{}.{}.log'.format(env,test_file)
 156 file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10)
 157 #file_handler = log.handlers.TimedRotatingFileHandler(log_file, 'D', 1, 5)
 158 file_handler.setLevel(numeric_level)
 159 file_handler.setFormatter(file_formatter)
 160
 161 # add handlers to the logger
 162 #
 163 log.addHandler(file_handler)
 164 log.addHandler(console_handler)
 165
 166 # Check if log exists and should therefore be rolled
 167 needRoll = os.path.isfile(log_file)
 168
 169
 170 # This is a stale log, so roll it
 171 if needRoll:
 172         # Add timestamp
 173         log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime())
 174
 175         # Roll over on application start
 176         log.handlers[0].doRollover()
 177
 178 # Add timestamp
 179 log.debug('\n---------\nLog started on %s.\n---------\n' % time.asctime())
 180
 181 log.debug("runrapid.py version: "+version)
 182 #========================================================================
 183 def connect_socket(client):
 184         attempts = 1
 185         log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 186         sock = None
 187         while True:
 188                 sock = client.prox_sock()
 189                 if sock is not None:
 190                         break
 191                 attempts += 1
 192                 if attempts > 20:
 193                         log.exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 194                         raise Exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 195                 time.sleep(2)
 196                 log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 197         log.info("Connected to PROX on %s" % client.ip())
 198         return sock
 199
 200 def connect_client(client):
 201         attempts = 1
 202         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 203         while True:
 204                 try:
 205                         client.connect()
 206                         break
 207                 except RuntimeWarning, ex:
 208                         attempts += 1
 209                         if attempts > 20:
 210                                 log.exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 211                                 raise Exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 212                         time.sleep(2)
 213                         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 214         log.debug("Connected to VM on %s" % client.ip())
 215
 216 def report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,tx,rx,tot_drop,elapsed_time,speed_prefix='',lat_avg_prefix='',lat_perc_prefix='',lat_max_prefix='',abs_drop_rate_prefix='',drop_rate_prefix=''):
 217         if flow_number < 0:
 218                 flow_number_str = '| ({:>4}) |'.format(abs(flow_number))
 219         else:
 220                 flow_number_str = '|{:>7} |'.format(flow_number)
 221         if pps_req_tx == None:
 222                 pps_req_tx_str = '{0: >14}'.format('   NA     |')
 223         else:
 224                 pps_req_tx_str = '{:>7.3f} Mpps |'.format(pps_req_tx)
 225         if pps_tx == None:
 226                 pps_tx_str = '{0: >14}'.format('   NA     |')
 227         else:
 228                 pps_tx_str = '{:>7.3f} Mpps |'.format(pps_tx)
 229         if pps_sut_tx == None:
 230                 pps_sut_tx_str = '{0: >14}'.format('   NA     |')
 231         else:
 232                 pps_sut_tx_str = '{:>7.3f} Mpps |'.format(pps_sut_tx)
 233         if pps_rx == None:
 234                 pps_rx_str = '{0: >24|}'.format('NA        ')
 235         else:
 236                 pps_rx_str = bcolors.OKBLUE + '{:>4.1f} Gb/s |{:7.3f} Mpps {}|'.format(get_speed(pps_rx,size),pps_rx,bcolors.ENDC)
 237         if tot_drop == None:
 238                 tot_drop_str = ' |       NA  | '
 239         else:
 240                 tot_drop_str = ' | {:>9.0f} | '.format(tot_drop)
 241         if lat_perc == None:
 242                 lat_perc_str = ' |{:^10.10}|'.format('NA')
 243         elif lat_perc_max == True:
 244                 lat_perc_str = ' |>{}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC)
 245         else:
 246                 lat_perc_str = ' | {}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC)
 247         if elapsed_time == None:
 248                 elapsed_time_str = ' NA |'
 249         else:
 250                 elapsed_time_str = '{:>3.0f} |'.format(elapsed_time)
 251         return(flow_number_str + '{:>5.1f}'.format(speed) + '% '+speed_prefix +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+ pps_req_tx_str + pps_tx_str + bcolors.ENDC + pps_sut_tx_str + pps_rx_str +lat_avg_prefix+ ' {:>5.0f}'.format(lat_avg)+' us'+lat_perc_str+lat_max_prefix+'{:>6.0f}'.format(lat_max)+' us | ' + '{:>9.0f}'.format(tx) + ' | {:>9.0f}'.format(rx) + ' | '+ abs_drop_rate_prefix+ '{:>9.0f}'.format(tx-rx) + tot_drop_str +drop_rate_prefix+ '{:>5.2f}'.format(float(tx-rx)/tx)  +bcolors.ENDC+' |' + elapsed_time_str)
 252
 253 def run_iteration(gensock, sutsock, requested_duration,flow_number,size,speed):
 254         r = 0;
 255         sleep_time = 2
 256         while (r < TST009_MAXr):
 257                 time.sleep(sleep_time)
 258                 # Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
 259                 t1_rx, t1_non_dp_rx, t1_tx, t1_non_dp_tx, t1_drop, t1_tx_fail, t1_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
 260                 t1_dp_rx = t1_rx - t1_non_dp_rx
 261                 t1_dp_tx = t1_tx - t1_non_dp_tx
 262                 gensock.start(gencores)
 263                 time.sleep(2) ## Needs to be 2 seconds since this the time that PROX uses to refresh the stats. Note that this can be changed in PROX!! Don't do it.
 264                 if sutsock!='none':
 265                         t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 266                         ##t2_sut_rx = t2_sut_rx - t2_sut_non_dp_rx
 267                         ##t2_sut_tx = t2_sut_tx - t2_sut_non_dp_tx
 268                 t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 269                 tx = t2_tx - t1_tx
 270                 dp_tx =  tx - (t2_non_dp_tx - t1_non_dp_tx )
 271                 dp_rx =  t2_rx - t1_rx - (t2_non_dp_rx - t1_non_dp_rx)
 272                 tot_dp_drop = dp_tx - dp_rx
 273                 if tx == 0:
 274                         log.critical("TX = 0. Test interrupted since no packet has been sent.")
 275                         raise Exception("TX = 0")
 276                 if dp_tx == 0:
 277                         log.critical("Only non-dataplane packets (e.g. ARP) sent. Test interrupted since no packet has been sent.")
 278                         raise Exception("Only non-dataplane packets (e.g. ARP) sent")
 279                 # Ask PROX to calibrate the bucket size once we have a PROX function to do this.
 280                 # Measure latency statistics per second
 281                 lat_min, lat_max, lat_avg, used_avg, t2_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
 282                 lat_samples = sum(buckets)
 283                 sample_count = 0
 284                 for sample_percentile, bucket in enumerate(buckets,start=1):
 285                         sample_count += bucket
 286                         if sample_count > (lat_samples * LAT_PERCENTILE):
 287                                 break
 288                 if sample_percentile == len(buckets):
 289                         percentile_max = True
 290                 else:
 291                         percentile_max = False
 292                 sample_percentile = sample_percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
 293                 if test == 'fixed_rate':
 294                         log.info(report_result(flow_number,size,speed,None,None,None,None,lat_avg,sample_percentile,percentile_max,lat_max, dp_tx, dp_rx , None, None))
 295                 tot_rx = tot_non_dp_rx = tot_tx = tot_non_dp_tx = tot_drop = 0
 296                 lat_avg = used_avg = 0
 297                 buckets_total = [0] * 128
 298                 tot_lat_samples = 0
 299                 tot_lat_measurement_duration = float(0)
 300                 tot_core_measurement_duration = float(0)
 301                 tot_sut_core_measurement_duration = float(0)
 302                 tot_sut_rx = tot_sut_non_dp_rx = tot_sut_tx = tot_sut_non_dp_tx = tot_sut_drop = tot_sut_tx_fail = tot_sut_tsc = 0
 303                 lat_avail = core_avail = sut_avail = False
 304                 while (tot_core_measurement_duration - float(requested_duration) <= 0.1) or (tot_lat_measurement_duration - float(requested_duration) <= 0.1):
 305                         time.sleep(0.5)
 306                         lat_min_sample, lat_max_sample, lat_avg_sample, used_sample, t3_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
 307                         # Get statistics after some execution time
 308                         if t3_lat_tsc != t2_lat_tsc:
 309                                 single_lat_measurement_duration = (t3_lat_tsc - t2_lat_tsc) * 1.0 / lat_hz  # time difference between the 2 measurements, expressed in seconds.
 310                                 # A second has passed in between to lat_stats requests. Hence we need to process the results
 311                                 tot_lat_measurement_duration = tot_lat_measurement_duration + single_lat_measurement_duration
 312                                 if lat_min > lat_min_sample:
 313                                         lat_min = lat_min_sample
 314                                 if lat_max < lat_max_sample:
 315                                         lat_max = lat_max_sample
 316                                 lat_avg = lat_avg + lat_avg_sample * single_lat_measurement_duration # Sometimes, There is more than 1 second between 2 lat_stats. Hence we will take the latest measurement
 317                                 used_avg = used_avg + used_sample * single_lat_measurement_duration  # and give it more weigth.
 318                                 lat_samples = sum(buckets)
 319                                 tot_lat_samples += lat_samples
 320                                 sample_count = 0
 321                                 for sample_percentile, bucket in enumerate(buckets,start=1):
 322                                         sample_count += bucket
 323                                         if sample_count > lat_samples * LAT_PERCENTILE:
 324                                                 break
 325                                 if sample_percentile == len(buckets):
 326                                         percentile_max = True
 327                                 else:
 328                                         percentile_max = False
 329                                 sample_percentile = sample_percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
 330                                 buckets_total = [buckets_total[i] + buckets[i] for i in range(len(buckets_total))]
 331                                 t2_lat_tsc = t3_lat_tsc
 332                                 lat_avail = True
 333                         t3_rx, t3_non_dp_rx, t3_tx, t3_non_dp_tx, t3_drop, t3_tx_fail, t3_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 334                         if t3_tsc != t2_tsc:
 335                                 single_core_measurement_duration = (t3_tsc - t2_tsc) * 1.0 / tsc_hz  # time difference between the 2 measurements, expressed in seconds.
 336                                 tot_core_measurement_duration = tot_core_measurement_duration + single_core_measurement_duration
 337                                 delta_rx = t3_rx - t2_rx
 338                                 tot_rx += delta_rx
 339                                 delta_non_dp_rx = t3_non_dp_rx - t2_non_dp_rx
 340                                 tot_non_dp_rx += delta_non_dp_rx
 341                                 delta_tx = t3_tx - t2_tx
 342                                 tot_tx += delta_tx
 343                                 delta_non_dp_tx = t3_non_dp_tx - t2_non_dp_tx
 344                                 tot_non_dp_tx += delta_non_dp_tx
 345                                 delta_dp_tx = delta_tx -delta_non_dp_tx
 346                                 delta_dp_rx = delta_rx -delta_non_dp_rx
 347                                 delta_dp_drop = delta_dp_tx - delta_dp_rx
 348                                 tot_dp_drop += delta_dp_drop
 349                                 delta_drop = t3_drop - t2_drop
 350                                 tot_drop += delta_drop
 351                                 t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc = t3_rx, t3_non_dp_rx, t3_tx, t3_non_dp_tx, t3_drop, t3_tx_fail, t3_tsc
 352                                 core_avail = True
 353                         if sutsock!='none':
 354                                 t3_sut_rx, t3_sut_non_dp_rx, t3_sut_tx, t3_sut_non_dp_tx, t3_sut_drop, t3_sut_tx_fail, t3_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 355                                 if t3_sut_tsc != t2_sut_tsc:
 356                                         single_sut_core_measurement_duration = (t3_sut_tsc - t2_sut_tsc) * 1.0 / tsc_hz  # time difference between the 2 measurements, expressed in seconds.
 357                                         tot_sut_core_measurement_duration = tot_sut_core_measurement_duration + single_sut_core_measurement_duration
 358                                         tot_sut_rx += t3_sut_rx - t2_sut_rx
 359                                         tot_sut_non_dp_rx += t3_sut_non_dp_rx - t2_sut_non_dp_rx
 360                                         delta_sut_tx = t3_sut_tx - t2_sut_tx
 361                                         tot_sut_tx += delta_sut_tx
 362                                         delta_sut_non_dp_tx = t3_sut_non_dp_tx - t2_sut_non_dp_tx
 363                                         tot_sut_non_dp_tx += delta_sut_non_dp_tx
 364                                         t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc = t3_sut_rx, t3_sut_non_dp_rx, t3_sut_tx, t3_sut_non_dp_tx, t3_sut_drop, t3_sut_tx_fail, t3_sut_tsc
 365                                         sut_avail = True
 366                         if test == 'fixed_rate':
 367                                 if lat_avail == core_avail == True:
 368                                         lat_avail = core_avail = False
 369                                         pps_req_tx = (delta_tx + delta_drop - delta_rx)/single_core_measurement_duration/1000000
 370                                         pps_tx = delta_tx/single_core_measurement_duration/1000000
 371                                         if sutsock!='none' and sut_avail:
 372                                                 pps_sut_tx = delta_sut_tx/single_sut_core_measurement_duration/1000000
 373                                                 sut_avail = False
 374                                         else:
 375                                                 pps_sut_tx = None
 376                                         pps_rx = delta_rx/single_core_measurement_duration/1000000
 377                                         log.info(report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg_sample,sample_percentile,percentile_max,lat_max_sample,delta_dp_tx,delta_dp_rx,tot_dp_drop,single_core_measurement_duration))
 378                 #Stop generating
 379                 gensock.stop(gencores)
 380                 r += 1
 381                 lat_avg = lat_avg / float(tot_lat_measurement_duration)
 382                 used_avg = used_avg / float(tot_lat_measurement_duration)
 383                 t4_tsc = t2_tsc
 384                 while t4_tsc == t2_tsc:
 385                         t4_rx, t4_non_dp_rx, t4_tx, t4_non_dp_tx, t4_drop, t4_tx_fail, t4_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
 386                 if test == 'fixed_rate':
 387                         t4_lat_tsc = t2_lat_tsc
 388                         while t4_lat_tsc == t2_lat_tsc:
 389                                 lat_min_sample, lat_max_sample, lat_avg_sample, used_sample, t4_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
 390                         sample_count = 0
 391                         lat_samples = sum(buckets)
 392                         for percentile, bucket in enumerate(buckets,start=1):
 393                                 sample_count += bucket
 394                                 if sample_count > lat_samples * LAT_PERCENTILE:
 395                                         break
 396                         if percentile == len(buckets):
 397                                 percentile_max = True
 398                         else:
 399                                 percentile_max = False
 400                         percentile = percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
 401                         lat_max = lat_max_sample
 402                         lat_avg = lat_avg_sample
 403                         delta_rx = t4_rx - t2_rx
 404                         delta_non_dp_rx = t4_non_dp_rx - t2_non_dp_rx
 405                         delta_tx = t4_tx - t2_tx
 406                         delta_non_dp_tx = t4_non_dp_tx - t2_non_dp_tx
 407                         delta_dp_tx = delta_tx -delta_non_dp_tx
 408                         delta_dp_rx = delta_rx -delta_non_dp_rx
 409                         dp_tx = delta_dp_tx
 410                         dp_rx = delta_dp_rx
 411                         tot_dp_drop += delta_dp_tx - delta_dp_rx
 412                         pps_req_tx = None
 413                         pps_tx = None
 414                         pps_sut_tx = None
 415                         pps_rx = None
 416                         drop_rate = 100.0*(dp_tx-dp_rx)/dp_tx
 417                         tot_core_measurement_duration = None
 418                         break ## Not really needed since the while loop will stop when evaluating the value of r
 419                 else:
 420                         sample_count = 0
 421                         for percentile, bucket in enumerate(buckets_total,start=1):
 422                                 sample_count += bucket
 423                                 if sample_count > tot_lat_samples * LAT_PERCENTILE:
 424                                         break
 425                         if percentile == len(buckets):
 426                                 percentile_max = True
 427                         else:
 428                                 percentile_max = False
 429                         percentile = percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
 430                         pps_req_tx = (tot_tx + tot_drop - tot_rx)/tot_core_measurement_duration/1000000.0 # tot_drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 431                         pps_tx = tot_tx/tot_core_measurement_duration/1000000.0 # tot_tx is all generated packets actually accepted by the interface
 432                         pps_rx = tot_rx/tot_core_measurement_duration/1000000.0 # tot_rx is all packets received by the nop task = all packets received in the gen VM
 433                         if sutsock!='none' and sut_avail:
 434                                 pps_sut_tx = tot_sut_tx / tot_sut_core_measurement_duration / 1000000.0
 435                         else:
 436                                 pps_sut_tx = None
 437                         dp_tx = (t4_tx - t1_tx) - (t4_non_dp_tx - t1_non_dp_tx)
 438                         dp_rx = (t4_rx - t1_rx) - (t4_non_dp_rx - t1_non_dp_rx)
 439                         tot_dp_drop = dp_tx - dp_rx
 440                         drop_rate = 100.0*tot_dp_drop/dp_tx
 441                         if ((drop_rate < DROP_RATE_TRESHOLD) or (tot_dp_drop == DROP_RATE_TRESHOLD ==0) or (tot_dp_drop > TST009_MAXz)):
 442                                 break
 443         return(pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,percentile,percentile_max,lat_max,dp_tx,dp_rx,tot_dp_drop,(t4_tx_fail - t1_tx_fail),drop_rate,lat_min,used_avg,r,tot_core_measurement_duration)
 444
 445 def new_speed(speed,size,success):
 446         if test == 'fixed_rate':
 447                 return (STARTSPEED)
 448         elif TST009:
 449                 global TST009_m
 450                 global TST009_L
 451                 global TST009_R
 452                 if success:
 453                         TST009_L = TST009_m + 1
 454                 else:
 455                         TST009_R = max(TST009_m - 1, TST009_L)
 456                 TST009_m = int ((TST009_L + TST009_R)/2)
 457                 return (get_percentageof10Gbps(TST009_S[TST009_m],size))
 458         else:
 459                 global minspeed
 460                 global maxspeed
 461                 if success:
 462                         minspeed = speed
 463                 else:
 464                         maxspeed = speed
 465                 return ((minspeed + maxspeed)/2.0)
 466
 467 def get_start_speed_and_init(size):
 468         if test == 'fixed_rate':
 469                 return (STARTSPEED)
 470         elif TST009:
 471                 global TST009_L
 472                 global TST009_R
 473                 global TST009_m
 474                 TST009_L = 0
 475                 TST009_R = TST009_n - 1
 476                 TST009_m = int((TST009_L + TST009_R) / 2)
 477                 return (get_percentageof10Gbps(TST009_S[TST009_m],size))
 478         else:
 479                 global minspeed
 480                 global maxspeed
 481                 minspeed = 0
 482                 maxspeed = STARTSPEED
 483                 return (STARTSPEED)
 484
 485 def resolution_achieved():
 486         if test == 'fixed_rate':
 487                 return (True)
 488         elif TST009:
 489                 return (TST009_L == TST009_R)
 490         else:
 491                 return ((maxspeed - minspeed) <= ACCURACY)
 492
 493 def get_percentageof10Gbps(pps_speed,size):
 494         # speed is given in pps, returning % of 10Gb/s
 495         return (pps_speed / 1000000.0 * 0.08 * (size+24))
 496
 497 def get_pps(speed,size):
 498         # speed is given in % of 10Gb/s, returning Mpps
 499         return (speed * 100.0 / (8*(size+24)))
 500
 501 def get_speed(packet_speed,size):
 502         # return speed in Gb/s
 503         return (packet_speed / 1000.0 * (8*(size+24)))
 504
 505 def set_background_flows(source_port,destination_port):
 506         for sock in background_gen_socks:
 507                 sock.set_random(gencores,0,34,source_port,2)
 508                 sock.set_random(gencores,0,36,destination_port,2)
 509
 510 def set_background_speed(speed):
 511         for sock in background_gen_socks:
 512                 sock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
 513
 514 def start_background_traffic():
 515         for sock in background_gen_socks:
 516                 sock.start(gencores+latcores)
 517
 518 def stop_background_traffic():
 519         for sock in background_gen_socks:
 520                 sock.stop(gencores+latcores)
 521
 522 def run_flow_size_test(gensock,sutsock):
 523         global fieldnames
 524         global writer
 525         #fieldnames = ['Flows','PacketSize','Gbps','Mpps','AvgLatency','MaxLatency','PacketsDropped','PacketDropRate']
 526         fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Sent','Received','Lost','LostTotal']
 527         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 528         writer.writeheader()
 529         gensock.start(latcores)
 530         for size in packet_size_list:
 531                 size = size-4
 532                 gensock.set_size(gencores,0,size) # This is setting the frame size
 533                 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 534                 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 535                 # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 536                 if background_gen_socks:
 537                         backgroundinfo = '{}Running {} x background traffic not represented in the table{}'.format(bcolors.FLASH,len(background_gen_socks),bcolors.ENDC)
 538                 else:
 539                         backgroundinfo = '{}{}'.format(bcolors.FLASH,bcolors.ENDC)
 540                 log.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 541                 log.info('| UDP, {:>5} bytes, different number of flows by randomizing SRC & DST UDP port. {:116.116}|'.format((size+4),backgroundinfo))
 542                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 543                 log.info('| Flows  | Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core           | Avg. Lat.|{:.0f} Pcentil| Max. Lat.|   Sent    |  Received |    Lost   | Total Lost|L.Ratio|Time|'.format(LAT_PERCENTILE*100))
 544                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 545                 for flow_number in flow_size_list:
 546                         attempts = 0
 547                         gensock.reset_stats()
 548                         if sutsock!='none':
 549                                 sutsock.reset_stats()
 550                         source_port,destination_port = flows[flow_number]
 551                         gensock.set_random(gencores,0,34,source_port,2)
 552                         gensock.set_random(gencores,0,36,destination_port,2)
 553                         set_background_flows(source_port,destination_port)
 554                         endspeed = None
 555                         speed = get_start_speed_and_init(size)
 556                         while True:
 557                                 attempts += 1
 558                                 endwarning = False
 559                                 print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 560                                 sys.stdout.flush()
 561                                 # Start generating packets at requested speed (in % of a 10Gb/s link)
 562                                 gensock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
 563                                 set_background_speed(speed)
 564                                 start_background_traffic()
 565                                 ##time.sleep(1)
 566                                 # Get statistics now that the generation is stable and initial ARP messages are dealt with
 567                                 pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc , lat_perc_max, lat_max, abs_tx,abs_rx,abs_dropped, abs_tx_fail, drop_rate, lat_min, lat_used, r, actual_duration = run_iteration(gensock,sutsock,float(runtime),flow_number,size,speed)
 568                                 stop_background_traffic()
 569                                 if r > 1:
 570                                         retry_warning = bcolors.WARNING + ' {:1} retries needed'.format(r) +  bcolors.ENDC
 571                                 else:
 572                                         retry_warning = ''
 573                                 # Drop rate is expressed in percentage. lat_used is a ratio (0 to 1). The sum of these 2 should be 100%.
 574                                 # If the sum is lower than 95, it means that more than 5% of the latency measurements where dropped for accuracy reasons.
 575                                 if (drop_rate + lat_used * 100) < 95:
 576                                         lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 577                                 else:
 578                                         lat_warning = ''
 579                                 if test == 'fixed_rate':
 580                                         endspeed = speed
 581                                         endpps_req_tx = None
 582                                         endpps_tx = None
 583                                         endpps_sut_tx = None
 584                                         endpps_rx = None
 585                                         endlat_avg = lat_avg
 586                                         endlat_perc = lat_perc
 587                                         endlat_perc_max = lat_perc_max
 588                                         endlat_max = lat_max
 589                                         endabs_dropped = abs_dropped
 590                                         enddrop_rate = drop_rate
 591                                         endabs_tx = abs_tx
 592                                         endabs_rx = abs_rx
 593                                         if lat_warning or gen_warning or retry_warning:
 594                                                 endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning + gen_warning)
 595                                         success = True
 596                                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 597                                 # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold
 598                                 # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
 599                                 # This can be specified by putting 0 in the .test file
 600                                 elif ((drop_rate < DROP_RATE_TRESHOLD) or (abs_dropped==DROP_RATE_TRESHOLD==0)) and (lat_avg< LAT_AVG_TRESHOLD) and (lat_perc< LAT_PERC_TRESHOLD) and (lat_max < LAT_MAX_TRESHOLD):
 601                                         if ((get_pps(speed,size) - pps_tx)/get_pps(speed,size))>0.01:
 602                                                 speed_prefix = bcolors.WARNING
 603                                                 if abs_tx_fail > 0:
 604                                                         gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
 605                                                 else:
 606                                                         gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(get_pps(speed,size), pps_tx) + bcolors.ENDC
 607                                         else:
 608                                                 speed_prefix = bcolors.ENDC
 609                                                 gen_warning = ''
 610                                         endspeed = speed
 611                                         endspeed_prefix = speed_prefix
 612                                         endpps_req_tx = pps_req_tx
 613                                         endpps_tx = pps_tx
 614                                         endpps_sut_tx = pps_sut_tx
 615                                         endpps_rx = pps_rx
 616                                         endlat_avg = lat_avg
 617                                         endlat_perc = lat_perc
 618                                         endlat_perc_max = lat_perc_max
 619                                         endlat_max = lat_max
 620                                         endabs_dropped = None
 621                                         enddrop_rate = drop_rate
 622                                         endabs_tx = abs_tx
 623                                         endabs_rx = abs_rx
 624                                         if lat_warning or gen_warning or retry_warning:
 625                                                 endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning + gen_warning)
 626                                         success = True
 627                                         success_message=' SUCCESS'
 628                                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 629                                         log.debug(report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
 630                                 else:
 631                                         success_message=' FAILED'
 632                                         gen_warning = ''
 633                                         abs_drop_rate_prefix = bcolors.ENDC
 634                                         if ((abs_dropped>0) and (DROP_RATE_TRESHOLD ==0)):
 635                                                 abs_drop_rate_prefix = bcolors.FAIL
 636                                         if (drop_rate < DROP_RATE_TRESHOLD):
 637                                                 drop_rate_prefix = bcolors.ENDC
 638                                         else:
 639                                                 drop_rate_prefix = bcolors.FAIL
 640                                         if (lat_avg< LAT_AVG_TRESHOLD):
 641                                                 lat_avg_prefix = bcolors.ENDC
 642                                         else:
 643                                                 lat_avg_prefix = bcolors.FAIL
 644                                         if (lat_perc< LAT_PERC_TRESHOLD):
 645                                                 lat_perc_prefix = bcolors.ENDC
 646                                         else:
 647                                                 lat_perc_prefix = bcolors.FAIL
 648                                         if (lat_max< LAT_MAX_TRESHOLD):
 649                                                 lat_max_prefix = bcolors.ENDC
 650                                         else:
 651                                                 lat_max_prefix = bcolors.FAIL
 652                                         if (((get_pps(speed,size) - pps_tx)/get_pps(speed,size))<0.001):
 653                                                 speed_prefix = bcolors.ENDC
 654                                         else:
 655                                                 speed_prefix = bcolors.FAIL
 656                                         success = False
 657                                         log.debug(report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
 658                                 speed = new_speed(speed, size, success)
 659                                 if resolution_achieved():
 660                                         break
 661                         if endspeed !=  None:
 662                                 speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 663                                 log.info(report_result(flow_number,size,endspeed,endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_perc,endlat_perc_max,endlat_max,endabs_tx,endabs_rx,endabs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix))
 664                                 if endwarning:
 665                                         log.info (endwarning)
 666                                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 667                                 writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':get_pps(endspeed,size),'GeneratedPPS':endpps_req_tx,'SentPPS':endpps_tx,'ForwardedPPS':endpps_sut_tx,'ReceivedPPS':endpps_rx,'AvgLatencyUSEC':endlat_avg,'MaxLatencyUSEC':endlat_max,'Sent':endabs_tx,'Received':endabs_rx,'Lost':endabs_dropped,'LostTotal':endabs_dropped})
 668                                 if PushGateway:
 669                                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
 670                                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nSent {}\nReceived {}\nLost {}\nLostTotal {}\n'.format(flow_number,size+4,get_pps(endspeed,size),endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_max,endabs_tx,endabs_rx,endabs_Dropped,endabs_dropped)
 671                                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 672                                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 673                         else:
 674                                 log.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
 675         gensock.stop(latcores)
 676
 677 def run_core_stats(socks):
 678         fieldnames = ['PROXID','Time','Received','Sent','NonDPReceived','NonDPSent','Delta','NonDPDelta','Dropped']
 679         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 680         writer.writeheader()
 681         log.info("+------------------------------------------------------------------------------------------------------------------+")
 682         log.info("| Measuring core statistics on 1 or more PROX instances                                                            |")
 683         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 684         log.info("| PROX ID   |    Time   |    RX      |     TX     | non DP RX  | non DP TX  |   TX - RX  | nonDP TX-RX|  DROP TOT  |")
 685         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 686         for sock in socks:
 687                 sock.reset_stats()
 688         duration = float(runtime)
 689         tot_drop = []
 690         old_rx = []; old_non_dp_rx = []; old_tx = []; old_non_dp_tx = []; old_drop = []; old_tx_fail = []; old_tsc = []
 691         new_rx = []; new_non_dp_rx = []; new_tx = []; new_non_dp_tx = []; new_drop = []; new_tx_fail = []; new_tsc = []
 692         sockets_to_go = len (socks)
 693         for i,sock in enumerate(socks,start=0):
 694                 tot_drop.append(0)
 695                 old_rx.append(0); old_non_dp_rx.append(0); old_tx.append(0); old_non_dp_tx.append(0); old_drop.append(0); old_tx_fail.append(0); old_tsc.append(0)
 696                 old_rx[-1], old_non_dp_rx[-1], old_tx[-1], old_non_dp_tx[-1], old_drop[-1], old_tx_fail[-1], old_tsc[-1], tsc_hz = sock.core_stats(cores[i],tasks)
 697                 new_rx.append(0); new_non_dp_rx.append(0); new_tx.append(0); new_non_dp_tx.append(0); new_drop.append(0); new_tx_fail.append(0); new_tsc.append(0)
 698         while (duration > 0):
 699                 time.sleep(0.5)
 700                 # Get statistics after some execution time
 701                 for i,sock in enumerate(socks,start=0):
 702                         new_rx[i], new_non_dp_rx[i], new_tx[i], new_non_dp_tx[i], new_drop[i], new_tx_fail[i], new_tsc[i], tsc_hz = sock.core_stats(cores[i],tasks)
 703                         drop = new_drop[i]-old_drop[i]
 704                         rx = new_rx[i] - old_rx[i]
 705                         tx = new_tx[i] - old_tx[i]
 706                         non_dp_rx = new_non_dp_rx[i] - old_non_dp_rx[i]
 707                         non_dp_tx = new_non_dp_tx[i] - old_non_dp_tx[i]
 708                         tsc = new_tsc[i] - old_tsc[i]
 709                         if tsc == 0 :
 710                                 continue
 711                         sockets_to_go -= 1
 712                         old_drop[i] = new_drop[i]
 713                         old_rx[i] = new_rx[i]
 714                         old_tx[i] = new_tx[i]
 715                         old_non_dp_rx[i] = new_non_dp_rx[i]
 716                         old_non_dp_tx[i] = new_non_dp_tx[i]
 717                         old_tsc[i] = new_tsc[i]
 718                         tot_drop[i] = tot_drop[i] + tx - rx
 719                         log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(non_dp_rx)+' | '+'{:>10.0f}'.format(non_dp_tx)+' | ' + '{:>10.0f}'.format(tx-rx) + ' | '+ '{:>10.0f}'.format(non_dp_tx-non_dp_rx) + ' | '+'{:>10.0f}'.format(tot_drop[i]) +' |')
 720                         writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NonDPReceived':non_dp_rx,'NonDPSent':non_dp_tx,'Delta':tx-rx,'NonDPDelta':non_dp_tx-non_dp_rx,'Dropped':tot_drop[i]})
 721                         if PushGateway:
 722                                 URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
 723                                 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNonDPReceived {}\nNonDPSent {}\nDelta {}\nNonDPDelta {}\nDropped {}\n'.format(i,duration,rx,tx,non_dp_rx,non_dp_tx,tx-rx,non_dp_tx-non_dp_rx,tot_drop[i])
 724                                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 725                                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
 726                         if sockets_to_go == 0:
 727                                 duration = duration - 1
 728                                 sockets_to_go = len (socks)
 729         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 730
 731 def run_port_stats(socks):
 732         fieldnames = ['PROXID','Time','Received','Sent','NoMbufs','iErrMiss']
 733         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 734         writer.writeheader()
 735         log.info("+---------------------------------------------------------------------------+")
 736         log.info("| Measuring port statistics on 1 or more PROX instances                     |")
 737         log.info("+-----------+-----------+------------+------------+------------+------------+")
 738         log.info("| PROX ID   |    Time   |    RX      |     TX     | no MBUFS   | ierr&imiss |")
 739         log.info("+-----------+-----------+------------+------------+------------+------------+")
 740         for sock in socks:
 741                 sock.reset_stats()
 742         duration = float(runtime)
 743         old_rx = []; old_tx = []; old_no_mbufs = []; old_errors = []; old_tsc = []
 744         new_rx = []; new_tx = []; new_no_mbufs = []; new_errors = []; new_tsc = []
 745         sockets_to_go = len (socks)
 746         for i,sock in enumerate(socks,start=0):
 747                 old_rx.append(0); old_tx.append(0); old_no_mbufs.append(0); old_errors.append(0); old_tsc.append(0)
 748                 old_rx[-1], old_tx[-1], old_no_mbufs[-1], old_errors[-1], old_tsc[-1] = sock.multi_port_stats(ports[i])
 749                 new_rx.append(0); new_tx.append(0); new_no_mbufs.append(0); new_errors.append(0); new_tsc.append(0)
 750         while (duration > 0):
 751                 time.sleep(0.5)
 752                 # Get statistics after some execution time
 753                 for i,sock in enumerate(socks,start=0):
 754                         new_rx[i], new_tx[i], new_no_mbufs[i], new_errors[i], new_tsc[i] = sock.multi_port_stats(ports[i])
 755                         rx = new_rx[i] - old_rx[i]
 756                         tx = new_tx[i] - old_tx[i]
 757                         no_mbufs = new_no_mbufs[i] - old_no_mbufs[i]
 758                         errors = new_errors[i] - old_errors[i]
 759                         tsc = new_tsc[i] - old_tsc[i]
 760                         if tsc == 0 :
 761                                 continue
 762                         sockets_to_go -= 1
 763                         old_rx[i] = new_rx[i]
 764                         old_tx[i] = new_tx[i]
 765                         old_no_mbufs[i] = new_no_mbufs[i]
 766                         old_errors[i] = new_errors[i]
 767                         old_tsc[i] = new_tsc[i]
 768                         log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(no_mbufs)+' | '+'{:>10.0f}'.format(errors)+' |')
 769                         writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NoMbufs':no_mbufs,'iErrMiss':errors})
 770                         if PushGateway:
 771                                 URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
 772                                 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNoMbufs {}\niErrMiss {}\n'.format(i,duration,rx,tx,no_mbufs,errors)
 773                                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 774                                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
 775                         if sockets_to_go == 0:
 776                                 duration = duration - 1
 777                                 sockets_to_go = len (socks)
 778         log.info("+-----------+-----------+------------+------------+------------+------------+")
 779
 780 def run_irqtest(socks):
 781         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 782         log.info("| Measuring time probably spent dealing with an interrupt. Interrupting DPDK cores for more than 50us might be problematic   ")
 783         log.info("| and result in packet loss. The first row shows the interrupted time buckets: first number is the bucket between 0us and    ")
 784         log.info("| that number expressed in us and so on. The numbers in the other rows show how many times per second, the program was       ")
 785         log.info("| interrupted for a time as specified by its bucket. '0' is printed when there are no interrupts in this bucket throughout   ")
 786         log.info("| the duration of the test. 0.00 means there were interrupts in this bucket but very few. Due to rounding this shows as 0.00 ")
 787         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 788         sys.stdout.flush()
 789         for sock_index,sock in enumerate(socks,start=0):
 790                 buckets=socks[sock_index].show_irq_buckets(1)
 791                 print('Measurement ongoing ... ',end='\r')
 792                 socks[sock_index].stop(cores[sock_index])
 793                 old_irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
 794                 irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
 795                 irq[0][0] = 'bucket us'
 796                 for j,bucket in enumerate(buckets,start=1):
 797                         irq[0][j] = '<'+ bucket
 798                 irq[0][-1] = '>'+ buckets [-2]
 799                 socks[sock_index].start(cores[sock_index])
 800                 time.sleep(2)
 801                 for j,bucket in enumerate(buckets,start=1):
 802                         for i,irqcore in enumerate(cores[sock_index],start=1):
 803                                 old_irq[i][j] = socks[sock_index].irq_stats(irqcore,j-1)
 804                 time.sleep(float(runtime))
 805                 socks[sock_index].stop(cores[sock_index])
 806                 for i,irqcore in enumerate(cores[sock_index],start=1):
 807                         irq[i][0]='core %s '%irqcore
 808                         for j,bucket in enumerate(buckets,start=1):
 809                                 diff =  socks[sock_index].irq_stats(irqcore,j-1) - old_irq[i][j]
 810                                 if diff == 0:
 811                                         irq[i][j] = '0'
 812                                 else:
 813                                         irq[i][j] = str(round(diff/float(runtime), 2))
 814                 log.info('Results for PROX instance %s'%sock_index)
 815                 for row in irq:
 816                         log.info(''.join(['{:>12}'.format(item) for item in row]))
 817
 818 def run_impairtest(gensock,sutsock):
 819         fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Dropped','DropRate']
 820         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 821         writer.writeheader()
 822         size=PACKETSIZE-4
 823         log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 824         log.info("| Generator is sending UDP ("+'{:>5}'.format(FLOWSIZE)+" flow) packets ("+ '{:>5}'.format(size+4) +" bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test    |")
 825         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 826         log.info("| Test   |  Speed requested   | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
 827         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 828         attempts = 0
 829         gensock.set_size(gencores,0,size) # This is setting the frame size
 830         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 831         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 832         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 833         source_port,destination_port = flows[FLOWSIZE]
 834         gensock.set_random(gencores,0,34,source_port,2)
 835         gensock.set_random(gencores,0,36,destination_port,2)
 836         gensock.start(latcores)
 837         speed = STARTSPEED
 838         gensock.speed(speed / len(gencores) / len(gentasks), gencores, gentasks)
 839         while True:
 840                 attempts += 1
 841                 print('Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 842                 sys.stdout.flush()
 843                 time.sleep(1)
 844                 # Get statistics now that the generation is stable and NO ARP messages any more
 845                 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg, lat_perc, lat_perc_max, lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used, r, actual_duration = run_iteration(gensock,sutsock,runtime)
 846                 drop_rate = 100.0*abs_dropped/abs_tx
 847                 if lat_used < 0.95:
 848                         lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 849                 else:
 850                         lat_warning = ''
 851                 log.info('|{:>7}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | '+ '{:>9.3f}'.format(pps_tx) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9.3f}'.format(pps_rx)+' Mpps | '+ '{:>9.0f}'.format(lat_avg)+' us   | '+ '{:>9.0f}'.format(lat_max)+' us   | '+ '{:>14d}'.format(abs_dropped)+ ' |''{:>9.2f}'.format(drop_rate)+ '%  |'+lat_warning)
 852                 writer.writerow({'Flows':FLOWSIZE,'PacketSize':(size+4),'RequestedPPS':get_pps(speed,size),'GeneratedPPS':pps_req_tx,'SentPPS':pps_tx,'ForwardedPPS':pps_sut_tx_str,'ReceivedPPS':pps_rx,'AvgLatencyUSEC':lat_avg,'MaxLatencyUSEC':lat_max,'Dropped':abs_dropped,'DropRate':drop_rate})
 853                 if PushGateway:
 854                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
 855                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nDropped {}\nDropRate {}\n'.format(FLOWSIZE,size+4,get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,drop_rate)
 856                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 857                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 858
 859 def run_warmuptest(gensock):
 860 # Running at low speed to make sure the ARP messages can get through.
 861 # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
 862 # Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
 863 # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
 864         gensock.speed(WARMUPSPEED / len(gencores) /len (gentasks), gencores, gentasks)
 865         size=PACKETSIZE-4
 866         gensock.set_size(gencores,0,size) # This is setting the frame size
 867         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 868         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 869         gensock.set_value(gencores,0,56,1,1)
 870         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 871         source_port,destination_port = flows[FLOWSIZE]
 872         gensock.set_random(gencores,0,34,source_port,2)
 873         gensock.set_random(gencores,0,36,destination_port,2)
 874         gensock.start(genstatcores)
 875         time.sleep(WARMUPTIME)
 876         gensock.stop(genstatcores)
 877         gensock.set_value(gencores,0,56,50,1)
 878         time.sleep(WARMUPTIME)
 879
 880 def exit_handler():
 881         log.debug ('exit cleanup')
 882         for index, sock in enumerate(socks):
 883                 if socks_control[index]:
 884                         sock.quit()
 885         for client in clients:
 886                 client.close()
 887         data_csv_file.close
 888         sys.exit(0)
 889
 890 def get_BinarySearchParams() :
 891         global  DROP_RATE_TRESHOLD
 892         global  LAT_AVG_TRESHOLD
 893         global  LAT_PERC_TRESHOLD
 894         global  LAT_MAX_TRESHOLD
 895         global  ACCURACY
 896         global  STARTSPEED
 897         global  TST009
 898         global  TST009_MAXr
 899         global  TST009_MAXz
 900         DROP_RATE_TRESHOLD = float(testconfig.get('BinarySearchParams', 'drop_rate_threshold'))
 901         LAT_AVG_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_avg_threshold'))
 902         LAT_PERC_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_perc_threshold'))
 903         LAT_MAX_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_max_threshold'))
 904         ACCURACY = float(testconfig.get('BinarySearchParams', 'accuracy'))
 905         STARTSPEED = float(testconfig.get('BinarySearchParams', 'startspeed'))
 906         TST009_MAXr = 1
 907         TST009_MAXz = inf
 908         TST009 = False
 909
 910 def get_FixedRateParams() :
 911         global  DROP_RATE_TRESHOLD
 912         global  LAT_AVG_TRESHOLD
 913         global  LAT_PERC_TRESHOLD
 914         global  LAT_MAX_TRESHOLD
 915         global  flow_size_list
 916         global  packet_size_list
 917         global  STARTSPEED
 918         global  TST009
 919         global  TST009_MAXr
 920         global  TST009_MAXz
 921         DROP_RATE_TRESHOLD = inf
 922         LAT_AVG_TRESHOLD = inf
 923         LAT_PERC_TRESHOLD = inf
 924         LAT_MAX_TRESHOLD = inf
 925         TST009_MAXr = 1
 926         TST009_MAXz = inf
 927         TST009 = False
 928         packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
 929         flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
 930         STARTSPEED = float(testconfig.get('test%d'%test_nr, 'speed'))
 931
 932 def get_TST009SearchParams() :
 933         global  DROP_RATE_TRESHOLD
 934         global  LAT_AVG_TRESHOLD
 935         global  LAT_PERC_TRESHOLD
 936         global  LAT_MAX_TRESHOLD
 937         global  TST009
 938         global  TST009_MAXr
 939         global  TST009_MAXz
 940         global  TST009_MAXFramesAllIngress
 941         global  TST009_StepSize
 942         global  TST009_n
 943         global  TST009_L
 944         global  TST009_R
 945         global  TST009_S
 946         if testconfig.has_option('TST009SearchParams', 'drop_rate_threshold'):
 947                 DROP_RATE_TRESHOLD = float(testconfig.get('TST009SearchParams', 'drop_rate_threshold'))
 948         else:
 949                 DROP_RATE_TRESHOLD = 0
 950         LAT_AVG_TRESHOLD = inf
 951         LAT_PERC_TRESHOLD = inf
 952         LAT_MAX_TRESHOLD = inf
 953         TST009_MAXr = float(testconfig.get('TST009SearchParams', 'MAXr'))
 954         TST009_MAXz = float(testconfig.get('TST009SearchParams', 'MAXz'))
 955         TST009_MAXFramesAllIngress = int(testconfig.get('TST009SearchParams', 'MAXFramesPerSecondAllIngress'))
 956         TST009_StepSize = int(testconfig.get('TST009SearchParams', 'StepSize'))
 957         TST009_n = int(ceil(TST009_MAXFramesAllIngress / TST009_StepSize))
 958         TST009 = True
 959         TST009_L = 0
 960         TST009_R = TST009_n - 1
 961         for m in range(0, TST009_n):
 962                 TST009_S.append((m+1) * TST009_StepSize)
 963 # To generate a desired number of flows, PROX will randomize the bits in source and destination ports, as specified by the bit masks in the flows variable.
 964 flows={\
 965 1:      ['1000000000000000','1000000000000000'],\
 966 2:      ['1000000000000000','100000000000000X'],\
 967 4:      ['100000000000000X','100000000000000X'],\
 968 8:      ['100000000000000X','10000000000000XX'],\
 969 16:     ['10000000000000XX','10000000000000XX'],\
 970 32:     ['10000000000000XX','1000000000000XXX'],\
 971 64:     ['1000000000000XXX','1000000000000XXX'],\
 972 128:    ['1000000000000XXX','100000000000XXXX'],\
 973 256:    ['100000000000XXXX','100000000000XXXX'],\
 974 512:    ['100000000000XXXX','10000000000XXXXX'],\
 975 1024:   ['10000000000XXXXX','10000000000XXXXX'],\
 976 2048:   ['10000000000XXXXX','1000000000XXXXXX'],\
 977 4096:   ['1000000000XXXXXX','1000000000XXXXXX'],\
 978 8192:   ['1000000000XXXXXX','100000000XXXXXXX'],\
 979 16384:  ['100000000XXXXXXX','100000000XXXXXXX'],\
 980 32768:  ['100000000XXXXXXX','10000000XXXXXXXX'],\
 981 65536:  ['10000000XXXXXXXX','10000000XXXXXXXX'],\
 982 131072: ['10000000XXXXXXXX','1000000XXXXXXXXX'],\
 983 262144: ['1000000XXXXXXXXX','1000000XXXXXXXXX'],\
 984 524288: ['1000000XXXXXXXXX','100000XXXXXXXXXX'],\
 985 1048576:['100000XXXXXXXXXX','100000XXXXXXXXXX'],}
 986 clients =[]
 987 socks =[]
 988 socks_control =[]
 989 vmDPIP =[]
 990 vmAdminIP =[]
 991 vmDPmac =[]
 992 hexDPIP =[]
 993 config_file =[]
 994 prox_socket =[]
 995 prox_launch_exit =[]
 996 auto_start =[]
 997 mach_type =[]
 998 sock_type =[]
 999 monitor =[]
1000 sock_monitor =[]
1001 cores = []
1002 ports = []
1003 tasks = {}
1004 TST009_S = []
1005
1006 data_file = 'RUN{}.{}.csv'.format(env,test_file)
1007 data_csv_file = open(data_file,'w')
1008 testconfig = ConfigParser.RawConfigParser()
1009 testconfig.read(test_file)
1010 required_number_of_test_machines = testconfig.get('DEFAULT', 'total_number_of_test_machines')
1011 TestName = testconfig.get('DEFAULT', 'name')
1012 if testconfig.has_option('DEFAULT', 'PushGateway'):
1013         PushGateway = testconfig.get('DEFAULT', 'PushGateway')
1014         log.info('Measurements will be pushed to %s'%PushGateway)
1015 else:
1016         PushGateway = None
1017 if testconfig.has_option('DEFAULT', 'lat_percentile'):
1018         LAT_PERCENTILE = float(testconfig.get('DEFAULT', 'lat_percentile')) /100.0
1019 else:
1020         LAT_PERCENTILE = 0.99
1021 log.info('Latency percentile measured at {:.0f}%'.format(LAT_PERCENTILE*100))
1022 config = ConfigParser.RawConfigParser()
1023 config.read(env)
1024 machine_map = ConfigParser.RawConfigParser()
1025 machine_map.read(machine_map_file)
1026 key = config.get('ssh', 'key')
1027 user = config.get('ssh', 'user')
1028 total_number_of_machines = config.get('rapid', 'total_number_of_machines')
1029 if int(required_number_of_test_machines) > int(total_number_of_machines):
1030         log.exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
1031         raise Exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
1032 for vm in range(1, int(total_number_of_machines)+1):
1033         vmAdminIP.append(config.get('M%d'%vm, 'admin_ip'))
1034         vmDPmac.append(config.get('M%d'%vm, 'dp_mac'))
1035         vmDPIP.append(config.get('M%d'%vm, 'dp_ip'))
1036         ip = vmDPIP[-1].split('.')
1037         hexDPIP.append(hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2))
1038 machine_index = []
1039 for vm in range(1, int(required_number_of_test_machines)+1):
1040         machine_index.append(int(machine_map.get('TestM%d'%vm, 'machine_index'))-1)
1041         prox_socket.append(testconfig.getboolean('TestM%d'%vm, 'prox_socket'))
1042 for vm in range(1, int(required_number_of_test_machines)+1):
1043         if prox_socket[vm-1]:
1044                 prox_launch_exit.append(testconfig.getboolean('TestM%d'%vm, 'prox_launch_exit'))
1045                 config_file.append(testconfig.get('TestM%d'%vm, 'config_file'))
1046                 # Looking for all task definitions in the PROX cfg files. Constructing a list of all tasks used
1047                 textfile =  open (config_file[-1], 'r')
1048                 filetext = textfile.read()
1049                 textfile.close()
1050                 tasks_for_this_cfg = set(re.findall("task\s*=\s*(\d+)",filetext))
1051                 with open('{}_{}_parameters{}.lua'.format(env,test_file,vm), "w") as f:
1052                         f.write('name="%s"\n'% testconfig.get('TestM%d'%vm, 'name'))
1053                         f.write('local_ip="%s"\n'% vmDPIP[machine_index[vm-1]])
1054                         f.write('local_hex_ip="%s"\n'% hexDPIP[machine_index[vm-1]])
1055                         if testconfig.has_option('TestM%d'%vm, 'cores'):
1056                                 cores.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'cores')))
1057                                 f.write('cores="%s"\n'% ','.join(map(str, cores[-1])))
1058                         else:
1059                                 cores.append(None)
1060                         if testconfig.has_option('TestM%d'%vm, 'ports'):
1061                                 ports.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'ports')))
1062                                 f.write('ports="%s"\n'% ','.join(map(str, ports[-1])))
1063                         else:
1064                                 ports.append(None)
1065                         if testconfig.has_option('TestM%d'%vm, 'monitor'):
1066                                 monitor.append(testconfig.getboolean('TestM%d'%vm, 'monitor'))
1067                         else:
1068                                 monitor.append(True)
1069                         if re.match('(l2){0,1}gen(_bare){0,1}.*\.cfg',config_file[-1]):
1070                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
1071                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
1072                                 genstatcores = gencores + latcores
1073                                 gentasks = tasks_for_this_cfg
1074                                 auto_start.append(False)
1075                                 mach_type.append('gen')
1076                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
1077                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
1078                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
1079                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
1080                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
1081                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
1082                                 if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'):
1083                                         BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp'))
1084                                 else:
1085                                         BUCKET_SIZE_EXP = 11
1086                                 f.write('bucket_size_exp="%s"\n'% BUCKET_SIZE_EXP)
1087                                 if testconfig.has_option('TestM%d'%vm, 'heartbeat'):
1088                                         heartbeat = int(testconfig.get('TestM%d'%vm, 'heartbeat'))
1089                                 else:
1090                                         heartbeat = 60
1091                                 f.write('heartbeat="%s"\n'% heartbeat)
1092                         elif re.match('(l2){0,1}gen_gw.*\.cfg',config_file[-1]):
1093                                 if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'):
1094                                         BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp'))
1095                                 else:
1096                                         BUCKET_SIZE_EXP = 11
1097                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
1098                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
1099                                 genstatcores = gencores + latcores
1100                                 gentasks = tasks_for_this_cfg
1101                                 auto_start.append(False)
1102                                 mach_type.append('gen')
1103                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
1104                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
1105                                 gwVMindex = int(testconfig.get('TestM%d'%vm, 'gw_vm')) -1
1106                                 f.write('gw_ip="%s"\n'% vmDPIP[machine_index[gwVMindex]])
1107                                 f.write('gw_hex_ip="%s"\n'% hexDPIP[machine_index[gwVMindex]])
1108                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
1109                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
1110                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
1111                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
1112                                 if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'):
1113                                         BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp'))
1114                                 else:
1115                                         BUCKET_SIZE_EXP = 11
1116                                 f.write('bucket_size_exp="%s"\n'% BUCKET_SIZE_EXP)
1117                                 if testconfig.has_option('TestM%d'%vm, 'heartbeat'):
1118                                         heartbeat = int(testconfig.get('TestM%d'%vm, 'heartbeat'))
1119                                 else:
1120                                         heartbeat = 60
1121                                 f.write('heartbeat="%s"\n'% heartbeat)
1122                         elif re.match('(l2){0,1}swap.*\.cfg',config_file[-1]):
1123                                 sutstatcores = cores[-1]
1124                                 auto_start.append(True)
1125                                 mach_type.append('sut')
1126                         elif re.match('secgw1.*\.cfg',config_file[-1]):
1127                                 auto_start.append(True)
1128                                 mach_type.append('none')
1129                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
1130                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
1131                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
1132                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
1133                         elif re.match('secgw2.*\.cfg',config_file[-1]):
1134                                 sutstatcores = cores[-1]
1135                                 auto_start.append(True)
1136                                 mach_type.append('sut')
1137                         else:
1138                                 auto_start.append(True)
1139                                 mach_type.append('none')
1140                 f.close
1141                 tasks = tasks_for_this_cfg.union(tasks)
1142 log.debug("Tasks detected in all PROX config files %r"%tasks)
1143 #####################################################################################
1144 for vm in range(0, int(required_number_of_test_machines)):
1145         if prox_socket[vm]:
1146                 clients.append(prox_ctrl(vmAdminIP[machine_index[vm]], key,user))
1147                 connect_client(clients[-1])
1148 # Creating script to bind the right network interface to the poll mode driver
1149                 devbindfile = '{}_{}_devbindvm{}.sh'.format(env,test_file, vm+1)
1150                 with open(devbindfile, "w") as f:
1151                         newText= 'link="$(ip -o link | grep '+vmDPmac[machine_index[vm]]+' |cut -d":" -f 2)"\n'
1152                         f.write(newText)
1153                         newText= 'if [ -n "$link" ];\n'
1154                         f.write(newText)
1155                         newText= 'then\n'
1156                         f.write(newText)
1157                         newText= '        echo Need to bind\n'
1158                         f.write(newText)
1159                         newText= '        sudo ' + rundir + '/dpdk/usertools/dpdk-devbind.py --force --bind igb_uio $('+rundir+'/dpdk/usertools/dpdk-devbind.py --status |grep  $link | cut -d" " -f 1)\n'
1160                         f.write(newText)
1161                         newText= 'else\n'
1162                         f.write(newText)
1163                         newText= '       echo Assuming port is already bound to DPDK\n'
1164                         f.write(newText)
1165                         newText= 'fi\n'
1166                         f.write(newText)
1167                         newText= 'exit 0\n'
1168                         f.write(newText)
1169                 st = os.stat(devbindfile)
1170                 os.chmod(devbindfile, st.st_mode | stat.S_IEXEC)
1171                 clients[-1].scp_put('./%s'%devbindfile, rundir+'/devbind.sh')
1172                 cmd = 'sudo ' + rundir+ '/devbind.sh'
1173                 clients[-1].run_cmd(cmd)
1174                 log.debug("devbind.sh running on VM%d"%(vm+1))
1175                 clients[-1].scp_put('./%s'%config_file[vm], rundir+'/%s'%config_file[vm])
1176                 clients[-1].scp_put('./{}_{}_parameters{}.lua'.format(env,test_file, vm+1), rundir + '/parameters.lua')
1177                 if not configonly:
1178                         if prox_launch_exit[vm]:
1179                                 log.debug("Starting PROX on VM%d"%(vm+1))
1180                                 if auto_start[vm]:
1181                                         cmd = 'sudo ' +rundir + '/prox/build/prox -t -o cli -f ' + rundir + '/%s'%config_file[vm]
1182                                 else:
1183                                         cmd = 'sudo ' +rundir + '/prox/build/prox -e -t -o cli -f ' + rundir + '/%s'%config_file[vm]
1184                                 clients[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1))
1185                         socks_control.append(prox_launch_exit[vm])
1186                         socks.append(connect_socket(clients[-1]))
1187                         sock_type.append(mach_type[vm])
1188                         sock_monitor.append(monitor[vm])
1189 atexit.register(exit_handler)
1190 monitor_gen = monitor_sut = False
1191 background_gen_socks =[]
1192 for index, sock in enumerate(socks, start=0):
1193         if sock_type[index] == 'gen':
1194                 if sock_monitor[index]:
1195                         if monitor_gen:
1196                                 log.exception("Can only monitor 1 generator")
1197                                 raise Exception("Can only monitor 1 generator")
1198                         else:
1199                                 monitor_gen = True
1200                                 gensock_index = index
1201                 else:
1202                         background_gen_socks.append(sock)
1203         elif sock_type[index] == 'sut' and sock_monitor[index]:
1204                 if monitor_sut:
1205                         log.exception("Can only monitor 1 sut")
1206                         raise Exception("Can only monitor 1 sut")
1207                 else:
1208                         monitor_sut = True
1209                         sutsock_index = index
1210 if not(monitor_gen and monitor_sut):
1211         log.exception("No generator and/or sut to monitor")
1212         raise Exception("No generator and/or sut to monitor")
1213 if configonly:
1214         sys.exit()
1215 ####################################################
1216 # Run test cases
1217 # Best to run the flow test at the end since otherwise the tests coming after might be influenced by the big number of entries in the switch flow tables
1218 ####################################################
1219 number_of_tests = testconfig.get('DEFAULT', 'number_of_tests')
1220 for test_nr in range(1, int(number_of_tests)+1):
1221         test=testconfig.get('test%d'%test_nr,'test')
1222         log.info(test)
1223         if test == 'flowsizetest':
1224                 get_BinarySearchParams()
1225                 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1226                 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1227                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1228         elif test == 'TST009test':
1229                 # This test implements some of the testing as defined in https://docbox.etsi.org/ISG/NFV/open/Publications_pdf/Specs-Reports/NFV-TST%20009v3.2.1%20-%20GS%20-%20NFVI_Benchmarks.pdf
1230                 get_TST009SearchParams()
1231                 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1232                 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1233                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1234         elif test == 'fixed_rate':
1235                 get_FixedRateParams()
1236                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1237         elif test == 'corestats':
1238                 run_core_stats(socks)
1239         elif test == 'portstats':
1240                 run_port_stats(socks)
1241         elif test == 'impairtest':
1242                 get_BinarySearchParams()
1243                 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1244                 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1245                 run_impairtest(socks[gensock_index],socks[sutsock_index])
1246         elif test == 'irqtest':
1247                 run_irqtest(socks)
1248         elif test == 'warmuptest':
1249                 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1250                 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1251                 WARMUPSPEED = int(testconfig.get('test%d'%test_nr, 'warmupspeed'))
1252                 WARMUPTIME = int(testconfig.get('test%d'%test_nr, 'warmuptime'))
1253                 run_warmuptest(socks[gensock_index])
1254 ####################################################