VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py

   1 #!/usr/bin/python
   2
   3 ##
   4 ## Copyright (c) 2010-2019 Intel Corporation
   5 ##
   6 ## Licensed under the Apache License, Version 2.0 (the "License");
   7 ## you may not use this file except in compliance with the License.
   8 ## You may obtain a copy of the License at
   9 ##
  10 ##     http://www.apache.org/licenses/LICENSE-2.0
  11 ##
  12 ## Unless required by applicable law or agreed to in writing, software
  13 ## distributed under the License is distributed on an "AS IS" BASIS,
  14 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 ## See the License for the specific language governing permissions and
  16 ## limitations under the License.
  17 ##
  18
  19 from __future__ import print_function
  20
  21 import os
  22 import stat
  23 import sys
  24 import time
  25 import subprocess
  26 import getopt
  27 import re
  28 import logging
  29 from logging.handlers import RotatingFileHandler
  30 from logging import handlers
  31 from prox_ctrl import prox_ctrl
  32 import ConfigParser
  33 import ast
  34 import atexit
  35 import csv
  36 import requests
  37
  38 version="19.11.21"
  39 env = "rapid.env" #Default string for environment
  40 test_file = "basicrapid.test" #Default string for test
  41 machine_map_file = "machine.map" #Default string for machine map file
  42 loglevel="DEBUG" # sets log level for writing to file
  43 screenloglevel="INFO" # sets log level for writing to screen
  44 runtime=10 # time in seconds for 1 test run
  45 configonly = False # IF True, the system will upload all the necessary config fiels to the VMs, but not start PROX and the actual testing
  46 rundir = "/home/centos" # Directory where to find the tools in the machines running PROX
  47
  48 def usage():
  49         print("usage: runrapid    [--version] [-v]")
  50         print("                   [--env ENVIRONMENT_NAME]")
  51         print("                   [--test TEST_NAME]")
  52         print("                   [--map MACHINE_MAP_FILE]")
  53         print("                   [--runtime TIME_FOR_TEST]")
  54         print("                   [--configonly False|True]")
  55         print("                   [--log DEBUG|INFO|WARNING|ERROR|CRITICAL]")
  56         print("                   [-h] [--help]")
  57         print("")
  58         print("Command-line interface to runrapid")
  59         print("")
  60         print("optional arguments:")
  61         print("  -v,  --version                 Show program's version number and exit")
  62         print("  --env ENVIRONMENT_NAME         Parameters will be read from ENVIRONMENT_NAME. Default is %s."%env)
  63         print("  --test TEST_NAME               Test cases will be read from TEST_NAME. Default is %s."%test_file)
  64         print("  --map MACHINE_MAP_FILE Machine mapping will be read from MACHINE_MAP_FILE. Default is %s."%machine_map_file)
  65         print("  --runtime                      Specify time in seconds for 1 test run")
  66         print("  --configonly                   If this option is specified, only upload all config files to the VMs, do not run the tests")
  67         print("  --log                          Specify logging level for log file output, default is DEBUG")
  68         print("  --screenlog                    Specify logging level for screen output, default is INFO")
  69         print("  -h, --help                     Show help message and exit.")
  70         print("")
  71
  72 try:
  73         opts, args = getopt.getopt(sys.argv[1:], "vh", ["version","help", "env=", "test=", "map=", "runtime=","configonly","log=","screenlog="])
  74 except getopt.GetoptError as err:
  75         print("===========================================")
  76         print(str(err))
  77         print("===========================================")
  78         usage()
  79         sys.exit(2)
  80 if args:
  81         usage()
  82         sys.exit(2)
  83 for opt, arg in opts:
  84         if opt in ["-h", "--help"]:
  85                 usage()
  86                 sys.exit()
  87         if opt in ["-v", "--version"]:
  88                 print("Rapid Automated Performance Indication for Dataplane "+version)
  89                 sys.exit()
  90         if opt in ["--env"]:
  91                 env = arg
  92         if opt in ["--test"]:
  93                 test_file = arg
  94         if opt in ["--map"]:
  95                 machine_map_file = arg
  96         if opt in ["--runtime"]:
  97                 runtime = arg
  98         if opt in ["--configonly"]:
  99                 configonly = True
 100                 print('No actual runs, only uploading configuration files')
 101         if opt in ["--log"]:
 102                 loglevel = arg
 103                 print ("Log level: "+ loglevel)
 104         if opt in ["--screenlog"]:
 105                 screenloglevel = arg
 106                 print ("Screen Log level: "+ screenloglevel)
 107
 108 print ("Using '"+env+"' as name for the environment")
 109 print ("Using '"+test_file+"' for test case definition")
 110 print ("Using '"+machine_map_file+"' for machine mapping")
 111 print ("Runtime: "+ str(runtime))
 112
 113 class bcolors:
 114         HEADER = '\033[95m'
 115         OKBLUE = '\033[94m'
 116         OKGREEN = '\033[92m'
 117         WARNING = '\033[93m'
 118         FAIL = '\033[91m'
 119         ENDC = '\033[0m'
 120         BOLD = '\033[1m'
 121         UNDERLINE = '\033[4m'
 122
 123 # create formatters
 124 screen_formatter = logging.Formatter("%(message)s")
 125 file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 126
 127 # get a top-level logger,
 128 # set its log level,
 129 # BUT PREVENT IT from propagating messages to the root logger
 130 #
 131 log = logging.getLogger()
 132 numeric_level = getattr(logging, loglevel.upper(), None)
 133 if not isinstance(numeric_level, int):
 134         raise ValueError('Invalid log level: %s' % loglevel)
 135 log.setLevel(numeric_level)
 136 log.propagate = 0
 137
 138 # create a console handler
 139 # and set its log level to the command-line option
 140 #
 141 console_handler = logging.StreamHandler(sys.stdout)
 142 #console_handler.setLevel(logging.INFO)
 143 numeric_screenlevel = getattr(logging, screenloglevel.upper(), None)
 144 if not isinstance(numeric_screenlevel, int):
 145         raise ValueError('Invalid screenlog level: %s' % screenloglevel)
 146 console_handler.setLevel(numeric_screenlevel)
 147 console_handler.setFormatter(screen_formatter)
 148
 149 # create a file handler
 150 # and set its log level
 151 #
 152 log_file = 'RUN{}.{}.log'.format(env,test_file)
 153 file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10)
 154 #file_handler = log.handlers.TimedRotatingFileHandler(log_file, 'D', 1, 5)
 155 file_handler.setLevel(numeric_level)
 156 file_handler.setFormatter(file_formatter)
 157
 158 # add handlers to the logger
 159 #
 160 log.addHandler(file_handler)
 161 log.addHandler(console_handler)
 162
 163 # Check if log exists and should therefore be rolled
 164 needRoll = os.path.isfile(log_file)
 165
 166
 167 # This is a stale log, so roll it
 168 if needRoll:
 169         # Add timestamp
 170         log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime())
 171
 172         # Roll over on application start
 173         log.handlers[0].doRollover()
 174
 175 # Add timestamp
 176 log.debug('\n---------\nLog started on %s.\n---------\n' % time.asctime())
 177
 178 log.debug("runrapid.py version: "+version)
 179 #========================================================================
 180 def connect_socket(client):
 181         attempts = 1
 182         log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 183         sock = None
 184         while True:
 185                 sock = client.prox_sock()
 186                 if sock is not None:
 187                         break
 188                 attempts += 1
 189                 if attempts > 20:
 190                         log.exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 191                         raise Exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 192                 time.sleep(2)
 193                 log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 194         log.info("Connected to PROX on %s" % client.ip())
 195         return sock
 196
 197 def connect_client(client):
 198         attempts = 1
 199         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 200         while True:
 201                 try:
 202                         client.connect()
 203                         break
 204                 except RuntimeWarning, ex:
 205                         attempts += 1
 206                         if attempts > 20:
 207                                 log.exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 208                                 raise Exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 209                         time.sleep(2)
 210                         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 211         log.debug("Connected to VM on %s" % client.ip())
 212
 213 def run_iteration(gensock,sutsock):
 214         sleep_time = 2
 215         # Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
 216         time.sleep(sleep_time)
 217         abs_old_rx, abs_old_non_dp_rx, abs_old_tx, abs_old_non_dp_tx, abs_old_drop, abs_old_tx_fail, abs_old_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
 218         abs_old_rx = abs_old_rx - abs_old_non_dp_rx
 219         abs_old_tx = abs_old_tx - abs_old_non_dp_tx
 220         gensock.start(gencores)
 221         time.sleep(sleep_time)
 222         if sutsock!='none':
 223                 old_sut_rx, old_sut_non_dp_rx, old_sut_tx, old_sut_non_dp_tx, old_sut_drop, old_sut_tx_fail, old_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 224                 old_sut_rx = old_sut_rx - old_sut_non_dp_rx
 225                 old_sut_tx = old_sut_tx - old_sut_non_dp_tx
 226         old_rx, old_non_dp_rx, old_tx, old_non_dp_tx, old_drop, old_tx_fail, old_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 227         old_rx = old_rx - old_non_dp_rx
 228         old_tx = old_tx - old_non_dp_tx
 229         # Measure latency statistics per second
 230         n_loops = 0
 231         lat_min = 0
 232         lat_max = 0
 233         lat_avg = 0
 234         used_avg = 0
 235         while n_loops < float(runtime):
 236                 n_loops +=1
 237                 time.sleep(1)
 238                 lat_min_sample, lat_max_sample, lat_avg_sample, used_sample = gensock.lat_stats(latcores)
 239                 if lat_min > lat_min_sample:
 240                         lat_min = lat_min_sample
 241                 if lat_max < lat_max_sample:
 242                         lat_max = lat_max_sample
 243                 lat_avg = lat_avg + lat_avg_sample
 244                 used_avg = used_avg + used_sample
 245         lat_avg = lat_avg / n_loops
 246         used_avg = used_avg / n_loops
 247         # Get statistics after some execution time
 248         new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 249         new_rx = new_rx - new_non_dp_rx
 250         new_tx = new_tx - new_non_dp_tx
 251         if sutsock!='none':
 252                 new_sut_rx, new_sut_non_dp_rx, new_sut_tx, new_sut_non_dp_tx, new_sut_drop, new_sut_tx_fail, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 253                 new_sut_rx = new_sut_rx - new_sut_non_dp_rx
 254                 new_sut_tx = new_sut_tx - new_sut_non_dp_tx
 255         #Stop generating
 256         gensock.stop(gencores)
 257         time.sleep(sleep_time)
 258         abs_new_rx, abs_new_non_dp_rx, abs_new_tx, abs_new_non_dp_tx, abs_new_drop, abs_new_tx_fail, abs_new_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
 259         abs_new_rx = abs_new_rx - abs_new_non_dp_rx
 260         abs_new_tx = abs_new_tx - abs_new_non_dp_tx
 261         drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 262         rx = new_rx - old_rx     # rx is all packets received by the nop task = all packets received in the gen VM
 263         tx = new_tx - old_tx     # tx is all generated packets actually accepted by the interface
 264         abs_dropped = (abs_new_tx - abs_old_tx) - (abs_new_rx - abs_old_rx)
 265         tsc = new_tsc - old_tsc  # time difference between the 2 measurements, expressed in cycles.
 266         pps_req_tx = (tx+drop-rx)*tsc_hz*1.0/(tsc*1000000)
 267         pps_tx = tx*tsc_hz*1.0/(tsc*1000000)
 268         pps_rx = rx*tsc_hz*1.0/(tsc*1000000)
 269         if sutsock!='none':
 270                 sut_rx = new_sut_rx - old_sut_rx
 271                 sut_tx = new_sut_tx - old_sut_tx
 272                 sut_tsc = new_sut_tsc - old_sut_tsc
 273                 pps_sut_tx = sut_tx*sut_tsc_hz*1.0/(sut_tsc*1000000)
 274                 pps_sut_tx_str = '{:>9.3f}'.format(pps_sut_tx)
 275         else:
 276                 pps_sut_tx = 0
 277                 pps_sut_tx_str = 'NO MEAS.'
 278         if (tx == 0):
 279                 log.critical("TX = 0. Test interrupted since no packet has been sent.")
 280                 raise Exception("TX = 0")
 281         return(pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,(abs_new_tx_fail - abs_old_tx_fail),(abs_new_tx - abs_old_tx),lat_min,used_avg)
 282
 283 def new_speed(speed,minspeed,maxspeed,success):
 284         if success:
 285                 minspeed = speed
 286         else:
 287                 maxspeed = speed
 288         newspeed = (maxspeed+minspeed)/2.0
 289         return (newspeed,minspeed,maxspeed)
 290
 291 def get_pps(speed,size):
 292         # speed is given in % of 10Gb/s, returning Mpps
 293         return (speed * 100.0 / (8*(size+24)))
 294
 295 def get_speed(packet_speed,size):
 296         # return speed in Gb/s
 297         return (packet_speed / 1000.0 * (8*(size+24)))
 298
 299
 300 def run_flow_size_test(gensock,sutsock):
 301         fieldnames = ['Flows','PacketSize','Gbps','Mpps','AvgLatency','MaxLatency','PacketsDropped','PacketDropRate']
 302         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 303         writer.writeheader()
 304         gensock.start(latcores)
 305         for size in packet_size_list:
 306                 size = size-4
 307                 gensock.set_size(gencores,0,size) # This is setting the frame size
 308                 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 309                 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 310                 # This will only work when using sending UDP packets. For different protocls and ehternet types, we would need a different calculation
 311                 log.info("+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 312                 log.info("| UDP, "+ '{:>5}'.format(size+4) +" bytes, different number of flows by randomizing SRC & DST UDP port                                                                                           |")
 313                 log.info("+--------+--------------------+----------------+----------------+----------------+------------------------+----------------+----------------+----------------+------------+")
 314                 log.info("| Flows  |  Speed requested   | core generated | Sent by Gen NIC| Forward by SUT |      core received     |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
 315                 log.info("+--------+--------------------+----------------+----------------+----------------+------------------------+----------------+----------------+----------------+------------+")
 316                 for flow_number in flow_size_list:
 317                         attempts = 0
 318                         gensock.reset_stats()
 319                         if sutsock!='none':
 320                                 sutsock.reset_stats()
 321                         source_port,destination_port = flows[flow_number]
 322                         gensock.set_random(gencores,0,34,source_port,2)
 323                         gensock.set_random(gencores,0,36,destination_port,2)
 324                         endpps_sut_tx_str = 'NO_RESULTS'
 325                         maxspeed = speed = STARTSPEED
 326                         minspeed = 0
 327                         while (maxspeed-minspeed > ACCURACY):
 328                                 attempts += 1
 329                                 endwarning =''
 330                                 print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 331                                 sys.stdout.flush()
 332                                 # Start generating packets at requested speed (in % of a 10Gb/s link)
 333                                 gensock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
 334                                 time.sleep(1)
 335                                 # Get statistics now that the generation is stable and initial ARP messages are dealt with
 336                                 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used = run_iteration(gensock,sutsock)
 337                                 drop_rate = 100.0*abs_dropped/abs_tx
 338                                 if lat_used < 0.95:
 339                                         lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 340                                 else:
 341                                         lat_warning = ''
 342                                 # The following if statement is testing if we pass the success criteria of a certain drop rate, average latenecy and maximum latency below the threshold
 343                                 # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
 344                                 # This can be specified by putting 0 in the .test file
 345                                 if ((drop_rate < DROP_RATE_TRESHOLD) or (abs_dropped==DROP_RATE_TRESHOLD ==0)) and (lat_avg< LAT_AVG_TRESHOLD) and (lat_max < LAT_MAX_TRESHOLD):
 346                                         lat_avg_prefix = bcolors.ENDC
 347                                         lat_max_prefix = bcolors.ENDC
 348                                         abs_drop_rate_prefix = bcolors.ENDC
 349                                         drop_rate_prefix = bcolors.ENDC
 350                                         if ((get_pps(speed,size) - pps_tx)/get_pps(speed,size))>0.01:
 351                                                 speed_prefix = bcolors.WARNING
 352                                                 if abs_tx_fail > 0:
 353                                                         gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
 354                                                 else:
 355                                                         gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(get_pps(speed,size), pps_tx) + bcolors.ENDC
 356                                         else:
 357                                                 speed_prefix = bcolors.ENDC
 358                                                 gen_warning = ''
 359                                         endspeed = speed
 360                                         endspeed_prefix = speed_prefix
 361                                         endpps_req_tx = pps_req_tx
 362                                         endpps_tx = pps_tx
 363                                         endpps_sut_tx_str = pps_sut_tx_str
 364                                         endpps_rx = pps_rx
 365                                         endlat_avg = lat_avg
 366                                         endlat_max = lat_max
 367                                         endabs_dropped = abs_dropped
 368                                         enddrop_rate = drop_rate
 369                                         if lat_warning or gen_warning:
 370                                                 endwarning = '|        | {:167.167} |'.format(lat_warning + gen_warning)
 371                                         success = True
 372                                         success_message='%  | SUCCESS'
 373                                 else:
 374                                         success_message='%  | FAILED'
 375                                         gen_warning = ''
 376                                         abs_drop_rate_prefix = bcolors.ENDC
 377                                         if ((abs_dropped>0) and (DROP_RATE_TRESHOLD ==0)):
 378                                                 abs_drop_rate_prefix = bcolors.FAIL
 379                                         if (drop_rate < DROP_RATE_TRESHOLD):
 380                                                 drop_rate_prefix = bcolors.ENDC
 381                                         else:
 382                                                 drop_rate_prefix = bcolors.FAIL
 383                                         if (lat_avg< LAT_AVG_TRESHOLD):
 384                                                 lat_avg_prefix = bcolors.ENDC
 385                                         else:
 386                                                 lat_avg_prefix = bcolors.FAIL
 387                                         if (lat_max< LAT_MAX_TRESHOLD):
 388                                                 lat_max_prefix = bcolors.ENDC
 389                                         else:
 390                                                 lat_max_prefix = bcolors.FAIL
 391                                         if (((get_pps(speed,size) - pps_tx)/get_pps(speed,size))<0.001):
 392                                                 speed_prefix = bcolors.ENDC
 393                                         else:
 394                                                 speed_prefix = bcolors.FAIL
 395                                         success = False
 396                                 log.debug('|step{:>3}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% '+speed_prefix +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | ' + '{:>9.3f}'.format(pps_tx) +' Mpps | '+ bcolors.ENDC  + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+bcolors.OKBLUE + '{:>4.1f}'.format(get_speed(pps_rx,size)) + 'Gb/s{:>9.3f}'.format(pps_rx)+' Mpps'+bcolors.ENDC+' | '+lat_avg_prefix+ '{:>9.0f}'.format(lat_avg)+' us   | '+lat_max_prefix+ '{:>9.0f}'.format(lat_max)+' us   | '+ abs_drop_rate_prefix + '{:>14d}'.format(abs_dropped)+drop_rate_prefix+ ' |''{:>9.2f}'.format(drop_rate)+bcolors.ENDC+ success_message +lat_warning + gen_warning)
 397                                 speed,minspeed,maxspeed = new_speed(speed,minspeed,maxspeed,success)
 398                         if endpps_sut_tx_str !=  'NO_RESULTS':
 399                                 log.info('|{:>7}'.format(str(flow_number))+" | " + '{:>5.1f}'.format(endspeed) + '% ' + endspeed_prefix + '{:>6.3f}'.format(get_pps(endspeed,size)) + ' Mpps | '+ '{:>9.3f}'.format(endpps_req_tx)+ ' Mpps | '+ '{:>9.3f}'.format(endpps_tx) + ' Mpps | ' + bcolors.ENDC + '{:>9}'.format(endpps_sut_tx_str) +' Mpps | '+bcolors.OKBLUE + '{:>4.1f}'.format(get_speed(endpps_rx,size)) + 'Gb/s{:>9.3f}'.format(endpps_rx)+' Mpps'+bcolors.ENDC+' | '+ '{:>9.0f}'.format(endlat_avg)+' us   | '+ '{:>9.0f}'.format(endlat_max)+' us   | '+ '{:>14d}'.format(endabs_dropped)+ ' |'+'{:>9.2f}'.format(enddrop_rate)+ '%  |')
 400                                 if endwarning:
 401                                         log.info (endwarning)
 402                                 log.info("+--------+--------------------+----------------+----------------+----------------+------------------------+----------------+----------------+----------------+------------+")
 403                                 writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'Gbps':get_speed(endpps_rx,size),'Mpps':endpps_rx,'AvgLatency':endlat_avg,'MaxLatency':endlat_max,'PacketsDropped':endabs_dropped,'PacketDropRate':enddrop_rate})
 404                                 if PushGateway:
 405                                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
 406                                         DATA = 'Flows {}\nPacketSize {}\nGbps {}\nMpps {}\nAvgLatency {}\nMaxLatency {}\nPacketsDropped {}\nPacketDropRate {}\n'.format(flow_number,size+4,get_speed(endpps_rx,size),endpps_rx,endlat_avg,endlat_max,endabs_dropped,enddrop_rate)
 407                                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 408                                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 409                         else:
 410                                 log.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
 411         gensock.stop(latcores)
 412
 413
 414 def run_fixed_rate(gensock,sutsock):
 415         fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Sent','Received','Lost','LostTotal']
 416         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 417         writer.writeheader()
 418         gensock.start(latcores)
 419         sleep_time=3
 420         for size in packet_size_list:
 421                 size = size-4
 422                 gensock.set_size(gencores,0,size) # This is setting the frame size
 423                 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 424                 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 425                 # This will only work when using sending UDP packets. For different protocols and ehternet types, we would need a different calculation
 426                 log.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 427                 log.info("| UDP, "+ '{:>5}'.format(size+4) +" bytes, different number of flows by randomizing SRC & DST UDP port                                                                                |")
 428                 log.info("+--------+------------------+-------------+-------------+-------------+-------------+-------------+-------------+-----------+-----------+---------+------------+")
 429                 log.info("| Flows  | Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core| Avg. Latency| Max. Latency|   Sent    |  Received |  Lost   | Total Lost |")
 430                 log.info("+--------+------------------+-------------+-------------+-------------+-------------+-------------+-------------+-----------+-----------+---------+------------+")
 431                 for flow_number in flow_size_list:
 432                         time.sleep(sleep_time)
 433                         gensock.reset_stats()
 434                         if sutsock!='none':
 435                                 sutsock.reset_stats()
 436                         source_port,destination_port = flows[flow_number]
 437                         gensock.set_random(gencores,0,34,source_port,2)
 438                         gensock.set_random(gencores,0,36,destination_port,2)
 439                         endpps_sut_tx_str = 'NO_RESULTS'
 440                         speed = STARTSPEED
 441                         # Start generating packets at requested speed (in % of a 10Gb/s link)
 442                         gensock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
 443                         duration = float(runtime)
 444                         first = 1
 445                         tot_drop = 0
 446                         if sutsock!='none':
 447                                 old_sut_rx, old_sut_non_dp_rx, old_sut_tx, old_sut_non_dp_tx, old_sut_drop, old_sut_tx_fail, old_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 448                                 old_sut_rx = old_sut_rx - old_sut_non_dp_rx
 449                                 old_sut_tx = old_sut_tx - old_sut_non_dp_tx
 450                         old_rx, old_non_dp_rx, old_tx, old_non_dp_tx, old_drop, old_tx_fail, old_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 451                         old_rx = old_rx - old_non_dp_rx
 452                         old_tx = old_tx - old_non_dp_tx
 453                         gensock.start(gencores)
 454                         while (duration > 0):
 455                                 time.sleep(0.5)
 456                                 lat_min, lat_max, lat_avg, lat_used = gensock.lat_stats(latcores)
 457                                 if lat_used < 0.95:
 458                                         lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 459                                 else:
 460                                         lat_warning = ''
 461                                 # Get statistics after some execution time
 462                                 new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 463                                 new_rx = new_rx - new_non_dp_rx
 464                                 new_tx = new_tx - new_non_dp_tx
 465                                 if sutsock!='none':
 466                                         new_sut_rx, new_sut_non_dp_rx, new_sut_tx, new_sut_non_dp_tx, new_sut_drop, new_sut_tx_fail, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 467                                         new_sut_rx = new_sut_rx - new_sut_non_dp_rx
 468                                         new_sut_tx = new_sut_tx - new_sut_non_dp_tx
 469                                         drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 470                                         rx = new_rx - old_rx     # rx is all packets received by the nop task = all packets received in the gen VM
 471                                         tx = new_tx - old_tx     # tx is all generated packets actually accepted by the interface
 472                                         tsc = new_tsc - old_tsc  # time difference between the 2 measurements, expressed in cycles.
 473                                 if tsc == 0 :
 474                                         continue
 475                                 if sutsock!='none':
 476                                         sut_rx = new_sut_rx - old_sut_rx
 477                                         sut_tx = new_sut_tx - old_sut_tx
 478                                         sut_tsc = new_sut_tsc - old_sut_tsc
 479                                         if sut_tsc == 0 :
 480                                                 continue
 481                                 duration = duration - 1
 482                                 old_drop = new_drop
 483                                 old_rx = new_rx
 484                                 old_tx = new_tx
 485                                 old_tsc = new_tsc
 486                                 pps_req_tx = (tx+drop-rx)*tsc_hz*1.0/(tsc*1000000)
 487                                 pps_tx = tx*tsc_hz*1.0/(tsc*1000000)
 488                                 pps_rx = rx*tsc_hz*1.0/(tsc*1000000)
 489                                 if sutsock!='none':
 490                                         old_sut_tx = new_sut_tx
 491                                         old_sut_rx = new_sut_rx
 492                                         old_sut_tsc= new_sut_tsc
 493                                         pps_sut_tx = sut_tx*sut_tsc_hz*1.0/(sut_tsc*1000000)
 494                                         pps_sut_tx_str = '{:>7.3f}'.format(pps_sut_tx)
 495                                 else:
 496                                         pps_sut_tx = 0
 497                                         pps_sut_tx_str = 'NO MEAS.'
 498                                 if (tx == 0):
 499                                         log.critical("TX = 0. Test interrupted since no packet has been sent.")
 500                                         raise Exception("TX = 0")
 501                                 tot_drop = tot_drop + tx - rx
 502
 503                                 if pps_sut_tx_str !=  'NO_RESULTS':
 504                                         # First second mpps are not valid as there is no alignement between time the generator is started and per seconds stats
 505                                         if (first):
 506                                                 log.info('|{:>7}'.format(flow_number)+" |" + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+'             |' +'             |'  +'             |'+ '             |'+ '{:>8.0f}'.format(lat_avg)+' us  |'+'{:>8.0f}'.format(lat_max)+' us  | ' + '{:>9.0f}'.format(tx) + ' | '+ '{:>9.0f}'.format(rx) + ' | '+ '{:>7.0f}'.format(tx-rx) + ' | '+'{:>7.0f}'.format(tot_drop) +'    |'+lat_warning)
 507                                         else:
 508                                                 log.info('|{:>7}'.format(flow_number)+" |" + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+ '{:>7.3f}'.format(pps_req_tx)+' Mpps |'+ '{:>7.3f}'.format(pps_tx) +' Mpps |' + '{:>7}'.format(pps_sut_tx_str) +' Mpps |'+ '{:>7.3f}'.format(pps_rx)+' Mpps |'+ '{:>8.0f}'.format(lat_avg)+' us  |'+'{:>8.0f}'.format(lat_max)+' us  | ' + '{:>9.0f}'.format(tx) + ' | '+ '{:>9.0f}'.format(rx) + ' | '+ '{:>7.0f}'.format(tx-rx) + ' | '+ '{:>7.0f}'.format(tot_drop) +'    |'+lat_warning)
 509                                                 writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':get_pps(speed,size),'GeneratedPPS':pps_req_tx,'SentPPS':pps_tx,'ForwardedPPS':pps_sut_tx,'ReceivedPPS':pps_rx,'AvgLatencyUSEC':lat_avg,'MaxLatencyUSEC':lat_max,'Sent':tx,'Received':rx,'Lost':(tx-rx),'LostTotal':tot_drop})
 510                                                 if PushGateway:
 511                                                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
 512                                                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nSent {}\nReceived {}\nLost {}\nLostTotal {}\n'.format(flow_number,size+4,get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_max,tx,rx,(tx-rx),tot_drop)
 513                                                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 514                                                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 515                                 else:
 516                                         log.debug('|{:>7} | Speed 0 or close to 0'.format(str(size)))
 517                                 first = 0
 518                                 if (duration <= 0):
 519                                         #Stop generating
 520                                         gensock.stop(gencores)
 521                                         time.sleep(sleep_time)
 522                                         lat_min, lat_max, lat_avg, lat_used = gensock.lat_stats(latcores)
 523                                         if lat_used < 0.95:
 524                                                 lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 525                                         else:
 526                                                 lat_warning = ''
 527                                         # Get statistics after some execution time
 528                                         new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
 529                                         new_rx = new_rx - new_non_dp_rx
 530                                         new_tx = new_tx - new_non_dp_tx
 531                                         if sutsock!='none':
 532                                                 new_sut_rx, new_sut_non_dp_rx, new_sut_tx, new_sut_non_dp_tx, new_sut_drop, new_sut_tx_fail, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
 533                                                 new_sut_rx = new_sut_rx - new_sut_non_dp_rx
 534                                                 new_sut_tx = new_sut_tx - new_sut_non_dp_tx
 535                                         drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 536                                         rx = new_rx - old_rx     # rx is all packets received by the nop task = all packets received in the gen VM
 537                                         tx = new_tx - old_tx     # tx is all generated packets actually accepted by the interface
 538                                         tsc = new_tsc - old_tsc  # time difference between the 2 measurements, expressed in cycles.
 539                                         tot_drop = tot_drop + tx - rx
 540                                         if sutsock!='none':
 541                                                 sut_rx = new_sut_rx - old_sut_rx
 542                                                 sut_tx = new_sut_tx - old_sut_tx
 543                                                 sut_tsc = new_sut_tsc - old_sut_tsc
 544                                         if pps_sut_tx_str !=  'NO_RESULTS':
 545                                                 log.info('|{:>7}'.format(flow_number)+" |" + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+'             |' +'             |'  +'             |'+ '             |'+ '{:>8.0f}'.format(lat_avg)+' us  |'+'{:>8.0f}'.format(lat_max)+' us  | ' + '{:>9.0f}'.format(tx) + ' | '+ '{:>9.0f}'.format(rx) + ' | '+ '{:>7.0f}'.format(tx-rx) + ' | '+ '{:>7.0f}'.format(tot_drop) +'    |'+lat_warning)
 546                         log.info("+--------+------------------+-------------+-------------+-------------+-------------+-------------+-------------+-----------+-----------+---------+------------+")
 547         gensock.stop(latcores)
 548
 549 def run_core_stats(socks):
 550         fieldnames = ['PROXID','Time','Received','Sent','NonDPReceived','NonDPSent','Delta','NonDPDelta','Dropped']
 551         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 552         writer.writeheader()
 553         log.info("+------------------------------------------------------------------------------------------------------------------+")
 554         log.info("| Measuring core statistics on 1 or more PROX instances                                                            |")
 555         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 556         log.info("| PROX ID   |    Time   |    RX      |     TX     | non DP RX  | non DP TX  |   TX - RX  | nonDP TX-RX|  DROP TOT  |")
 557         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 558         for sock in socks:
 559                 sock.reset_stats()
 560         duration = float(runtime)
 561         tot_drop = []
 562         old_rx = []; old_non_dp_rx = []; old_tx = []; old_non_dp_tx = []; old_drop = []; old_tx_fail = []; old_tsc = []
 563         new_rx = []; new_non_dp_rx = []; new_tx = []; new_non_dp_tx = []; new_drop = []; new_tx_fail = []; new_tsc = []
 564         sockets_to_go = len (socks)
 565         for i,sock in enumerate(socks,start=0):
 566                 tot_drop.append(0)
 567                 old_rx.append(0); old_non_dp_rx.append(0); old_tx.append(0); old_non_dp_tx.append(0); old_drop.append(0); old_tx_fail.append(0); old_tsc.append(0)
 568                 old_rx[-1], old_non_dp_rx[-1], old_tx[-1], old_non_dp_tx[-1], old_drop[-1], old_tx_fail[-1], old_tsc[-1], tsc_hz = sock.core_stats(cores[i],tasks)
 569                 new_rx.append(0); new_non_dp_rx.append(0); new_tx.append(0); new_non_dp_tx.append(0); new_drop.append(0); new_tx_fail.append(0); new_tsc.append(0)
 570         while (duration > 0):
 571                 time.sleep(0.5)
 572                 # Get statistics after some execution time
 573                 for i,sock in enumerate(socks,start=0):
 574                         new_rx[i], new_non_dp_rx[i], new_tx[i], new_non_dp_tx[i], new_drop[i], new_tx_fail[i], new_tsc[i], tsc_hz = sock.core_stats(cores[i],tasks)
 575                         drop = new_drop[i]-old_drop[i]
 576                         rx = new_rx[i] - old_rx[i]
 577                         tx = new_tx[i] - old_tx[i]
 578                         non_dp_rx = new_non_dp_rx[i] - old_non_dp_rx[i]
 579                         non_dp_tx = new_non_dp_tx[i] - old_non_dp_tx[i]
 580                         tsc = new_tsc[i] - old_tsc[i]
 581                         if tsc == 0 :
 582                                 continue
 583                         sockets_to_go -= 1
 584                         old_drop[i] = new_drop[i]
 585                         old_rx[i] = new_rx[i]
 586                         old_tx[i] = new_tx[i]
 587                         old_non_dp_rx[i] = new_non_dp_rx[i]
 588                         old_non_dp_tx[i] = new_non_dp_tx[i]
 589                         old_tsc[i] = new_tsc[i]
 590                         tot_drop[i] = tot_drop[i] + tx - rx
 591                         log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(non_dp_rx)+' | '+'{:>10.0f}'.format(non_dp_tx)+' | ' + '{:>10.0f}'.format(tx-rx) + ' | '+ '{:>10.0f}'.format(non_dp_tx-non_dp_rx) + ' | '+'{:>10.0f}'.format(tot_drop[i]) +' |')
 592                         writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NonDPReceived':non_dp_rx,'NonDPSent':non_dp_tx,'Delta':tx-rx,'NonDPDelta':non_dp_tx-non_dp_rx,'Dropped':tot_drop[i]})
 593                         if PushGateway:
 594                                 URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
 595                                 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNonDPReceived {}\nNonDPSent {}\nDelta {}\nNonDPDelta {}\nDropped {}\n'.format(i,duration,rx,tx,non_dp_rx,non_dp_tx,tx-rx,non_dp_tx-non_dp_rx,tot_drop[i])
 596                                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 597                                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
 598                         if sockets_to_go == 0:
 599                                 duration = duration - 1
 600                                 sockets_to_go = len (socks)
 601         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
 602
 603 def run_port_stats(socks):
 604         fieldnames = ['PROXID','Time','Received','Sent','NoMbufs','iErrMiss']
 605         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 606         writer.writeheader()
 607         log.info("+---------------------------------------------------------------------------+")
 608         log.info("| Measuring port statistics on 1 or more PROX instances                     |")
 609         log.info("+-----------+-----------+------------+------------+------------+------------+")
 610         log.info("| PROX ID   |    Time   |    RX      |     TX     | no MBUFS   | ierr&imiss |")
 611         log.info("+-----------+-----------+------------+------------+------------+------------+")
 612         for sock in socks:
 613                 sock.reset_stats()
 614         duration = float(runtime)
 615         old_rx = []; old_tx = []; old_no_mbufs = []; old_errors = []; old_tsc = []
 616         new_rx = []; new_tx = []; new_no_mbufs = []; new_errors = []; new_tsc = []
 617         sockets_to_go = len (socks)
 618         for i,sock in enumerate(socks,start=0):
 619                 old_rx.append(0); old_tx.append(0); old_no_mbufs.append(0); old_errors.append(0); old_tsc.append(0)
 620                 old_rx[-1], old_tx[-1], old_no_mbufs[-1], old_errors[-1], old_tsc[-1] = sock.multi_port_stats(ports[i])
 621                 new_rx.append(0); new_tx.append(0); new_no_mbufs.append(0); new_errors.append(0); new_tsc.append(0)
 622         while (duration > 0):
 623                 time.sleep(0.5)
 624                 # Get statistics after some execution time
 625                 for i,sock in enumerate(socks,start=0):
 626                         new_rx[i], new_tx[i], new_no_mbufs[i], new_errors[i], new_tsc[i] = sock.multi_port_stats(ports[i])
 627                         rx = new_rx[i] - old_rx[i]
 628                         tx = new_tx[i] - old_tx[i]
 629                         no_mbufs = new_no_mbufs[i] - old_no_mbufs[i]
 630                         errors = new_errors[i] - old_errors[i]
 631                         tsc = new_tsc[i] - old_tsc[i]
 632                         if tsc == 0 :
 633                                 continue
 634                         sockets_to_go -= 1
 635                         old_rx[i] = new_rx[i]
 636                         old_tx[i] = new_tx[i]
 637                         old_no_mbufs[i] = new_no_mbufs[i]
 638                         old_errors[i] = new_errors[i]
 639                         old_tsc[i] = new_tsc[i]
 640                         log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(no_mbufs)+' | '+'{:>10.0f}'.format(errors)+' |')
 641                         writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NoMbufs':no_mbufs,'iErrMiss':errors})
 642                         if PushGateway:
 643                                 URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
 644                                 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNoMbufs {}\niErrMiss {}\n'.format(i,duration,rx,tx,no_mbufs,errors)
 645                                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 646                                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
 647                         if sockets_to_go == 0:
 648                                 duration = duration - 1
 649                                 sockets_to_go = len (socks)
 650         log.info("+-----------+-----------+------------+------------+------------+------------+")
 651
 652 def run_irqtest(socks):
 653         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 654         log.info("| Measuring time probably spent dealing with an interrupt. Interrupting DPDK cores for more than 50us might be problematic   ")
 655         log.info("| and result in packet loss. The first row shows the interrupted time buckets: first number is the bucket between 0us and    ")
 656         log.info("| that number expressed in us and so on. The numbers in the other rows show how many times per second, the program was       ")
 657         log.info("| interrupted for a time as specified by its bucket. '0' is printed when there are no interrupts in this bucket throughout   ")
 658         log.info("| the duration of the test. 0.00 means there were interrupts in this bucket but very few. Due to rounding this shows as 0.00 ")
 659         log.info("+----------------------------------------------------------------------------------------------------------------------------")
 660         sys.stdout.flush()
 661         for sock_index,sock in enumerate(socks,start=0):
 662                 buckets=socks[sock_index].show_irq_buckets(1)
 663                 print('Measurement ongoing ... ',end='\r')
 664                 socks[sock_index].stop(cores[sock_index])
 665                 old_irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
 666                 irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
 667                 irq[0][0] = 'bucket us'
 668                 for j,bucket in enumerate(buckets,start=1):
 669                         irq[0][j] = '<'+ bucket
 670                 irq[0][-1] = '>'+ buckets [-2]
 671                 socks[sock_index].start(cores[sock_index])
 672                 time.sleep(2)
 673                 for j,bucket in enumerate(buckets,start=1):
 674                         for i,irqcore in enumerate(cores[sock_index],start=1):
 675                                 old_irq[i][j] = socks[sock_index].irq_stats(irqcore,j-1)
 676                 time.sleep(float(runtime))
 677                 socks[sock_index].stop(cores[sock_index])
 678                 for i,irqcore in enumerate(cores[sock_index],start=1):
 679                         irq[i][0]='core %s '%irqcore
 680                         for j,bucket in enumerate(buckets,start=1):
 681                                 diff =  socks[sock_index].irq_stats(irqcore,j-1) - old_irq[i][j]
 682                                 if diff == 0:
 683                                         irq[i][j] = '0'
 684                                 else:
 685                                         irq[i][j] = str(round(diff/float(runtime), 2))
 686                 log.info('Results for PROX instance %s'%sock_index)
 687                 for row in irq:
 688                         log.info(''.join(['{:>12}'.format(item) for item in row]))
 689
 690 def run_impairtest(gensock,sutsock):
 691         fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Dropped','DropRate']
 692         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 693         writer.writeheader()
 694         size=PACKETSIZE-4
 695         log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 696         log.info("| Generator is sending UDP ("+'{:>5}'.format(FLOWSIZE)+" flow) packets ("+ '{:>5}'.format(size+4) +" bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test    |")
 697         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 698         log.info("| Test   |  Speed requested   | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
 699         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 700         attempts = 0
 701         gensock.set_size(gencores,0,size) # This is setting the frame size
 702         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 703         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 704         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 705         source_port,destination_port = flows[FLOWSIZE]
 706         gensock.set_random(gencores,0,34,source_port,2)
 707         gensock.set_random(gencores,0,36,destination_port,2)
 708         gensock.start(latcores)
 709         speed = STARTSPEED
 710         gensock.speed(speed / len(gencores) / len(gentasks), gencores, gentasks)
 711         while True:
 712                 attempts += 1
 713                 print('Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 714                 sys.stdout.flush()
 715                 time.sleep(1)
 716                 # Get statistics now that the generation is stable and NO ARP messages any more
 717                 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used = run_iteration(gensock,sutsock)
 718                 drop_rate = 100.0*abs_dropped/abs_tx
 719                 if lat_used < 0.95:
 720                         lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 721                 else:
 722                         lat_warning = ''
 723                 log.info('|{:>7}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | '+ '{:>9.3f}'.format(pps_tx) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9.3f}'.format(pps_rx)+' Mpps | '+ '{:>9.0f}'.format(lat_avg)+' us   | '+ '{:>9.0f}'.format(lat_max)+' us   | '+ '{:>14d}'.format(abs_dropped)+ ' |''{:>9.2f}'.format(drop_rate)+ '%  |'+lat_warning)
 724                 writer.writerow({'Flows':FLOWSIZE,'PacketSize':(size+4),'RequestedPPS':get_pps(speed,size),'GeneratedPPS':pps_req_tx,'SentPPS':pps_tx,'ForwardedPPS':pps_sut_tx_str,'ReceivedPPS':pps_rx,'AvgLatencyUSEC':lat_avg,'MaxLatencyUSEC':lat_max,'Dropped':abs_dropped,'DropRate':drop_rate})
 725                 if PushGateway:
 726                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
 727                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nDropped {}\nDropRate {}\n'.format(FLOWSIZE,size+4,get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,drop_rate)
 728                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 729                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 730
 731 def run_warmuptest(gensock):
 732 # Running at low speed to make sure the ARP messages can get through.
 733 # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
 734 # Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
 735 # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
 736         gensock.speed(WARMUPSPEED / len(gencores) /len (gentasks), gencores, gentasks)
 737         size=PACKETSIZE-4
 738         gensock.set_size(gencores,0,size) # This is setting the frame size
 739         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 740         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
 741         gensock.set_value(gencores,0,56,1,1)
 742         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
 743         source_port,destination_port = flows[FLOWSIZE]
 744         gensock.set_random(gencores,0,34,source_port,2)
 745         gensock.set_random(gencores,0,36,destination_port,2)
 746         gensock.start(genstatcores)
 747         time.sleep(WARMUPTIME)
 748         gensock.stop(genstatcores)
 749         gensock.set_value(gencores,0,56,50,1)
 750
 751 global sutstatcores
 752 global genstatcores
 753 global latcores
 754 global gencores
 755 global irqcores
 756 global PACKETSIZE
 757 global packet_size_list
 758 global FLOWSIZE
 759 global flow_size_list
 760 global WARMUPTIME
 761 global WARMUPSPEED
 762 global required_number_of_test_machines
 763 # To generate a desired number of flows, PROX will randomize the bits in source and destination ports, as specified by the bit masks in the flows variable.
 764 flows={\
 765 1:      ['1000000000000000','1000000000000000'],\
 766 2:      ['1000000000000000','100000000000000X'],\
 767 4:      ['100000000000000X','100000000000000X'],\
 768 8:      ['100000000000000X','10000000000000XX'],\
 769 16:     ['10000000000000XX','10000000000000XX'],\
 770 32:     ['10000000000000XX','1000000000000XXX'],\
 771 64:     ['1000000000000XXX','1000000000000XXX'],\
 772 128:    ['1000000000000XXX','100000000000XXXX'],\
 773 256:    ['100000000000XXXX','100000000000XXXX'],\
 774 512:    ['100000000000XXXX','10000000000XXXXX'],\
 775 1024:   ['10000000000XXXXX','10000000000XXXXX'],\
 776 2048:   ['10000000000XXXXX','1000000000XXXXXX'],\
 777 4096:   ['1000000000XXXXXX','1000000000XXXXXX'],\
 778 8192:   ['1000000000XXXXXX','100000000XXXXXXX'],\
 779 16384:  ['100000000XXXXXXX','100000000XXXXXXX'],\
 780 32768:  ['100000000XXXXXXX','10000000XXXXXXXX'],\
 781 65536:  ['10000000XXXXXXXX','10000000XXXXXXXX'],\
 782 131072: ['10000000XXXXXXXX','1000000XXXXXXXXX'],\
 783 262144: ['1000000XXXXXXXXX','1000000XXXXXXXXX'],\
 784 524288: ['1000000XXXXXXXXX','100000XXXXXXXXXX'],\
 785 1048576:['100000XXXXXXXXXX','100000XXXXXXXXXX'],}
 786 clients =[]
 787 socks =[]
 788 socks_control =[]
 789 vmDPIP =[]
 790 vmAdminIP =[]
 791 vmDPmac =[]
 792 hexDPIP =[]
 793 config_file =[]
 794 prox_socket =[]
 795 prox_launch_exit =[]
 796 auto_start =[]
 797 mach_type =[]
 798 sock_type =[]
 799 cores = []
 800 ports = []
 801 tasks = {}
 802
 803 data_file = 'RUN{}.{}.csv'.format(env,test_file)
 804 data_csv_file = open(data_file,'w')
 805 testconfig = ConfigParser.RawConfigParser()
 806 testconfig.read(test_file)
 807 required_number_of_test_machines = testconfig.get('DEFAULT', 'total_number_of_test_machines')
 808 TestName = testconfig.get('DEFAULT', 'name')
 809 if testconfig.has_option('DEFAULT', 'PushGateway'):
 810         PushGateway = testconfig.get('DEFAULT', 'PushGateway')
 811         log.info('Measurements will be pushed to %s'%PushGateway)
 812 else:
 813         PushGateway = None
 814 config = ConfigParser.RawConfigParser()
 815 config.read(env)
 816 machine_map = ConfigParser.RawConfigParser()
 817 machine_map.read(machine_map_file)
 818 key = config.get('ssh', 'key')
 819 user = config.get('ssh', 'user')
 820 total_number_of_machines = config.get('rapid', 'total_number_of_machines')
 821 if int(required_number_of_test_machines) > int(total_number_of_machines):
 822         log.exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
 823         raise Exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
 824 for vm in range(1, int(total_number_of_machines)+1):
 825         vmAdminIP.append(config.get('M%d'%vm, 'admin_ip'))
 826         vmDPmac.append(config.get('M%d'%vm, 'dp_mac'))
 827         vmDPIP.append(config.get('M%d'%vm, 'dp_ip'))
 828         ip = vmDPIP[-1].split('.')
 829         hexDPIP.append(hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2))
 830 machine_index = []
 831 for vm in range(1, int(required_number_of_test_machines)+1):
 832         machine_index.append(int(machine_map.get('TestM%d'%vm, 'machine_index'))-1)
 833         prox_socket.append(testconfig.getboolean('TestM%d'%vm, 'prox_socket'))
 834 for vm in range(1, int(required_number_of_test_machines)+1):
 835         if prox_socket[vm-1]:
 836                 prox_launch_exit.append(testconfig.getboolean('TestM%d'%vm, 'prox_launch_exit'))
 837                 config_file.append(testconfig.get('TestM%d'%vm, 'config_file'))
 838                 # Looking for all task definitions in the PROX cfg files. Constructing a list of all tasks used
 839                 textfile =  open (config_file[-1], 'r')
 840                 filetext = textfile.read()
 841                 textfile.close()
 842                 tasks_for_this_cfg = set(re.findall("task\s*=\s*(\d+)",filetext))
 843                 with open('{}_{}_parameters{}.lua'.format(env,test_file,vm), "w") as f:
 844                         f.write('name="%s"\n'% testconfig.get('TestM%d'%vm, 'name'))
 845                         f.write('local_ip="%s"\n'% vmDPIP[machine_index[vm-1]])
 846                         f.write('local_hex_ip="%s"\n'% hexDPIP[machine_index[vm-1]])
 847                         if testconfig.has_option('TestM%d'%vm, 'cores'):
 848                                 cores.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'cores')))
 849                                 f.write('cores="%s"\n'% ','.join(map(str, cores[-1])))
 850                         else:
 851                                 cores.append(None)
 852                         if testconfig.has_option('TestM%d'%vm, 'ports'):
 853                                 ports.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'ports')))
 854                                 f.write('ports="%s"\n'% ','.join(map(str, ports[-1])))
 855                         else:
 856                                 ports.append(None)
 857                         if re.match('(l2){0,1}gen(_bare){0,1}.*\.cfg',config_file[-1]):
 858                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
 859                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
 860                                 genstatcores = gencores + latcores
 861                                 gentasks = tasks_for_this_cfg
 862                                 auto_start.append(False)
 863                                 mach_type.append('gen')
 864                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
 865                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
 866                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 867                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 868                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 869                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 870                         elif re.match('(l2){0,1}gen_gw.*\.cfg',config_file[-1]):
 871                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
 872                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
 873                                 genstatcores = gencores + latcores
 874                                 gentasks = tasks_for_this_cfg
 875                                 auto_start.append(False)
 876                                 mach_type.append('gen')
 877                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
 878                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
 879                                 gwVMindex = int(testconfig.get('TestM%d'%vm, 'gw_vm')) -1
 880                                 f.write('gw_ip="%s"\n'% vmDPIP[machine_index[gwVMindex]])
 881                                 f.write('gw_hex_ip="%s"\n'% hexDPIP[machine_index[gwVMindex]])
 882                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 883                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 884                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 885                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 886                         elif re.match('(l2){0,1}swap.*\.cfg',config_file[-1]):
 887                                 sutstatcores = cores[-1]
 888                                 auto_start.append(True)
 889                                 mach_type.append('sut')
 890                         elif re.match('secgw1.*\.cfg',config_file[-1]):
 891                                 auto_start.append(True)
 892                                 mach_type.append('none')
 893                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
 894                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
 895                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
 896                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
 897                         elif re.match('secgw2.*\.cfg',config_file[-1]):
 898                                 sutstatcores = cores[-1]
 899                                 auto_start.append(True)
 900                                 mach_type.append('sut')
 901                         else:
 902                                 auto_start.append(True)
 903                                 mach_type.append('none')
 904                 f.close
 905                 tasks = tasks_for_this_cfg.union(tasks)
 906 log.debug("Tasks detected in all PROX config files %r"%tasks)
 907 #####################################################################################
 908 def exit_handler():
 909         log.debug ('exit cleanup')
 910         for index, sock in enumerate(socks):
 911                 if socks_control[index]:
 912                         sock.quit()
 913         for client in clients:
 914                 client.close()
 915         data_csv_file.close
 916         sys.exit(0)
 917
 918 atexit.register(exit_handler)
 919
 920 for vm in range(0, int(required_number_of_test_machines)):
 921         if prox_socket[vm]:
 922                 clients.append(prox_ctrl(vmAdminIP[machine_index[vm]], key,user))
 923                 connect_client(clients[-1])
 924 # Creating script to bind the right network interface to the poll mode driver
 925                 devbindfile = '{}_{}_devbindvm{}.sh'.format(env,test_file, vm+1)
 926                 with open(devbindfile, "w") as f:
 927                         newText= 'link="$(ip -o link | grep '+vmDPmac[machine_index[vm]]+' |cut -d":" -f 2)"\n'
 928                         f.write(newText)
 929                         newText= 'if [ -n "$link" ];\n'
 930                         f.write(newText)
 931                         newText= 'then\n'
 932                         f.write(newText)
 933                         newText= '        echo Need to bind\n'
 934                         f.write(newText)
 935                         newText= '        sudo ' + rundir + '/dpdk/usertools/dpdk-devbind.py --force --bind igb_uio $('+rundir+'/dpdk/usertools/dpdk-devbind.py --status |grep  $link | cut -d" " -f 1)\n'
 936                         f.write(newText)
 937                         newText= 'else\n'
 938                         f.write(newText)
 939                         newText= '       echo Assuming port is already bound to DPDK\n'
 940                         f.write(newText)
 941                         newText= 'fi\n'
 942                         f.write(newText)
 943                         newText= 'exit 0\n'
 944                         f.write(newText)
 945                 st = os.stat(devbindfile)
 946                 os.chmod(devbindfile, st.st_mode | stat.S_IEXEC)
 947                 clients[-1].scp_put('./%s'%devbindfile, rundir+'/devbind.sh')
 948                 cmd = 'sudo ' + rundir+ '/devbind.sh'
 949                 clients[-1].run_cmd(cmd)
 950                 log.debug("devbind.sh running on VM%d"%(vm+1))
 951                 clients[-1].scp_put('./%s'%config_file[vm], rundir+'/%s'%config_file[vm])
 952                 clients[-1].scp_put('./{}_{}_parameters{}.lua'.format(env,test_file, vm+1), rundir + '/parameters.lua')
 953                 if not configonly:
 954                         if prox_launch_exit[vm]:
 955                                 log.debug("Starting PROX on VM%d"%(vm+1))
 956                                 if auto_start[vm]:
 957                                         cmd = 'sudo ' +rundir + '/prox/build/prox -t -o cli -f ' + rundir + '/%s'%config_file[vm]
 958                                 else:
 959                                         cmd = 'sudo ' +rundir + '/prox/build/prox -e -t -o cli -f ' + rundir + '/%s'%config_file[vm]
 960                                 clients[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1))
 961                         socks_control.append(prox_launch_exit[vm])
 962                         socks.append(connect_socket(clients[-1]))
 963                         sock_type.append(mach_type[vm])
 964
 965 def get_BinarySearchParams() :
 966         global DROP_RATE_TRESHOLD
 967         global LAT_AVG_TRESHOLD
 968         global LAT_MAX_TRESHOLD
 969         global ACCURACY
 970         global STARTSPEED
 971         DROP_RATE_TRESHOLD = float(testconfig.get('BinarySearchParams', 'drop_rate_threshold'))
 972         LAT_AVG_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_avg_threshold'))
 973         LAT_MAX_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_max_threshold'))
 974         ACCURACY = float(testconfig.get('BinarySearchParams', 'accuracy'))
 975         STARTSPEED = float(testconfig.get('BinarySearchParams', 'startspeed'))
 976
 977 if configonly:
 978         sys.exit()
 979 ####################################################
 980 # Run test cases
 981 # Best to run the flow test at the end since otherwise the tests coming after might be influenced by the big number of entries in the switch flow tables
 982 ####################################################
 983 gensock_index = sock_type.index('gen') if 'gen' in sock_type else -1
 984 sutsock_index = sock_type.index('sut') if 'sut' in sock_type else -1
 985 number_of_tests = testconfig.get('DEFAULT', 'number_of_tests')
 986 for test_nr in range(1, int(number_of_tests)+1):
 987         test=testconfig.get('test%d'%test_nr,'test')
 988         log.info(test)
 989         if test == 'flowsizetest':
 990                 get_BinarySearchParams()
 991                 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
 992                 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
 993                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
 994         elif test == 'fixed_rate':
 995                 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
 996                 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
 997                 STARTSPEED = float(testconfig.get('test%d'%test_nr, 'speed'))
 998                 run_fixed_rate(socks[gensock_index],socks[sutsock_index])
 999         elif test == 'corestats':
1000                 run_core_stats(socks)
1001         elif test == 'portstats':
1002                 run_port_stats(socks)
1003         elif test == 'impairtest':
1004                 get_BinarySearchParams()
1005                 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1006                 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1007                 run_impairtest(socks[gensock_index],socks[sutsock_index])
1008         elif test == 'irqtest':
1009                 run_irqtest(socks)
1010         elif test == 'warmuptest':
1011                 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1012                 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1013                 WARMUPSPEED = int(testconfig.get('test%d'%test_nr, 'warmupspeed'))
1014                 WARMUPTIME = int(testconfig.get('test%d'%test_nr, 'warmuptime'))
1015                 run_warmuptest(socks[gensock_index])
1016 ####################################################