4 ## Copyright (c) 2010-2020 Intel Corporation
6 ## Licensed under the Apache License, Version 2.0 (the "License");
7 ## you may not use this file except in compliance with the License.
8 ## You may obtain a copy of the License at
10 ## http://www.apache.org/licenses/LICENSE-2.0
12 ## Unless required by applicable law or agreed to in writing, software
13 ## distributed under the License is distributed on an "AS IS" BASIS,
14 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 ## See the License for the specific language governing permissions and
16 ## limitations under the License.
19 from __future__ import print_function
29 from logging.handlers import RotatingFileHandler
30 from logging import handlers
31 from prox_ctrl import prox_ctrl
41 env = "rapid.env" #Default string for environment
42 test_file = "basicrapid.test" #Default string for test
43 machine_map_file = "machine.map" #Default string for machine map file
44 loglevel="DEBUG" # sets log level for writing to file
45 screenloglevel="INFO" # sets log level for writing to screen
46 runtime=10 # time in seconds for 1 test run
47 configonly = False # IF True, the system will upload all the necessary config fiels to the VMs, but not start PROX and the actual testing
48 #rundir = "/home/centos" # Directory where to find the tools in the machines running PROX
49 rundir = "~" # Directory where to find the tools in the machines running PROX
52 print("usage: runrapid [--version] [-v]")
53 print(" [--env ENVIRONMENT_NAME]")
54 print(" [--test TEST_NAME]")
55 print(" [--map MACHINE_MAP_FILE]")
56 print(" [--runtime TIME_FOR_TEST]")
57 print(" [--configonly False|True]")
58 print(" [--log DEBUG|INFO|WARNING|ERROR|CRITICAL]")
59 print(" [-h] [--help]")
61 print("Command-line interface to runrapid")
63 print("optional arguments:")
64 print(" -v, --version Show program's version number and exit")
65 print(" --env ENVIRONMENT_NAME Parameters will be read from ENVIRONMENT_NAME. Default is %s."%env)
66 print(" --test TEST_NAME Test cases will be read from TEST_NAME. Default is %s."%test_file)
67 print(" --map MACHINE_MAP_FILE Machine mapping will be read from MACHINE_MAP_FILE. Default is %s."%machine_map_file)
68 print(" --runtime Specify time in seconds for 1 test run")
69 print(" --configonly If this option is specified, only upload all config files to the VMs, do not run the tests")
70 print(" --log Specify logging level for log file output, default is DEBUG")
71 print(" --screenlog Specify logging level for screen output, default is INFO")
72 print(" -h, --help Show help message and exit.")
76 opts, args = getopt.getopt(sys.argv[1:], "vh", ["version","help", "env=", "test=", "map=", "runtime=","configonly","log=","screenlog="])
77 except getopt.GetoptError as err:
78 print("===========================================")
80 print("===========================================")
87 if opt in ["-h", "--help"]:
90 if opt in ["-v", "--version"]:
91 print("Rapid Automated Performance Indication for Dataplane "+version)
98 machine_map_file = arg
99 if opt in ["--runtime"]:
101 if opt in ["--configonly"]:
103 print('No actual runs, only uploading configuration files')
106 print ("Log level: "+ loglevel)
107 if opt in ["--screenlog"]:
109 print ("Screen Log level: "+ screenloglevel)
111 print ("Using '"+env+"' as name for the environment")
112 print ("Using '"+test_file+"' for test case definition")
113 print ("Using '"+machine_map_file+"' for machine mapping")
114 print ("Runtime: "+ str(runtime))
124 UNDERLINE = '\033[4m'
128 screen_formatter = logging.Formatter("%(message)s")
129 file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
131 # get a top-level logger,
133 # BUT PREVENT IT from propagating messages to the root logger
135 log = logging.getLogger()
136 numeric_level = getattr(logging, loglevel.upper(), None)
137 if not isinstance(numeric_level, int):
138 raise ValueError('Invalid log level: %s' % loglevel)
139 log.setLevel(numeric_level)
142 # create a console handler
143 # and set its log level to the command-line option
145 console_handler = logging.StreamHandler(sys.stdout)
146 #console_handler.setLevel(logging.INFO)
147 numeric_screenlevel = getattr(logging, screenloglevel.upper(), None)
148 if not isinstance(numeric_screenlevel, int):
149 raise ValueError('Invalid screenlog level: %s' % screenloglevel)
150 console_handler.setLevel(numeric_screenlevel)
151 console_handler.setFormatter(screen_formatter)
153 # create a file handler
154 # and set its log level
156 log_file = 'RUN{}.{}.log'.format(env,test_file)
157 file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10)
158 #file_handler = log.handlers.TimedRotatingFileHandler(log_file, 'D', 1, 5)
159 file_handler.setLevel(numeric_level)
160 file_handler.setFormatter(file_formatter)
162 # add handlers to the logger
164 log.addHandler(file_handler)
165 log.addHandler(console_handler)
167 # Check if log exists and should therefore be rolled
168 needRoll = os.path.isfile(log_file)
171 # This is a stale log, so roll it
174 log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime())
176 # Roll over on application start
177 log.handlers[0].doRollover()
180 log.debug('\n---------\nLog started on %s.\n---------\n' % time.asctime())
182 log.debug("runrapid.py version: "+version)
183 #========================================================================
184 def connect_socket(client):
186 log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
189 sock = client.prox_sock()
194 log.exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
195 raise Exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
197 log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
198 log.info("Connected to PROX on %s" % client.ip())
201 def connect_client(client):
203 log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
208 except RuntimeWarning, ex:
211 log.exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
212 raise Exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
214 log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
215 log.debug("Connected to VM on %s" % client.ip())
217 def report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,tx,rx,tot_drop,elapsed_time,speed_prefix='',lat_avg_prefix='',lat_perc_prefix='',lat_max_prefix='',abs_drop_rate_prefix='',drop_rate_prefix=''):
219 flow_number_str = '| ({:>4}) |'.format(abs(flow_number))
221 flow_number_str = '|{:>7} |'.format(flow_number)
222 if pps_req_tx is None:
223 pps_req_tx_str = '{0: >14}'.format(' NA |')
225 pps_req_tx_str = '{:>7.3f} Mpps |'.format(pps_req_tx)
227 pps_tx_str = '{0: >14}'.format(' NA |')
229 pps_tx_str = '{:>7.3f} Mpps |'.format(pps_tx)
230 if pps_sut_tx is None:
231 pps_sut_tx_str = '{0: >14}'.format(' NA |')
233 pps_sut_tx_str = '{:>7.3f} Mpps |'.format(pps_sut_tx)
235 pps_rx_str = '{0: >25}'.format('NA |')
237 pps_rx_str = bcolors.OKBLUE + '{:>4.1f} Gb/s |{:7.3f} Mpps {}|'.format(get_speed(pps_rx,size),pps_rx,bcolors.ENDC)
239 tot_drop_str = ' | NA | '
241 tot_drop_str = ' | {:>9.0f} | '.format(tot_drop)
243 lat_perc_str = ' |{:^10.10}|'.format('NA')
244 elif lat_perc_max == True:
245 lat_perc_str = '|>{}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC)
247 lat_perc_str = '| {}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC)
248 if elapsed_time is None:
249 elapsed_time_str = ' NA |'
251 elapsed_time_str = '{:>3.0f} |'.format(elapsed_time)
252 return(flow_number_str + '{:>5.1f}'.format(speed) + '% '+speed_prefix +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+ pps_req_tx_str + pps_tx_str + bcolors.ENDC + pps_sut_tx_str + pps_rx_str +lat_avg_prefix+ ' {:>6.0f}'.format(lat_avg)+' us'+lat_perc_str+lat_max_prefix+'{:>6.0f}'.format(lat_max)+' us | ' + '{:>9.0f}'.format(tx) + ' | {:>9.0f}'.format(rx) + ' | '+ abs_drop_rate_prefix+ '{:>9.0f}'.format(tx-rx) + tot_drop_str +drop_rate_prefix+ '{:>5.2f}'.format(float(tx-rx)/tx) +bcolors.ENDC+' |' + elapsed_time_str)
254 def run_iteration(gensock, sutsock, requested_duration,flow_number,size,speed):
257 while (r < TST009_MAXr):
258 time.sleep(sleep_time)
259 # Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
260 t1_rx, t1_non_dp_rx, t1_tx, t1_non_dp_tx, t1_drop, t1_tx_fail, t1_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
261 t1_dp_rx = t1_rx - t1_non_dp_rx
262 t1_dp_tx = t1_tx - t1_non_dp_tx
263 gensock.start(gencores)
264 time.sleep(2) ## Needs to be 2 seconds since this 1 sec is the time that PROX uses to refresh the stats. Note that this can be changed in PROX!! Don't do it.
266 t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
267 t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
269 dp_tx = tx - (t2_non_dp_tx - t1_non_dp_tx )
270 dp_rx = t2_rx - t1_rx - (t2_non_dp_rx - t1_non_dp_rx)
271 tot_dp_drop = dp_tx - dp_rx
273 log.critical("TX = 0. Test interrupted since no packet has been sent.")
274 raise Exception("TX = 0")
276 log.critical("Only non-dataplane packets (e.g. ARP) sent. Test interrupted since no packet has been sent.")
277 raise Exception("Only non-dataplane packets (e.g. ARP) sent")
278 # Ask PROX to calibrate the bucket size once we have a PROX function to do this.
279 # Measure latency statistics per second
280 lat_min, lat_max, lat_avg, used_avg, t2_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
281 lat_samples = sum(buckets)
283 for sample_percentile, bucket in enumerate(buckets,start=1):
284 sample_count += bucket
285 if sample_count > (lat_samples * LAT_PERCENTILE):
287 percentile_max = (sample_percentile == len(buckets))
288 sample_percentile = sample_percentile * float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
289 if test == 'fixed_rate':
290 log.info(report_result(flow_number,size,speed,None,None,None,None,lat_avg,sample_percentile,percentile_max,lat_max, dp_tx, dp_rx , None, None))
291 tot_rx = tot_non_dp_rx = tot_tx = tot_non_dp_tx = tot_drop = 0
292 lat_avg = used_avg = 0
293 buckets_total = [0] * 128
295 tot_lat_measurement_duration = float(0)
296 tot_core_measurement_duration = float(0)
297 tot_sut_core_measurement_duration = float(0)
298 tot_sut_rx = tot_sut_non_dp_rx = tot_sut_tx = tot_sut_non_dp_tx = tot_sut_drop = tot_sut_tx_fail = tot_sut_tsc = 0
299 lat_avail = core_avail = sut_avail = False
300 while (tot_core_measurement_duration - float(requested_duration) <= 0.1) or (tot_lat_measurement_duration - float(requested_duration) <= 0.1):
302 lat_min_sample, lat_max_sample, lat_avg_sample, used_sample, t3_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
303 # Get statistics after some execution time
304 if t3_lat_tsc != t2_lat_tsc:
305 single_lat_measurement_duration = (t3_lat_tsc - t2_lat_tsc) * 1.0 / lat_hz # time difference between the 2 measurements, expressed in seconds.
306 # A second has passed in between to lat_stats requests. Hence we need to process the results
307 tot_lat_measurement_duration = tot_lat_measurement_duration + single_lat_measurement_duration
308 if lat_min > lat_min_sample:
309 lat_min = lat_min_sample
310 if lat_max < lat_max_sample:
311 lat_max = lat_max_sample
312 lat_avg = lat_avg + lat_avg_sample * single_lat_measurement_duration # Sometimes, There is more than 1 second between 2 lat_stats. Hence we will take the latest measurement
313 used_avg = used_avg + used_sample * single_lat_measurement_duration # and give it more weigth.
314 lat_samples = sum(buckets)
315 tot_lat_samples += lat_samples
317 for sample_percentile, bucket in enumerate(buckets,start=1):
318 sample_count += bucket
319 if sample_count > lat_samples * LAT_PERCENTILE:
321 percentile_max = (sample_percentile == len(buckets))
322 sample_percentile = sample_percentile * float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
323 buckets_total = [buckets_total[i] + buckets[i] for i in range(len(buckets_total))]
324 t2_lat_tsc = t3_lat_tsc
326 t3_rx, t3_non_dp_rx, t3_tx, t3_non_dp_tx, t3_drop, t3_tx_fail, t3_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
328 single_core_measurement_duration = (t3_tsc - t2_tsc) * 1.0 / tsc_hz # time difference between the 2 measurements, expressed in seconds.
329 tot_core_measurement_duration = tot_core_measurement_duration + single_core_measurement_duration
330 delta_rx = t3_rx - t2_rx
332 delta_non_dp_rx = t3_non_dp_rx - t2_non_dp_rx
333 tot_non_dp_rx += delta_non_dp_rx
334 delta_tx = t3_tx - t2_tx
336 delta_non_dp_tx = t3_non_dp_tx - t2_non_dp_tx
337 tot_non_dp_tx += delta_non_dp_tx
338 delta_dp_tx = delta_tx -delta_non_dp_tx
339 delta_dp_rx = delta_rx -delta_non_dp_rx
340 delta_dp_drop = delta_dp_tx - delta_dp_rx
341 tot_dp_drop += delta_dp_drop
342 delta_drop = t3_drop - t2_drop
343 tot_drop += delta_drop
344 t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc = t3_rx, t3_non_dp_rx, t3_tx, t3_non_dp_tx, t3_drop, t3_tx_fail, t3_tsc
347 t3_sut_rx, t3_sut_non_dp_rx, t3_sut_tx, t3_sut_non_dp_tx, t3_sut_drop, t3_sut_tx_fail, t3_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
348 if t3_sut_tsc != t2_sut_tsc:
349 single_sut_core_measurement_duration = (t3_sut_tsc - t2_sut_tsc) * 1.0 / tsc_hz # time difference between the 2 measurements, expressed in seconds.
350 tot_sut_core_measurement_duration = tot_sut_core_measurement_duration + single_sut_core_measurement_duration
351 tot_sut_rx += t3_sut_rx - t2_sut_rx
352 tot_sut_non_dp_rx += t3_sut_non_dp_rx - t2_sut_non_dp_rx
353 delta_sut_tx = t3_sut_tx - t2_sut_tx
354 tot_sut_tx += delta_sut_tx
355 delta_sut_non_dp_tx = t3_sut_non_dp_tx - t2_sut_non_dp_tx
356 tot_sut_non_dp_tx += delta_sut_non_dp_tx
357 t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc = t3_sut_rx, t3_sut_non_dp_rx, t3_sut_tx, t3_sut_non_dp_tx, t3_sut_drop, t3_sut_tx_fail, t3_sut_tsc
359 if test == 'fixed_rate':
360 if lat_avail == core_avail == True:
361 lat_avail = core_avail = False
362 pps_req_tx = (delta_tx + delta_drop - delta_rx)/single_core_measurement_duration/1000000
363 pps_tx = delta_tx/single_core_measurement_duration/1000000
364 if sutsock!='none' and sut_avail:
365 pps_sut_tx = delta_sut_tx/single_sut_core_measurement_duration/1000000
369 pps_rx = delta_rx/single_core_measurement_duration/1000000
370 log.info(report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg_sample,sample_percentile,percentile_max,lat_max_sample,delta_dp_tx,delta_dp_rx,tot_dp_drop,single_core_measurement_duration))
372 gensock.stop(gencores)
374 lat_avg = lat_avg / float(tot_lat_measurement_duration)
375 used_avg = used_avg / float(tot_lat_measurement_duration)
377 while t4_tsc == t2_tsc:
378 t4_rx, t4_non_dp_rx, t4_tx, t4_non_dp_tx, t4_drop, t4_tx_fail, t4_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
379 if test == 'fixed_rate':
380 t4_lat_tsc = t2_lat_tsc
381 while t4_lat_tsc == t2_lat_tsc:
382 lat_min_sample, lat_max_sample, lat_avg_sample, used_sample, t4_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
384 lat_samples = sum(buckets)
385 for percentile, bucket in enumerate(buckets,start=1):
386 sample_count += bucket
387 if sample_count > lat_samples * LAT_PERCENTILE:
389 percentile_max = (percentile == len(buckets))
390 percentile = percentile * float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
391 lat_max = lat_max_sample
392 lat_avg = lat_avg_sample
393 delta_rx = t4_rx - t2_rx
394 delta_non_dp_rx = t4_non_dp_rx - t2_non_dp_rx
395 delta_tx = t4_tx - t2_tx
396 delta_non_dp_tx = t4_non_dp_tx - t2_non_dp_tx
397 delta_dp_tx = delta_tx -delta_non_dp_tx
398 delta_dp_rx = delta_rx -delta_non_dp_rx
401 tot_dp_drop += delta_dp_tx - delta_dp_rx
406 drop_rate = 100.0*(dp_tx-dp_rx)/dp_tx
407 tot_core_measurement_duration = None
408 break ## Not really needed since the while loop will stop when evaluating the value of r
411 for percentile, bucket in enumerate(buckets_total,start=1):
412 sample_count += bucket
413 if sample_count > tot_lat_samples * LAT_PERCENTILE:
415 percentile_max = (percentile == len(buckets_total))
416 percentile = percentile * float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
417 pps_req_tx = (tot_tx + tot_drop - tot_rx)/tot_core_measurement_duration/1000000.0 # tot_drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
418 pps_tx = tot_tx/tot_core_measurement_duration/1000000.0 # tot_tx is all generated packets actually accepted by the interface
419 pps_rx = tot_rx/tot_core_measurement_duration/1000000.0 # tot_rx is all packets received by the nop task = all packets received in the gen VM
420 if sutsock!='none' and sut_avail:
421 pps_sut_tx = tot_sut_tx / tot_sut_core_measurement_duration / 1000000.0
424 dp_tx = (t4_tx - t1_tx) - (t4_non_dp_tx - t1_non_dp_tx)
425 dp_rx = (t4_rx - t1_rx) - (t4_non_dp_rx - t1_non_dp_rx)
426 tot_dp_drop = dp_tx - dp_rx
427 drop_rate = 100.0*tot_dp_drop/dp_tx
428 if ((drop_rate < DROP_RATE_TRESHOLD) or (tot_dp_drop == DROP_RATE_TRESHOLD ==0) or (tot_dp_drop > TST009_MAXz)):
430 return(pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,percentile,percentile_max,lat_max,dp_tx,dp_rx,tot_dp_drop,(t4_tx_fail - t1_tx_fail),drop_rate,lat_min,used_avg,r,tot_core_measurement_duration)
432 def new_speed(speed,size,success):
433 if test == 'fixed_rate':
440 TST009_L = TST009_m + 1
442 TST009_R = max(TST009_m - 1, TST009_L)
443 TST009_m = int ((TST009_L + TST009_R)/2)
444 return (get_percentageof10Gbps(TST009_S[TST009_m],size))
452 return ((minspeed + maxspeed)/2.0)
454 def get_start_speed_and_init(size):
455 if test == 'fixed_rate':
462 TST009_R = TST009_n - 1
463 TST009_m = int((TST009_L + TST009_R) / 2)
464 return (get_percentageof10Gbps(TST009_S[TST009_m],size))
469 maxspeed = STARTSPEED
472 def resolution_achieved():
473 if test == 'fixed_rate':
476 return (TST009_L == TST009_R)
478 return ((maxspeed - minspeed) <= ACCURACY)
480 def get_percentageof10Gbps(pps_speed,size):
481 # speed is given in pps, returning % of 10Gb/s
482 return (pps_speed / 1000000.0 * 0.08 * (size+24))
484 def get_pps(speed,size):
485 # speed is given in % of 10Gb/s, returning Mpps
486 return (speed * 100.0 / (8*(size+24)))
488 def get_speed(packet_speed,size):
489 # return speed in Gb/s
490 return (packet_speed / 1000.0 * (8*(size+24)))
492 def set_background_flows(source_port,destination_port):
493 for sock in background_gen_socks:
494 sock.set_random(gencores,0,34,source_port,2)
495 sock.set_random(gencores,0,36,destination_port,2)
497 def set_background_speed(speed):
498 for sock in background_gen_socks:
499 sock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
501 def start_background_traffic():
502 for sock in background_gen_socks:
503 sock.start(gencores+latcores)
505 def stop_background_traffic():
506 for sock in background_gen_socks:
507 sock.stop(gencores+latcores)
509 def run_flow_size_test(gensock,sutsock):
512 #fieldnames = ['Flows','PacketSize','Gbps','Mpps','AvgLatency','MaxLatency','PacketsDropped','PacketDropRate']
513 fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Sent','Received','Lost','LostTotal']
514 writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
516 gensock.start(latcores)
517 for size in packet_size_list:
519 gensock.set_size(gencores,0,size) # This is setting the frame size
520 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
521 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
522 # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
523 if background_gen_socks:
524 backgroundinfo = '{}Running {} x background traffic not represented in the table{}'.format(bcolors.FLASH,len(background_gen_socks),bcolors.ENDC)
526 backgroundinfo = '{}{}'.format(bcolors.FLASH,bcolors.ENDC)
527 log.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
528 log.info('| UDP, {:>5} bytes, different number of flows by randomizing SRC & DST UDP port. {:116.116}|'.format((size+4),backgroundinfo))
529 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
530 log.info('| Flows | Speed requested | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core | Avg. Lat.|{:.0f} Pcentil| Max. Lat.| Sent | Received | Lost | Total Lost|L.Ratio|Time|'.format(LAT_PERCENTILE*100))
531 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
532 for flow_number in flow_size_list:
534 gensock.reset_stats()
536 sutsock.reset_stats()
537 source_port,destination_port = flows[flow_number]
538 gensock.set_random(gencores,0,34,source_port,2)
539 gensock.set_random(gencores,0,36,destination_port,2)
540 set_background_flows(source_port,destination_port)
542 speed = get_start_speed_and_init(size)
546 print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '% ',end='\r')
548 # Start generating packets at requested speed (in % of a 10Gb/s link)
549 gensock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
550 set_background_speed(speed)
551 start_background_traffic()
553 # Get statistics now that the generation is stable and initial ARP messages are dealt with
554 pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc , lat_perc_max, lat_max, abs_tx,abs_rx,abs_dropped, abs_tx_fail, drop_rate, lat_min, lat_used, r, actual_duration = run_iteration(gensock,sutsock,float(runtime),flow_number,size,speed)
555 stop_background_traffic()
557 retry_warning = bcolors.WARNING + ' {:1} retries needed'.format(r) + bcolors.ENDC
560 # Drop rate is expressed in percentage. lat_used is a ratio (0 to 1). The sum of these 2 should be 100%.
561 # If the sum is lower than 95, it means that more than 5% of the latency measurements where dropped for accuracy reasons.
562 if (drop_rate + lat_used * 100) < 95:
563 lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) + bcolors.ENDC
566 if test == 'fixed_rate':
573 endlat_perc = lat_perc
574 endlat_perc_max = lat_perc_max
576 endabs_dropped = abs_dropped
577 enddrop_rate = drop_rate
580 if lat_warning or retry_warning:
581 endwarning = '| | {:177.177} |'.format(retry_warning + lat_warning)
583 speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
584 # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold
585 # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
586 # This can be specified by putting 0 in the .test file
587 elif ((drop_rate < DROP_RATE_TRESHOLD) or (abs_dropped==DROP_RATE_TRESHOLD==0)) and (lat_avg< LAT_AVG_TRESHOLD) and (lat_perc< LAT_PERC_TRESHOLD) and (lat_max < LAT_MAX_TRESHOLD):
588 if ((get_pps(speed,size) - pps_tx)/get_pps(speed,size))>0.01:
589 speed_prefix = bcolors.WARNING
591 gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
593 gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(get_pps(speed,size), pps_tx) + bcolors.ENDC
595 speed_prefix = bcolors.ENDC
598 endspeed_prefix = speed_prefix
599 endpps_req_tx = pps_req_tx
601 endpps_sut_tx = pps_sut_tx
604 endlat_perc = lat_perc
605 endlat_perc_max = lat_perc_max
607 endabs_dropped = None
608 enddrop_rate = drop_rate
611 if lat_warning or gen_warning or retry_warning:
612 endwarning = '| | {:177.177} |'.format(retry_warning + lat_warning + gen_warning)
614 success_message=' SUCCESS'
615 speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
616 log.debug(report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
618 success_message=' FAILED'
620 abs_drop_rate_prefix = bcolors.ENDC
621 if ((abs_dropped>0) and (DROP_RATE_TRESHOLD ==0)):
622 abs_drop_rate_prefix = bcolors.FAIL
623 if (drop_rate < DROP_RATE_TRESHOLD):
624 drop_rate_prefix = bcolors.ENDC
626 drop_rate_prefix = bcolors.FAIL
627 if (lat_avg< LAT_AVG_TRESHOLD):
628 lat_avg_prefix = bcolors.ENDC
630 lat_avg_prefix = bcolors.FAIL
631 if (lat_perc< LAT_PERC_TRESHOLD):
632 lat_perc_prefix = bcolors.ENDC
634 lat_perc_prefix = bcolors.FAIL
635 if (lat_max< LAT_MAX_TRESHOLD):
636 lat_max_prefix = bcolors.ENDC
638 lat_max_prefix = bcolors.FAIL
639 if (((get_pps(speed,size) - pps_tx)/get_pps(speed,size))<0.001):
640 speed_prefix = bcolors.ENDC
642 speed_prefix = bcolors.FAIL
644 log.debug(report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
645 speed = new_speed(speed, size, success)
646 if resolution_achieved():
648 if endspeed is not None:
649 speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
650 log.info(report_result(flow_number,size,endspeed,endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_perc,endlat_perc_max,endlat_max,endabs_tx,endabs_rx,endabs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix))
652 log.info (endwarning)
653 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
654 writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':get_pps(endspeed,size),'GeneratedPPS':endpps_req_tx,'SentPPS':endpps_tx,'ForwardedPPS':endpps_sut_tx,'ReceivedPPS':endpps_rx,'AvgLatencyUSEC':endlat_avg,'MaxLatencyUSEC':endlat_max,'Sent':endabs_tx,'Received':endabs_rx,'Lost':endabs_dropped,'LostTotal':endabs_dropped})
656 URL = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
657 if endabs_dropped == None:
661 DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nSent {}\nReceived {}\nLost {}\nLostTotal {}\n'.format(flow_number,size+4,get_pps(endspeed,size),endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_max,endabs_tx,endabs_rx,ead,ead)
662 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
663 response = requests.post(url=URL, data=DATA,headers=HEADERS)
664 if (response.status_code != 202) and (response.status_code != 200):
665 log.info('Cannot send metrics to {}'.format(URL))
668 log.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
669 gensock.stop(latcores)
671 def run_core_stats(socks):
672 fieldnames = ['PROXID','Time','Received','Sent','NonDPReceived','NonDPSent','Delta','NonDPDelta','Dropped']
673 writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
675 log.info("+------------------------------------------------------------------------------------------------------------------+")
676 log.info("| Measuring core statistics on 1 or more PROX instances |")
677 log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
678 log.info("| PROX ID | Time | RX | TX | non DP RX | non DP TX | TX - RX | nonDP TX-RX| DROP TOT |")
679 log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
682 duration = float(runtime)
684 old_rx = []; old_non_dp_rx = []; old_tx = []; old_non_dp_tx = []; old_drop = []; old_tx_fail = []; old_tsc = []
685 new_rx = []; new_non_dp_rx = []; new_tx = []; new_non_dp_tx = []; new_drop = []; new_tx_fail = []; new_tsc = []
686 sockets_to_go = len (socks)
687 for i,sock in enumerate(socks,start=0):
689 old_rx.append(0); old_non_dp_rx.append(0); old_tx.append(0); old_non_dp_tx.append(0); old_drop.append(0); old_tx_fail.append(0); old_tsc.append(0)
690 old_rx[-1], old_non_dp_rx[-1], old_tx[-1], old_non_dp_tx[-1], old_drop[-1], old_tx_fail[-1], old_tsc[-1], tsc_hz = sock.core_stats(cores[i],tasks)
691 new_rx.append(0); new_non_dp_rx.append(0); new_tx.append(0); new_non_dp_tx.append(0); new_drop.append(0); new_tx_fail.append(0); new_tsc.append(0)
692 while (duration > 0):
694 # Get statistics after some execution time
695 for i,sock in enumerate(socks,start=0):
696 new_rx[i], new_non_dp_rx[i], new_tx[i], new_non_dp_tx[i], new_drop[i], new_tx_fail[i], new_tsc[i], tsc_hz = sock.core_stats(cores[i],tasks)
697 drop = new_drop[i]-old_drop[i]
698 rx = new_rx[i] - old_rx[i]
699 tx = new_tx[i] - old_tx[i]
700 non_dp_rx = new_non_dp_rx[i] - old_non_dp_rx[i]
701 non_dp_tx = new_non_dp_tx[i] - old_non_dp_tx[i]
702 tsc = new_tsc[i] - old_tsc[i]
706 old_drop[i] = new_drop[i]
707 old_rx[i] = new_rx[i]
708 old_tx[i] = new_tx[i]
709 old_non_dp_rx[i] = new_non_dp_rx[i]
710 old_non_dp_tx[i] = new_non_dp_tx[i]
711 old_tsc[i] = new_tsc[i]
712 tot_drop[i] = tot_drop[i] + tx - rx
713 log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(non_dp_rx)+' | '+'{:>10.0f}'.format(non_dp_tx)+' | ' + '{:>10.0f}'.format(tx-rx) + ' | '+ '{:>10.0f}'.format(non_dp_tx-non_dp_rx) + ' | '+'{:>10.0f}'.format(tot_drop[i]) +' |')
714 writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NonDPReceived':non_dp_rx,'NonDPSent':non_dp_tx,'Delta':tx-rx,'NonDPDelta':non_dp_tx-non_dp_rx,'Dropped':tot_drop[i]})
716 URL = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
717 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNonDPReceived {}\nNonDPSent {}\nDelta {}\nNonDPDelta {}\nDropped {}\n'.format(i,duration,rx,tx,non_dp_rx,non_dp_tx,tx-rx,non_dp_tx-non_dp_rx,tot_drop[i])
718 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
719 response = requests.post(url=URL, data=DATA,headers=HEADERS)
720 if (response.status_code != 202) and (response.status_code != 200):
721 log.info('Cannot send metrics to {}'.format(URL))
723 if sockets_to_go == 0:
724 duration = duration - 1
725 sockets_to_go = len (socks)
726 log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
728 def run_port_stats(socks):
729 fieldnames = ['PROXID','Time','Received','Sent','NoMbufs','iErrMiss']
730 writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
732 log.info("+---------------------------------------------------------------------------+")
733 log.info("| Measuring port statistics on 1 or more PROX instances |")
734 log.info("+-----------+-----------+------------+------------+------------+------------+")
735 log.info("| PROX ID | Time | RX | TX | no MBUFS | ierr&imiss |")
736 log.info("+-----------+-----------+------------+------------+------------+------------+")
739 duration = float(runtime)
740 old_rx = []; old_tx = []; old_no_mbufs = []; old_errors = []; old_tsc = []
741 new_rx = []; new_tx = []; new_no_mbufs = []; new_errors = []; new_tsc = []
742 sockets_to_go = len (socks)
743 for i,sock in enumerate(socks,start=0):
744 old_rx.append(0); old_tx.append(0); old_no_mbufs.append(0); old_errors.append(0); old_tsc.append(0)
745 old_rx[-1], old_tx[-1], old_no_mbufs[-1], old_errors[-1], old_tsc[-1] = sock.multi_port_stats(ports[i])
746 new_rx.append(0); new_tx.append(0); new_no_mbufs.append(0); new_errors.append(0); new_tsc.append(0)
747 while (duration > 0):
749 # Get statistics after some execution time
750 for i,sock in enumerate(socks,start=0):
751 new_rx[i], new_tx[i], new_no_mbufs[i], new_errors[i], new_tsc[i] = sock.multi_port_stats(ports[i])
752 rx = new_rx[i] - old_rx[i]
753 tx = new_tx[i] - old_tx[i]
754 no_mbufs = new_no_mbufs[i] - old_no_mbufs[i]
755 errors = new_errors[i] - old_errors[i]
756 tsc = new_tsc[i] - old_tsc[i]
760 old_rx[i] = new_rx[i]
761 old_tx[i] = new_tx[i]
762 old_no_mbufs[i] = new_no_mbufs[i]
763 old_errors[i] = new_errors[i]
764 old_tsc[i] = new_tsc[i]
765 log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(no_mbufs)+' | '+'{:>10.0f}'.format(errors)+' |')
766 writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NoMbufs':no_mbufs,'iErrMiss':errors})
768 URL = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
769 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNoMbufs {}\niErrMiss {}\n'.format(i,duration,rx,tx,no_mbufs,errors)
770 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
771 response = requests.post(url=URL, data=DATA,headers=HEADERS)
772 if (response.status_code != 202) and (response.status_code != 200):
773 log.info('Cannot send metrics to {}'.format(URL))
775 if sockets_to_go == 0:
776 duration = duration - 1
777 sockets_to_go = len (socks)
778 log.info("+-----------+-----------+------------+------------+------------+------------+")
780 def run_irqtest(socks):
781 log.info("+----------------------------------------------------------------------------------------------------------------------------")
782 log.info("| Measuring time probably spent dealing with an interrupt. Interrupting DPDK cores for more than 50us might be problematic ")
783 log.info("| and result in packet loss. The first row shows the interrupted time buckets: first number is the bucket between 0us and ")
784 log.info("| that number expressed in us and so on. The numbers in the other rows show how many times per second, the program was ")
785 log.info("| interrupted for a time as specified by its bucket. '0' is printed when there are no interrupts in this bucket throughout ")
786 log.info("| the duration of the test. 0.00 means there were interrupts in this bucket but very few. Due to rounding this shows as 0.00 ")
787 log.info("+----------------------------------------------------------------------------------------------------------------------------")
789 for sock_index,sock in enumerate(socks,start=0):
790 buckets=socks[sock_index].show_irq_buckets(1)
791 print('Measurement ongoing ... ',end='\r')
792 socks[sock_index].stop(cores[sock_index])
793 old_irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
794 irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
795 irq[0][0] = 'bucket us'
796 for j,bucket in enumerate(buckets,start=1):
797 irq[0][j] = '<'+ bucket
798 irq[0][-1] = '>'+ buckets [-2]
799 socks[sock_index].start(cores[sock_index])
801 for j,bucket in enumerate(buckets,start=1):
802 for i,irqcore in enumerate(cores[sock_index],start=1):
803 old_irq[i][j] = socks[sock_index].irq_stats(irqcore,j-1)
804 time.sleep(float(runtime))
805 socks[sock_index].stop(cores[sock_index])
806 for i,irqcore in enumerate(cores[sock_index],start=1):
807 irq[i][0]='core %s '%irqcore
808 for j,bucket in enumerate(buckets,start=1):
809 diff = socks[sock_index].irq_stats(irqcore,j-1) - old_irq[i][j]
813 irq[i][j] = str(round(diff/float(runtime), 2))
814 log.info('Results for PROX instance %s'%sock_index)
816 log.info(''.join(['{:>12}'.format(item) for item in row]))
818 def run_impairtest(gensock,sutsock):
819 fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Dropped','DropRate']
820 writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
823 log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
824 log.info("| Generator is sending UDP ("+'{:>5}'.format(FLOWSIZE)+" flow) packets ("+ '{:>5}'.format(size+4) +" bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test |")
825 log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
826 log.info("| Test | Speed requested | Sent to NIC | Sent by Gen | Forward by SUT | Rec. by Gen | Avg. Latency | Max. Latency | Packets Lost | Loss Ratio |")
827 log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
829 gensock.set_size(gencores,0,size) # This is setting the frame size
830 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
831 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
832 # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
833 source_port,destination_port = flows[FLOWSIZE]
834 gensock.set_random(gencores,0,34,source_port,2)
835 gensock.set_random(gencores,0,36,destination_port,2)
836 gensock.start(latcores)
838 gensock.speed(speed / len(gencores) / len(gentasks), gencores, gentasks)
841 print('Measurement ongoing at speed: ' + str(round(speed,2)) + '% ',end='\r')
844 # Get statistics now that the generation is stable and NO ARP messages any more
845 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg, lat_perc, lat_perc_max, lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used, r, actual_duration = run_iteration(gensock,sutsock,runtime)
846 drop_rate = 100.0*abs_dropped/abs_tx
848 lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) + bcolors.ENDC
851 log.info('|{:>7}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | '+ '{:>9.3f}'.format(pps_tx) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9.3f}'.format(pps_rx)+' Mpps | '+ '{:>9.0f}'.format(lat_avg)+' us | '+ '{:>9.0f}'.format(lat_max)+' us | '+ '{:>14d}'.format(abs_dropped)+ ' |''{:>9.2f}'.format(drop_rate)+ '% |'+lat_warning)
852 writer.writerow({'Flows':FLOWSIZE,'PacketSize':(size+4),'RequestedPPS':get_pps(speed,size),'GeneratedPPS':pps_req_tx,'SentPPS':pps_tx,'ForwardedPPS':pps_sut_tx_str,'ReceivedPPS':pps_rx,'AvgLatencyUSEC':lat_avg,'MaxLatencyUSEC':lat_max,'Dropped':abs_dropped,'DropRate':drop_rate})
854 URL = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
855 DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nDropped {}\nDropRate {}\n'.format(FLOWSIZE,size+4,get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,drop_rate)
856 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
857 response = requests.post(url=URL, data=DATA,headers=HEADERS)
858 if (response.status_code != 202) and (response.status_code != 200):
859 log.info('Cannot send metrics to {}'.format(URL))
862 def run_warmuptest(gensock):
863 # Running at low speed to make sure the ARP messages can get through.
864 # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
865 # Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
866 # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
867 gensock.speed(WARMUPSPEED / len(gencores) /len (gentasks), gencores, gentasks)
869 gensock.set_size(gencores,0,size) # This is setting the frame size
870 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
871 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
872 gensock.set_value(gencores,0,56,1,1)
873 # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
874 source_port,destination_port = flows[FLOWSIZE]
875 gensock.set_random(gencores,0,34,source_port,2)
876 gensock.set_random(gencores,0,36,destination_port,2)
877 gensock.start(genstatcores)
878 time.sleep(WARMUPTIME)
879 gensock.stop(genstatcores)
880 gensock.set_value(gencores,0,56,50,1)
881 time.sleep(WARMUPTIME)
884 log.debug ('exit cleanup')
885 for index, sock in enumerate(socks):
886 if socks_control[index]:
888 for client in clients:
893 def get_BinarySearchParams() :
894 global DROP_RATE_TRESHOLD
895 global LAT_AVG_TRESHOLD
896 global LAT_PERC_TRESHOLD
897 global LAT_MAX_TRESHOLD
903 DROP_RATE_TRESHOLD = float(testconfig.get('BinarySearchParams', 'drop_rate_threshold'))
904 LAT_AVG_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_avg_threshold'))
905 LAT_PERC_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_perc_threshold'))
906 LAT_MAX_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_max_threshold'))
907 ACCURACY = float(testconfig.get('BinarySearchParams', 'accuracy'))
908 STARTSPEED = float(testconfig.get('BinarySearchParams', 'startspeed'))
913 def get_FixedRateParams() :
914 global DROP_RATE_TRESHOLD
915 global LAT_AVG_TRESHOLD
916 global LAT_PERC_TRESHOLD
917 global LAT_MAX_TRESHOLD
918 global flow_size_list
919 global packet_size_list
924 DROP_RATE_TRESHOLD = inf
925 LAT_AVG_TRESHOLD = inf
926 LAT_PERC_TRESHOLD = inf
927 LAT_MAX_TRESHOLD = inf
931 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
932 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
933 STARTSPEED = float(testconfig.get('test%d'%test_nr, 'speed'))
935 def get_TST009SearchParams() :
936 global DROP_RATE_TRESHOLD
937 global LAT_AVG_TRESHOLD
938 global LAT_PERC_TRESHOLD
939 global LAT_MAX_TRESHOLD
943 global TST009_MAXFramesAllIngress
944 global TST009_StepSize
949 if testconfig.has_option('TST009SearchParams', 'drop_rate_threshold'):
950 DROP_RATE_TRESHOLD = float(testconfig.get('TST009SearchParams', 'drop_rate_threshold'))
952 DROP_RATE_TRESHOLD = 0
953 LAT_AVG_TRESHOLD = inf
954 LAT_PERC_TRESHOLD = inf
955 LAT_MAX_TRESHOLD = inf
956 TST009_MAXr = float(testconfig.get('TST009SearchParams', 'MAXr'))
957 TST009_MAXz = float(testconfig.get('TST009SearchParams', 'MAXz'))
958 TST009_MAXFramesAllIngress = int(testconfig.get('TST009SearchParams', 'MAXFramesPerSecondAllIngress'))
959 TST009_StepSize = int(testconfig.get('TST009SearchParams', 'StepSize'))
960 TST009_n = int(ceil(TST009_MAXFramesAllIngress / TST009_StepSize))
963 TST009_R = TST009_n - 1
964 for m in range(0, TST009_n):
965 TST009_S.append((m+1) * TST009_StepSize)
966 # To generate a desired number of flows, PROX will randomize the bits in source and destination ports, as specified by the bit masks in the flows variable.
968 1: ['1000000000000000','1000000000000000'],\
969 2: ['1000000000000000','100000000000000X'],\
970 4: ['100000000000000X','100000000000000X'],\
971 8: ['100000000000000X','10000000000000XX'],\
972 16: ['10000000000000XX','10000000000000XX'],\
973 32: ['10000000000000XX','1000000000000XXX'],\
974 64: ['1000000000000XXX','1000000000000XXX'],\
975 128: ['1000000000000XXX','100000000000XXXX'],\
976 256: ['100000000000XXXX','100000000000XXXX'],\
977 512: ['100000000000XXXX','10000000000XXXXX'],\
978 1024: ['10000000000XXXXX','10000000000XXXXX'],\
979 2048: ['10000000000XXXXX','1000000000XXXXXX'],\
980 4096: ['1000000000XXXXXX','1000000000XXXXXX'],\
981 8192: ['1000000000XXXXXX','100000000XXXXXXX'],\
982 16384: ['100000000XXXXXXX','100000000XXXXXXX'],\
983 32768: ['100000000XXXXXXX','10000000XXXXXXXX'],\
984 65536: ['10000000XXXXXXXX','10000000XXXXXXXX'],\
985 131072: ['10000000XXXXXXXX','1000000XXXXXXXXX'],\
986 262144: ['1000000XXXXXXXXX','1000000XXXXXXXXX'],\
987 524288: ['1000000XXXXXXXXX','100000XXXXXXXXXX'],\
988 1048576:['100000XXXXXXXXXX','100000XXXXXXXXXX'],}
1009 data_file = 'RUN{}.{}.csv'.format(env,test_file)
1010 data_csv_file = open(data_file,'w')
1011 testconfig = ConfigParser.RawConfigParser()
1012 testconfig.read(test_file)
1013 required_number_of_test_machines = testconfig.get('DEFAULT', 'total_number_of_test_machines')
1014 TestName = testconfig.get('DEFAULT', 'name')
1015 if testconfig.has_option('DEFAULT', 'PushGateway'):
1016 PushGateway = testconfig.get('DEFAULT', 'PushGateway')
1017 log.info('Measurements will be pushed to %s'%PushGateway)
1020 if testconfig.has_option('DEFAULT', 'lat_percentile'):
1021 LAT_PERCENTILE = float(testconfig.get('DEFAULT', 'lat_percentile')) /100.0
1023 LAT_PERCENTILE = 0.99
1024 log.info('Latency percentile measured at {:.0f}%'.format(LAT_PERCENTILE*100))
1025 config = ConfigParser.RawConfigParser()
1027 machine_map = ConfigParser.RawConfigParser()
1028 machine_map.read(machine_map_file)
1029 vim_type = config.get('Varia', 'vim')
1030 key = config.get('ssh', 'key')
1031 user = config.get('ssh', 'user')
1032 total_number_of_machines = config.get('rapid', 'total_number_of_machines')
1033 if int(required_number_of_test_machines) > int(total_number_of_machines):
1034 log.exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
1035 raise Exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
1036 for vm in range(1, int(total_number_of_machines)+1):
1037 vmAdminIP.append(config.get('M%d'%vm, 'admin_ip'))
1038 vmDPmac.append(config.get('M%d'%vm, 'dp_mac'))
1039 vmDPIP.append(config.get('M%d'%vm, 'dp_ip'))
1040 ip = vmDPIP[-1].split('.')
1041 hexDPIP.append(hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2))
1042 if (vim_type == "kubernetes"):
1043 vmDPPCIDEV.append(config.get('M%d'%vm, 'dp_pci_dev'))
1045 atexit.register(exit_handler)
1046 for vm in range(1, int(required_number_of_test_machines)+1):
1047 machine_index.append(int(machine_map.get('TestM%d'%vm, 'machine_index'))-1)
1048 prox_socket.append(testconfig.getboolean('TestM%d'%vm, 'prox_socket'))
1049 for vm in range(1, int(required_number_of_test_machines)+1):
1050 if prox_socket[vm-1]:
1051 prox_launch_exit.append(testconfig.getboolean('TestM%d'%vm, 'prox_launch_exit'))
1052 config_file.append(testconfig.get('TestM%d'%vm, 'config_file'))
1053 # Looking for all task definitions in the PROX cfg files. Constructing a list of all tasks used
1054 textfile = open (config_file[-1], 'r')
1055 filetext = textfile.read()
1057 tasks_for_this_cfg = set(re.findall("task\s*=\s*(\d+)",filetext))
1058 with open('{}_{}_parameters{}.lua'.format(env,test_file,vm), "w") as f:
1059 f.write('name="%s"\n'% testconfig.get('TestM%d'%vm, 'name'))
1060 f.write('local_ip="%s"\n'% vmDPIP[machine_index[vm-1]])
1061 f.write('local_hex_ip="%s"\n'% hexDPIP[machine_index[vm-1]])
1062 if (vim_type == "kubernetes"):
1063 f.write("eal=\"--socket-mem=512,0 --file-prefix %s-%s-%s --pci-whitelist %s\"\n" % (env, test_file, vm, vmDPPCIDEV[machine_index[vm-1]]))
1065 f.write("eal=\"\"\n")
1066 if testconfig.has_option('TestM%d'%vm, 'cores'):
1067 cores.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'cores')))
1068 f.write('cores="%s"\n'% ','.join(map(str, cores[-1])))
1071 if testconfig.has_option('TestM%d'%vm, 'ports'):
1072 ports.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'ports')))
1073 f.write('ports="%s"\n'% ','.join(map(str, ports[-1])))
1076 if testconfig.has_option('TestM%d'%vm, 'monitor'):
1077 monitor.append(testconfig.getboolean('TestM%d'%vm, 'monitor'))
1079 monitor.append(True)
1080 if testconfig.has_option('TestM%d'%vm, 'gencores'):
1081 # This must be a generator VM
1082 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
1083 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
1084 genstatcores = gencores + latcores
1085 gentasks = tasks_for_this_cfg
1086 mach_type.append('gen')
1087 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
1088 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
1089 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
1090 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
1091 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
1092 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
1093 if testconfig.has_option('TestM%d'%vm, 'gw_vm'):
1094 gwVMindex = int(testconfig.get('TestM%d'%vm, 'gw_vm')) -1
1095 f.write('gw_ip="%s"\n'% vmDPIP[machine_index[gwVMindex]])
1096 f.write('gw_hex_ip="%s"\n'% hexDPIP[machine_index[gwVMindex]])
1097 if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'):
1098 BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp'))
1100 BUCKET_SIZE_EXP = 11
1101 f.write('bucket_size_exp="%s"\n'% BUCKET_SIZE_EXP)
1102 if testconfig.has_option('TestM%d'%vm, 'heartbeat'):
1103 heartbeat = int(testconfig.get('TestM%d'%vm, 'heartbeat'))
1106 f.write('heartbeat="%s"\n'% heartbeat)
1107 elif testconfig.has_option('TestM%d'%vm, 'dest_vm'):
1108 # This must be a machine acting as a GW that we will not monitor
1109 mach_type.append('none')
1110 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
1111 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
1112 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
1113 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
1114 elif re.findall('mode\s*=\s*(swap|esp_dec)',filetext,re.MULTILINE):
1115 sutstatcores = cores[-1]
1116 mach_type.append('sut')
1118 mach_type.append('none')
1120 tasks = tasks_for_this_cfg.union(tasks)
1121 log.debug("Tasks detected in all PROX config files %r"%tasks)
1122 #####################################################################################
1123 for vm in range(0, int(required_number_of_test_machines)):
1125 clients.append(prox_ctrl(vmAdminIP[machine_index[vm]], key,user))
1126 connect_client(clients[-1])
1127 if (vim_type == "OpenStack"):
1128 # Creating script to bind the right network interface to the poll mode driver
1129 devbindfile = '{}_{}_devbindvm{}.sh'.format(env,test_file, vm+1)
1130 with open(devbindfile, "w") as f:
1131 newText= 'link="$(ip -o link | grep '+vmDPmac[machine_index[vm]]+' |cut -d":" -f 2)"\n'
1133 newText= 'if [ -n "$link" ];\n'
1137 newText= ' echo Need to bind\n'
1139 newText= ' sudo ' + rundir + '/dpdk/usertools/dpdk-devbind.py --force --bind igb_uio $('+rundir+'/dpdk/usertools/dpdk-devbind.py --status |grep $link | cut -d" " -f 1)\n'
1143 newText= ' echo Assuming port is already bound to DPDK\n'
1149 st = os.stat(devbindfile)
1150 os.chmod(devbindfile, st.st_mode | stat.S_IEXEC)
1151 clients[-1].scp_put('./%s'%devbindfile, rundir+'/devbind.sh')
1152 cmd = 'sudo ' + rundir+ '/devbind.sh'
1153 clients[-1].run_cmd(cmd)
1154 log.debug("devbind.sh running on VM%d"%(vm+1))
1156 clients[-1].scp_put('./%s'%config_file[vm], rundir+'/%s'%config_file[vm])
1157 clients[-1].scp_put('./{}_{}_parameters{}.lua'.format(env,test_file, vm+1), rundir + '/parameters.lua')
1159 if prox_launch_exit[vm]:
1160 if mach_type[vm] == 'gen':
1161 cmd = 'sudo ' + rundir + '/prox -e -t -o cli -f ' + rundir + '/%s'%config_file[vm]
1163 cmd = 'sudo ' + rundir + '/prox -t -o cli -f ' + rundir + '/%s'%config_file[vm]
1164 log.debug("Starting PROX on VM{}: {}".format((vm+1),cmd))
1165 clients[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1))
1166 socks_control.append(prox_launch_exit[vm])
1167 socks.append(connect_socket(clients[-1]))
1168 sock_type.append(mach_type[vm])
1169 sock_monitor.append(monitor[vm])
1170 monitor_gen = monitor_sut = False
1171 background_gen_socks =[]
1172 for index, sock in enumerate(socks, start=0):
1173 if sock_type[index] == 'gen':
1174 if sock_monitor[index]:
1176 log.exception("Can only monitor 1 generator")
1177 raise Exception("Can only monitor 1 generator")
1180 gensock_index = index
1182 background_gen_socks.append(sock)
1183 elif sock_type[index] == 'sut' and sock_monitor[index]:
1185 log.exception("Can only monitor 1 sut")
1186 raise Exception("Can only monitor 1 sut")
1189 sutsock_index = index
1192 ####################################################
1194 # Best to run the flow test at the end since otherwise the tests coming after might be influenced by the big number of entries in the switch flow tables
1195 ####################################################
1196 number_of_tests = testconfig.get('DEFAULT', 'number_of_tests')
1197 for test_nr in range(1, int(number_of_tests)+1):
1198 test=testconfig.get('test%d'%test_nr,'test')
1200 if test == 'flowsizetest':
1201 get_BinarySearchParams()
1202 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1203 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1204 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1205 elif test == 'TST009test':
1206 # This test implements some of the testing as defined in https://docbox.etsi.org/ISG/NFV/open/Publications_pdf/Specs-Reports/NFV-TST%20009v3.2.1%20-%20GS%20-%20NFVI_Benchmarks.pdf
1207 get_TST009SearchParams()
1208 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1209 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1210 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1211 elif test == 'fixed_rate':
1212 get_FixedRateParams()
1213 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1214 elif test == 'corestats':
1215 run_core_stats(socks)
1216 elif test == 'portstats':
1217 run_port_stats(socks)
1218 elif test == 'impairtest':
1219 get_BinarySearchParams()
1220 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1221 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1222 run_impairtest(socks[gensock_index],socks[sutsock_index])
1223 elif test == 'irqtest':
1225 elif test == 'warmuptest':
1226 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1227 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1228 WARMUPSPEED = int(testconfig.get('test%d'%test_nr, 'warmupspeed'))
1229 WARMUPTIME = int(testconfig.get('test%d'%test_nr, 'warmuptime'))
1230 run_warmuptest(socks[gensock_index])
1231 ####################################################