4 ## Copyright (c) 2010-2019 Intel Corporation
6 ## Licensed under the Apache License, Version 2.0 (the "License");
7 ## you may not use this file except in compliance with the License.
8 ## You may obtain a copy of the License at
10 ## http://www.apache.org/licenses/LICENSE-2.0
12 ## Unless required by applicable law or agreed to in writing, software
13 ## distributed under the License is distributed on an "AS IS" BASIS,
14 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 ## See the License for the specific language governing permissions and
16 ## limitations under the License.
19 from __future__ import print_function
29 from logging.handlers import RotatingFileHandler
30 from logging import handlers
31 from prox_ctrl import prox_ctrl
39 env = "rapid.env" #Default string for environment
40 test_file = "basicrapid.test" #Default string for test
41 machine_map_file = "machine.map" #Default string for machine map file
42 loglevel="DEBUG" # sets log level for writing to file
43 screenloglevel="INFO" # sets log level for writing to screen
44 runtime=10 # time in seconds for 1 test run
45 configonly = False # IF True, the system will upload all the necessary config fiels to the VMs, but not start PROX and the actual testing
46 rundir = "/home/centos" # Directory where to find the tools in the machines running PROX
49 print("usage: runrapid [--version] [-v]")
50 print(" [--env ENVIRONMENT_NAME]")
51 print(" [--test TEST_NAME]")
52 print(" [--map MACHINE_MAP_FILE]")
53 print(" [--runtime TIME_FOR_TEST]")
54 print(" [--configonly False|True]")
55 print(" [--log DEBUG|INFO|WARNING|ERROR|CRITICAL]")
56 print(" [-h] [--help]")
58 print("Command-line interface to runrapid")
60 print("optional arguments:")
61 print(" -v, --version Show program's version number and exit")
62 print(" --env ENVIRONMENT_NAME Parameters will be read from ENVIRONMENT_NAME. Default is %s."%env)
63 print(" --test TEST_NAME Test cases will be read from TEST_NAME. Default is %s."%test_file)
64 print(" --map MACHINE_MAP_FILE Machine mapping will be read from MACHINE_MAP_FILE. Default is %s."%machine_map_file)
65 print(" --runtime Specify time in seconds for 1 test run")
66 print(" --configonly If this option is specified, only upload all config files to the VMs, do not run the tests")
67 print(" --log Specify logging level for log file output, default is DEBUG")
68 print(" --screenlog Specify logging level for screen output, default is INFO")
69 print(" -h, --help Show help message and exit.")
73 opts, args = getopt.getopt(sys.argv[1:], "vh", ["version","help", "env=", "test=", "map=", "runtime=","configonly","log=","screenlog="])
74 except getopt.GetoptError as err:
75 print("===========================================")
77 print("===========================================")
84 if opt in ["-h", "--help"]:
87 if opt in ["-v", "--version"]:
88 print("Rapid Automated Performance Indication for Dataplane "+version)
95 machine_map_file = arg
96 if opt in ["--runtime"]:
98 if opt in ["--configonly"]:
100 print('No actual runs, only uploading configuration files')
103 print ("Log level: "+ loglevel)
104 if opt in ["--screenlog"]:
106 print ("Screen Log level: "+ screenloglevel)
108 print ("Using '"+env+"' as name for the environment")
109 print ("Using '"+test_file+"' for test case definition")
110 print ("Using '"+machine_map_file+"' for machine mapping")
111 print ("Runtime: "+ str(runtime))
121 UNDERLINE = '\033[4m'
124 screen_formatter = logging.Formatter("%(message)s")
125 file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
127 # get a top-level logger,
129 # BUT PREVENT IT from propagating messages to the root logger
131 log = logging.getLogger()
132 numeric_level = getattr(logging, loglevel.upper(), None)
133 if not isinstance(numeric_level, int):
134 raise ValueError('Invalid log level: %s' % loglevel)
135 log.setLevel(numeric_level)
138 # create a console handler
139 # and set its log level to the command-line option
141 console_handler = logging.StreamHandler(sys.stdout)
142 #console_handler.setLevel(logging.INFO)
143 numeric_screenlevel = getattr(logging, screenloglevel.upper(), None)
144 if not isinstance(numeric_screenlevel, int):
145 raise ValueError('Invalid screenlog level: %s' % screenloglevel)
146 console_handler.setLevel(numeric_screenlevel)
147 console_handler.setFormatter(screen_formatter)
149 # create a file handler
150 # and set its log level
152 log_file = 'RUN{}.{}.log'.format(env,test_file)
153 file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10)
154 #file_handler = log.handlers.TimedRotatingFileHandler(log_file, 'D', 1, 5)
155 file_handler.setLevel(numeric_level)
156 file_handler.setFormatter(file_formatter)
158 # add handlers to the logger
160 log.addHandler(file_handler)
161 log.addHandler(console_handler)
163 # Check if log exists and should therefore be rolled
164 needRoll = os.path.isfile(log_file)
167 # This is a stale log, so roll it
170 log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime())
172 # Roll over on application start
173 log.handlers[0].doRollover()
176 log.debug('\n---------\nLog started on %s.\n---------\n' % time.asctime())
178 log.debug("runrapid.py version: "+version)
179 #========================================================================
180 def connect_socket(client):
182 log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
185 sock = client.prox_sock()
190 log.exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
191 raise Exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
193 log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
194 log.info("Connected to PROX on %s" % client.ip())
197 def connect_client(client):
199 log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
204 except RuntimeWarning, ex:
207 log.exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
208 raise Exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
210 log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
211 log.debug("Connected to VM on %s" % client.ip())
213 def run_iteration(gensock,sutsock):
215 # Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
216 time.sleep(sleep_time)
217 abs_old_rx, abs_old_non_dp_rx, abs_old_tx, abs_old_non_dp_tx, abs_old_drop, abs_old_tx_fail, abs_old_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
218 abs_old_rx = abs_old_rx - abs_old_non_dp_rx
219 abs_old_tx = abs_old_tx - abs_old_non_dp_tx
220 gensock.start(gencores)
221 time.sleep(sleep_time)
223 old_sut_rx, old_sut_non_dp_rx, old_sut_tx, old_sut_non_dp_tx, old_sut_drop, old_sut_tx_fail, old_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
224 old_sut_rx = old_sut_rx - old_sut_non_dp_rx
225 old_sut_tx = old_sut_tx - old_sut_non_dp_tx
226 old_rx, old_non_dp_rx, old_tx, old_non_dp_tx, old_drop, old_tx_fail, old_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
227 old_rx = old_rx - old_non_dp_rx
228 old_tx = old_tx - old_non_dp_tx
229 # Measure latency statistics per second
235 while n_loops < float(runtime):
238 lat_min_sample, lat_max_sample, lat_avg_sample, used_sample = gensock.lat_stats(latcores)
239 if lat_min > lat_min_sample:
240 lat_min = lat_min_sample
241 if lat_max < lat_max_sample:
242 lat_max = lat_max_sample
243 lat_avg = lat_avg + lat_avg_sample
244 used_avg = used_avg + used_sample
245 lat_avg = lat_avg / n_loops
246 used_avg = used_avg / n_loops
247 # Get statistics after some execution time
248 new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
249 new_rx = new_rx - new_non_dp_rx
250 new_tx = new_tx - new_non_dp_tx
252 new_sut_rx, new_sut_non_dp_rx, new_sut_tx, new_sut_non_dp_tx, new_sut_drop, new_sut_tx_fail, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
253 new_sut_rx = new_sut_rx - new_sut_non_dp_rx
254 new_sut_tx = new_sut_tx - new_sut_non_dp_tx
256 gensock.stop(gencores)
257 time.sleep(sleep_time)
258 abs_new_rx, abs_new_non_dp_rx, abs_new_tx, abs_new_non_dp_tx, abs_new_drop, abs_new_tx_fail, abs_new_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
259 abs_new_rx = abs_new_rx - abs_new_non_dp_rx
260 abs_new_tx = abs_new_tx - abs_new_non_dp_tx
261 drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
262 rx = new_rx - old_rx # rx is all packets received by the nop task = all packets received in the gen VM
263 tx = new_tx - old_tx # tx is all generated packets actually accepted by the interface
264 abs_dropped = (abs_new_tx - abs_old_tx) - (abs_new_rx - abs_old_rx)
265 tsc = new_tsc - old_tsc # time difference between the 2 measurements, expressed in cycles.
266 pps_req_tx = (tx+drop-rx)*tsc_hz*1.0/(tsc*1000000)
267 pps_tx = tx*tsc_hz*1.0/(tsc*1000000)
268 pps_rx = rx*tsc_hz*1.0/(tsc*1000000)
270 sut_rx = new_sut_rx - old_sut_rx
271 sut_tx = new_sut_tx - old_sut_tx
272 sut_tsc = new_sut_tsc - old_sut_tsc
273 pps_sut_tx = sut_tx*sut_tsc_hz*1.0/(sut_tsc*1000000)
274 pps_sut_tx_str = '{:>9.3f}'.format(pps_sut_tx)
277 pps_sut_tx_str = 'NO MEAS.'
279 log.critical("TX = 0. Test interrupted since no packet has been sent.")
280 raise Exception("TX = 0")
281 return(pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,(abs_new_tx_fail - abs_old_tx_fail),(abs_new_tx - abs_old_tx),lat_min,used_avg)
283 def new_speed(speed,minspeed,maxspeed,success):
288 newspeed = (maxspeed+minspeed)/2.0
289 return (newspeed,minspeed,maxspeed)
291 def get_pps(speed,size):
292 # speed is given in % of 10Gb/s, returning Mpps
293 return (speed * 100.0 / (8*(size+24)))
295 def get_speed(packet_speed,size):
296 # return speed in Gb/s
297 return (packet_speed / 1000.0 * (8*(size+24)))
300 def run_flow_size_test(gensock,sutsock):
301 fieldnames = ['Flows','PacketSize','Gbps','Mpps','AvgLatency','MaxLatency','PacketsDropped','PacketDropRate']
302 writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
304 gensock.start(latcores)
305 for size in packet_size_list:
307 gensock.set_size(gencores,0,size) # This is setting the frame size
308 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
309 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
310 # This will only work when using sending UDP packets. For different protocls and ehternet types, we would need a different calculation
311 log.info("+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
312 log.info("| UDP, "+ '{:>5}'.format(size+4) +" bytes, different number of flows by randomizing SRC & DST UDP port |")
313 log.info("+--------+--------------------+----------------+----------------+----------------+------------------------+----------------+----------------+----------------+------------+")
314 log.info("| Flows | Speed requested | core generated | Sent by Gen NIC| Forward by SUT | core received | Avg. Latency | Max. Latency | Packets Lost | Loss Ratio |")
315 log.info("+--------+--------------------+----------------+----------------+----------------+------------------------+----------------+----------------+----------------+------------+")
316 for flow_number in flow_size_list:
318 gensock.reset_stats()
320 sutsock.reset_stats()
321 source_port,destination_port = flows[flow_number]
322 gensock.set_random(gencores,0,34,source_port,2)
323 gensock.set_random(gencores,0,36,destination_port,2)
324 endpps_sut_tx_str = 'NO_RESULTS'
325 maxspeed = speed = STARTSPEED
327 while (maxspeed-minspeed > ACCURACY):
330 print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '% ',end='\r')
332 # Start generating packets at requested speed (in % of a 10Gb/s link)
333 gensock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
335 # Get statistics now that the generation is stable and initial ARP messages are dealt with
336 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used = run_iteration(gensock,sutsock)
337 drop_rate = 100.0*abs_dropped/abs_tx
339 lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) + bcolors.ENDC
342 # The following if statement is testing if we pass the success criteria of a certain drop rate, average latenecy and maximum latency below the threshold
343 # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
344 # This can be specified by putting 0 in the .test file
345 if ((drop_rate < DROP_RATE_TRESHOLD) or (abs_dropped==DROP_RATE_TRESHOLD ==0)) and (lat_avg< LAT_AVG_TRESHOLD) and (lat_max < LAT_MAX_TRESHOLD):
346 lat_avg_prefix = bcolors.ENDC
347 lat_max_prefix = bcolors.ENDC
348 abs_drop_rate_prefix = bcolors.ENDC
349 drop_rate_prefix = bcolors.ENDC
350 if ((get_pps(speed,size) - pps_tx)/get_pps(speed,size))>0.01:
351 speed_prefix = bcolors.WARNING
353 gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
355 gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(get_pps(speed,size), pps_tx) + bcolors.ENDC
357 speed_prefix = bcolors.ENDC
360 endspeed_prefix = speed_prefix
361 endpps_req_tx = pps_req_tx
363 endpps_sut_tx_str = pps_sut_tx_str
367 endabs_dropped = abs_dropped
368 enddrop_rate = drop_rate
369 if lat_warning or gen_warning:
370 endwarning = '| | {:167.167} |'.format(lat_warning + gen_warning)
372 success_message='% | SUCCESS'
374 success_message='% | FAILED'
376 abs_drop_rate_prefix = bcolors.ENDC
377 if ((abs_dropped>0) and (DROP_RATE_TRESHOLD ==0)):
378 abs_drop_rate_prefix = bcolors.FAIL
379 if (drop_rate < DROP_RATE_TRESHOLD):
380 drop_rate_prefix = bcolors.ENDC
382 drop_rate_prefix = bcolors.FAIL
383 if (lat_avg< LAT_AVG_TRESHOLD):
384 lat_avg_prefix = bcolors.ENDC
386 lat_avg_prefix = bcolors.FAIL
387 if (lat_max< LAT_MAX_TRESHOLD):
388 lat_max_prefix = bcolors.ENDC
390 lat_max_prefix = bcolors.FAIL
391 if (((get_pps(speed,size) - pps_tx)/get_pps(speed,size))<0.001):
392 speed_prefix = bcolors.ENDC
394 speed_prefix = bcolors.FAIL
396 log.debug('|step{:>3}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% '+speed_prefix +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | ' + '{:>9.3f}'.format(pps_tx) +' Mpps | '+ bcolors.ENDC + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+bcolors.OKBLUE + '{:>4.1f}'.format(get_speed(pps_rx,size)) + 'Gb/s{:>9.3f}'.format(pps_rx)+' Mpps'+bcolors.ENDC+' | '+lat_avg_prefix+ '{:>9.0f}'.format(lat_avg)+' us | '+lat_max_prefix+ '{:>9.0f}'.format(lat_max)+' us | '+ abs_drop_rate_prefix + '{:>14d}'.format(abs_dropped)+drop_rate_prefix+ ' |''{:>9.2f}'.format(drop_rate)+bcolors.ENDC+ success_message +lat_warning + gen_warning)
397 speed,minspeed,maxspeed = new_speed(speed,minspeed,maxspeed,success)
398 if endpps_sut_tx_str != 'NO_RESULTS':
399 log.info('|{:>7}'.format(str(flow_number))+" | " + '{:>5.1f}'.format(endspeed) + '% ' + endspeed_prefix + '{:>6.3f}'.format(get_pps(endspeed,size)) + ' Mpps | '+ '{:>9.3f}'.format(endpps_req_tx)+ ' Mpps | '+ '{:>9.3f}'.format(endpps_tx) + ' Mpps | ' + bcolors.ENDC + '{:>9}'.format(endpps_sut_tx_str) +' Mpps | '+bcolors.OKBLUE + '{:>4.1f}'.format(get_speed(endpps_rx,size)) + 'Gb/s{:>9.3f}'.format(endpps_rx)+' Mpps'+bcolors.ENDC+' | '+ '{:>9.0f}'.format(endlat_avg)+' us | '+ '{:>9.0f}'.format(endlat_max)+' us | '+ '{:>14d}'.format(endabs_dropped)+ ' |'+'{:>9.2f}'.format(enddrop_rate)+ '% |')
401 log.info (endwarning)
402 log.info("+--------+--------------------+----------------+----------------+----------------+------------------------+----------------+----------------+----------------+------------+")
403 writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'Gbps':get_speed(endpps_rx,size),'Mpps':endpps_rx,'AvgLatency':endlat_avg,'MaxLatency':endlat_max,'PacketsDropped':endabs_dropped,'PacketDropRate':enddrop_rate})
405 URL = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
406 DATA = 'Flows {}\nPacketSize {}\nGbps {}\nMpps {}\nAvgLatency {}\nMaxLatency {}\nPacketsDropped {}\nPacketDropRate {}\n'.format(flow_number,size+4,get_speed(endpps_rx,size),endpps_rx,endlat_avg,endlat_max,endabs_dropped,enddrop_rate)
407 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
408 response = requests.post(url=URL, data=DATA,headers=HEADERS)
410 log.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
411 gensock.stop(latcores)
414 def run_fixed_rate(gensock,sutsock):
415 fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Sent','Received','Lost','LostTotal']
416 writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
418 gensock.start(latcores)
420 for size in packet_size_list:
422 gensock.set_size(gencores,0,size) # This is setting the frame size
423 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
424 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
425 # This will only work when using sending UDP packets. For different protocols and ehternet types, we would need a different calculation
426 log.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------+")
427 log.info("| UDP, "+ '{:>5}'.format(size+4) +" bytes, different number of flows by randomizing SRC & DST UDP port |")
428 log.info("+--------+------------------+-------------+-------------+-------------+-------------+-------------+-------------+-----------+-----------+---------+------------+")
429 log.info("| Flows | Speed requested | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core| Avg. Latency| Max. Latency| Sent | Received | Lost | Total Lost |")
430 log.info("+--------+------------------+-------------+-------------+-------------+-------------+-------------+-------------+-----------+-----------+---------+------------+")
431 for flow_number in flow_size_list:
432 time.sleep(sleep_time)
433 gensock.reset_stats()
435 sutsock.reset_stats()
436 source_port,destination_port = flows[flow_number]
437 gensock.set_random(gencores,0,34,source_port,2)
438 gensock.set_random(gencores,0,36,destination_port,2)
439 endpps_sut_tx_str = 'NO_RESULTS'
441 # Start generating packets at requested speed (in % of a 10Gb/s link)
442 gensock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
443 duration = float(runtime)
447 old_sut_rx, old_sut_non_dp_rx, old_sut_tx, old_sut_non_dp_tx, old_sut_drop, old_sut_tx_fail, old_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
448 old_sut_rx = old_sut_rx - old_sut_non_dp_rx
449 old_sut_tx = old_sut_tx - old_sut_non_dp_tx
450 old_rx, old_non_dp_rx, old_tx, old_non_dp_tx, old_drop, old_tx_fail, old_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
451 old_rx = old_rx - old_non_dp_rx
452 old_tx = old_tx - old_non_dp_tx
453 gensock.start(gencores)
454 while (duration > 0):
456 lat_min, lat_max, lat_avg, lat_used = gensock.lat_stats(latcores)
458 lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) + bcolors.ENDC
461 # Get statistics after some execution time
462 new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
463 new_rx = new_rx - new_non_dp_rx
464 new_tx = new_tx - new_non_dp_tx
466 new_sut_rx, new_sut_non_dp_rx, new_sut_tx, new_sut_non_dp_tx, new_sut_drop, new_sut_tx_fail, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
467 new_sut_rx = new_sut_rx - new_sut_non_dp_rx
468 new_sut_tx = new_sut_tx - new_sut_non_dp_tx
469 drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
470 rx = new_rx - old_rx # rx is all packets received by the nop task = all packets received in the gen VM
471 tx = new_tx - old_tx # tx is all generated packets actually accepted by the interface
472 tsc = new_tsc - old_tsc # time difference between the 2 measurements, expressed in cycles.
476 sut_rx = new_sut_rx - old_sut_rx
477 sut_tx = new_sut_tx - old_sut_tx
478 sut_tsc = new_sut_tsc - old_sut_tsc
481 duration = duration - 1
486 pps_req_tx = (tx+drop-rx)*tsc_hz*1.0/(tsc*1000000)
487 pps_tx = tx*tsc_hz*1.0/(tsc*1000000)
488 pps_rx = rx*tsc_hz*1.0/(tsc*1000000)
490 old_sut_tx = new_sut_tx
491 old_sut_rx = new_sut_rx
492 old_sut_tsc= new_sut_tsc
493 pps_sut_tx = sut_tx*sut_tsc_hz*1.0/(sut_tsc*1000000)
494 pps_sut_tx_str = '{:>7.3f}'.format(pps_sut_tx)
497 pps_sut_tx_str = 'NO MEAS.'
499 log.critical("TX = 0. Test interrupted since no packet has been sent.")
500 raise Exception("TX = 0")
501 tot_drop = tot_drop + tx - rx
503 if pps_sut_tx_str != 'NO_RESULTS':
504 # First second mpps are not valid as there is no alignement between time the generator is started and per seconds stats
506 log.info('|{:>7}'.format(flow_number)+" |" + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+' |' +' |' +' |'+ ' |'+ '{:>8.0f}'.format(lat_avg)+' us |'+'{:>8.0f}'.format(lat_max)+' us | ' + '{:>9.0f}'.format(tx) + ' | '+ '{:>9.0f}'.format(rx) + ' | '+ '{:>7.0f}'.format(tx-rx) + ' | '+'{:>7.0f}'.format(tot_drop) +' |'+lat_warning)
508 log.info('|{:>7}'.format(flow_number)+" |" + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+ '{:>7.3f}'.format(pps_req_tx)+' Mpps |'+ '{:>7.3f}'.format(pps_tx) +' Mpps |' + '{:>7}'.format(pps_sut_tx_str) +' Mpps |'+ '{:>7.3f}'.format(pps_rx)+' Mpps |'+ '{:>8.0f}'.format(lat_avg)+' us |'+'{:>8.0f}'.format(lat_max)+' us | ' + '{:>9.0f}'.format(tx) + ' | '+ '{:>9.0f}'.format(rx) + ' | '+ '{:>7.0f}'.format(tx-rx) + ' | '+ '{:>7.0f}'.format(tot_drop) +' |'+lat_warning)
509 writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':get_pps(speed,size),'GeneratedPPS':pps_req_tx,'SentPPS':pps_tx,'ForwardedPPS':pps_sut_tx,'ReceivedPPS':pps_rx,'AvgLatencyUSEC':lat_avg,'MaxLatencyUSEC':lat_max,'Sent':tx,'Received':rx,'Lost':(tx-rx),'LostTotal':tot_drop})
511 URL = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
512 DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nSent {}\nReceived {}\nLost {}\nLostTotal {}\n'.format(flow_number,size+4,get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_max,tx,rx,(tx-rx),tot_drop)
513 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
514 response = requests.post(url=URL, data=DATA,headers=HEADERS)
516 log.debug('|{:>7} | Speed 0 or close to 0'.format(str(size)))
520 gensock.stop(gencores)
521 time.sleep(sleep_time)
522 lat_min, lat_max, lat_avg, lat_used = gensock.lat_stats(latcores)
524 lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) + bcolors.ENDC
527 # Get statistics after some execution time
528 new_rx, new_non_dp_rx, new_tx, new_non_dp_tx, new_drop, new_tx_fail, new_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
529 new_rx = new_rx - new_non_dp_rx
530 new_tx = new_tx - new_non_dp_tx
532 new_sut_rx, new_sut_non_dp_rx, new_sut_tx, new_sut_non_dp_tx, new_sut_drop, new_sut_tx_fail, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
533 new_sut_rx = new_sut_rx - new_sut_non_dp_rx
534 new_sut_tx = new_sut_tx - new_sut_non_dp_tx
535 drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
536 rx = new_rx - old_rx # rx is all packets received by the nop task = all packets received in the gen VM
537 tx = new_tx - old_tx # tx is all generated packets actually accepted by the interface
538 tsc = new_tsc - old_tsc # time difference between the 2 measurements, expressed in cycles.
539 tot_drop = tot_drop + tx - rx
541 sut_rx = new_sut_rx - old_sut_rx
542 sut_tx = new_sut_tx - old_sut_tx
543 sut_tsc = new_sut_tsc - old_sut_tsc
544 if pps_sut_tx_str != 'NO_RESULTS':
545 log.info('|{:>7}'.format(flow_number)+" |" + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+' |' +' |' +' |'+ ' |'+ '{:>8.0f}'.format(lat_avg)+' us |'+'{:>8.0f}'.format(lat_max)+' us | ' + '{:>9.0f}'.format(tx) + ' | '+ '{:>9.0f}'.format(rx) + ' | '+ '{:>7.0f}'.format(tx-rx) + ' | '+ '{:>7.0f}'.format(tot_drop) +' |'+lat_warning)
546 log.info("+--------+------------------+-------------+-------------+-------------+-------------+-------------+-------------+-----------+-----------+---------+------------+")
547 gensock.stop(latcores)
549 def run_core_stats(socks):
550 fieldnames = ['PROXID','Time','Received','Sent','NonDPReceived','NonDPSent','Delta','NonDPDelta','Dropped']
551 writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
553 log.info("+------------------------------------------------------------------------------------------------------------------+")
554 log.info("| Measuring core statistics on 1 or more PROX instances |")
555 log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
556 log.info("| PROX ID | Time | RX | TX | non DP RX | non DP TX | TX - RX | nonDP TX-RX| DROP TOT |")
557 log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
560 duration = float(runtime)
562 old_rx = []; old_non_dp_rx = []; old_tx = []; old_non_dp_tx = []; old_drop = []; old_tx_fail = []; old_tsc = []
563 new_rx = []; new_non_dp_rx = []; new_tx = []; new_non_dp_tx = []; new_drop = []; new_tx_fail = []; new_tsc = []
564 sockets_to_go = len (socks)
565 for i,sock in enumerate(socks,start=0):
567 old_rx.append(0); old_non_dp_rx.append(0); old_tx.append(0); old_non_dp_tx.append(0); old_drop.append(0); old_tx_fail.append(0); old_tsc.append(0)
568 old_rx[-1], old_non_dp_rx[-1], old_tx[-1], old_non_dp_tx[-1], old_drop[-1], old_tx_fail[-1], old_tsc[-1], tsc_hz = sock.core_stats(cores[i],tasks)
569 new_rx.append(0); new_non_dp_rx.append(0); new_tx.append(0); new_non_dp_tx.append(0); new_drop.append(0); new_tx_fail.append(0); new_tsc.append(0)
570 while (duration > 0):
572 # Get statistics after some execution time
573 for i,sock in enumerate(socks,start=0):
574 new_rx[i], new_non_dp_rx[i], new_tx[i], new_non_dp_tx[i], new_drop[i], new_tx_fail[i], new_tsc[i], tsc_hz = sock.core_stats(cores[i],tasks)
575 drop = new_drop[i]-old_drop[i]
576 rx = new_rx[i] - old_rx[i]
577 tx = new_tx[i] - old_tx[i]
578 non_dp_rx = new_non_dp_rx[i] - old_non_dp_rx[i]
579 non_dp_tx = new_non_dp_tx[i] - old_non_dp_tx[i]
580 tsc = new_tsc[i] - old_tsc[i]
584 old_drop[i] = new_drop[i]
585 old_rx[i] = new_rx[i]
586 old_tx[i] = new_tx[i]
587 old_non_dp_rx[i] = new_non_dp_rx[i]
588 old_non_dp_tx[i] = new_non_dp_tx[i]
589 old_tsc[i] = new_tsc[i]
590 tot_drop[i] = tot_drop[i] + tx - rx
591 log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(non_dp_rx)+' | '+'{:>10.0f}'.format(non_dp_tx)+' | ' + '{:>10.0f}'.format(tx-rx) + ' | '+ '{:>10.0f}'.format(non_dp_tx-non_dp_rx) + ' | '+'{:>10.0f}'.format(tot_drop[i]) +' |')
592 writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NonDPReceived':non_dp_rx,'NonDPSent':non_dp_tx,'Delta':tx-rx,'NonDPDelta':non_dp_tx-non_dp_rx,'Dropped':tot_drop[i]})
594 URL = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
595 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNonDPReceived {}\nNonDPSent {}\nDelta {}\nNonDPDelta {}\nDropped {}\n'.format(i,duration,rx,tx,non_dp_rx,non_dp_tx,tx-rx,non_dp_tx-non_dp_rx,tot_drop[i])
596 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
597 response = requests.post(url=URL, data=DATA,headers=HEADERS)
598 if sockets_to_go == 0:
599 duration = duration - 1
600 sockets_to_go = len (socks)
601 log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
603 def run_port_stats(socks):
604 fieldnames = ['PROXID','Time','Received','Sent','NoMbufs','iErrMiss']
605 writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
607 log.info("+---------------------------------------------------------------------------+")
608 log.info("| Measuring port statistics on 1 or more PROX instances |")
609 log.info("+-----------+-----------+------------+------------+------------+------------+")
610 log.info("| PROX ID | Time | RX | TX | no MBUFS | ierr&imiss |")
611 log.info("+-----------+-----------+------------+------------+------------+------------+")
614 duration = float(runtime)
615 old_rx = []; old_tx = []; old_no_mbufs = []; old_errors = []; old_tsc = []
616 new_rx = []; new_tx = []; new_no_mbufs = []; new_errors = []; new_tsc = []
617 sockets_to_go = len (socks)
618 for i,sock in enumerate(socks,start=0):
619 old_rx.append(0); old_tx.append(0); old_no_mbufs.append(0); old_errors.append(0); old_tsc.append(0)
620 old_rx[-1], old_tx[-1], old_no_mbufs[-1], old_errors[-1], old_tsc[-1] = sock.multi_port_stats(ports[i])
621 new_rx.append(0); new_tx.append(0); new_no_mbufs.append(0); new_errors.append(0); new_tsc.append(0)
622 while (duration > 0):
624 # Get statistics after some execution time
625 for i,sock in enumerate(socks,start=0):
626 new_rx[i], new_tx[i], new_no_mbufs[i], new_errors[i], new_tsc[i] = sock.multi_port_stats(ports[i])
627 rx = new_rx[i] - old_rx[i]
628 tx = new_tx[i] - old_tx[i]
629 no_mbufs = new_no_mbufs[i] - old_no_mbufs[i]
630 errors = new_errors[i] - old_errors[i]
631 tsc = new_tsc[i] - old_tsc[i]
635 old_rx[i] = new_rx[i]
636 old_tx[i] = new_tx[i]
637 old_no_mbufs[i] = new_no_mbufs[i]
638 old_errors[i] = new_errors[i]
639 old_tsc[i] = new_tsc[i]
640 log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(no_mbufs)+' | '+'{:>10.0f}'.format(errors)+' |')
641 writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NoMbufs':no_mbufs,'iErrMiss':errors})
643 URL = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
644 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNoMbufs {}\niErrMiss {}\n'.format(i,duration,rx,tx,no_mbufs,errors)
645 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
646 response = requests.post(url=URL, data=DATA,headers=HEADERS)
647 if sockets_to_go == 0:
648 duration = duration - 1
649 sockets_to_go = len (socks)
650 log.info("+-----------+-----------+------------+------------+------------+------------+")
652 def run_irqtest(socks):
653 log.info("+----------------------------------------------------------------------------------------------------------------------------")
654 log.info("| Measuring time probably spent dealing with an interrupt. Interrupting DPDK cores for more than 50us might be problematic ")
655 log.info("| and result in packet loss. The first row shows the interrupted time buckets: first number is the bucket between 0us and ")
656 log.info("| that number expressed in us and so on. The numbers in the other rows show how many times per second, the program was ")
657 log.info("| interrupted for a time as specified by its bucket. '0' is printed when there are no interrupts in this bucket throughout ")
658 log.info("| the duration of the test. 0.00 means there were interrupts in this bucket but very few. Due to rounding this shows as 0.00 ")
659 log.info("+----------------------------------------------------------------------------------------------------------------------------")
661 for sock_index,sock in enumerate(socks,start=0):
662 buckets=socks[sock_index].show_irq_buckets(1)
663 print('Measurement ongoing ... ',end='\r')
664 socks[sock_index].stop(cores[sock_index])
665 old_irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
666 irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
667 irq[0][0] = 'bucket us'
668 for j,bucket in enumerate(buckets,start=1):
669 irq[0][j] = '<'+ bucket
670 irq[0][-1] = '>'+ buckets [-2]
671 socks[sock_index].start(cores[sock_index])
673 for j,bucket in enumerate(buckets,start=1):
674 for i,irqcore in enumerate(cores[sock_index],start=1):
675 old_irq[i][j] = socks[sock_index].irq_stats(irqcore,j-1)
676 time.sleep(float(runtime))
677 socks[sock_index].stop(cores[sock_index])
678 for i,irqcore in enumerate(cores[sock_index],start=1):
679 irq[i][0]='core %s '%irqcore
680 for j,bucket in enumerate(buckets,start=1):
681 diff = socks[sock_index].irq_stats(irqcore,j-1) - old_irq[i][j]
685 irq[i][j] = str(round(diff/float(runtime), 2))
686 log.info('Results for PROX instance %s'%sock_index)
688 log.info(''.join(['{:>12}'.format(item) for item in row]))
690 def run_impairtest(gensock,sutsock):
691 fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Dropped','DropRate']
692 writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
695 log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
696 log.info("| Generator is sending UDP ("+'{:>5}'.format(FLOWSIZE)+" flow) packets ("+ '{:>5}'.format(size+4) +" bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test |")
697 log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
698 log.info("| Test | Speed requested | Sent to NIC | Sent by Gen | Forward by SUT | Rec. by Gen | Avg. Latency | Max. Latency | Packets Lost | Loss Ratio |")
699 log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
701 gensock.set_size(gencores,0,size) # This is setting the frame size
702 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
703 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
704 # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
705 source_port,destination_port = flows[FLOWSIZE]
706 gensock.set_random(gencores,0,34,source_port,2)
707 gensock.set_random(gencores,0,36,destination_port,2)
708 gensock.start(latcores)
710 gensock.speed(speed / len(gencores) / len(gentasks), gencores, gentasks)
713 print('Measurement ongoing at speed: ' + str(round(speed,2)) + '% ',end='\r')
716 # Get statistics now that the generation is stable and NO ARP messages any more
717 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used = run_iteration(gensock,sutsock)
718 drop_rate = 100.0*abs_dropped/abs_tx
720 lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) + bcolors.ENDC
723 log.info('|{:>7}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | '+ '{:>9.3f}'.format(pps_tx) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9.3f}'.format(pps_rx)+' Mpps | '+ '{:>9.0f}'.format(lat_avg)+' us | '+ '{:>9.0f}'.format(lat_max)+' us | '+ '{:>14d}'.format(abs_dropped)+ ' |''{:>9.2f}'.format(drop_rate)+ '% |'+lat_warning)
724 writer.writerow({'Flows':FLOWSIZE,'PacketSize':(size+4),'RequestedPPS':get_pps(speed,size),'GeneratedPPS':pps_req_tx,'SentPPS':pps_tx,'ForwardedPPS':pps_sut_tx_str,'ReceivedPPS':pps_rx,'AvgLatencyUSEC':lat_avg,'MaxLatencyUSEC':lat_max,'Dropped':abs_dropped,'DropRate':drop_rate})
726 URL = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
727 DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nDropped {}\nDropRate {}\n'.format(FLOWSIZE,size+4,get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,drop_rate)
728 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
729 response = requests.post(url=URL, data=DATA,headers=HEADERS)
731 def run_warmuptest(gensock):
732 # Running at low speed to make sure the ARP messages can get through.
733 # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
734 # Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
735 # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
736 gensock.speed(WARMUPSPEED / len(gencores) /len (gentasks), gencores, gentasks)
738 gensock.set_size(gencores,0,size) # This is setting the frame size
739 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
740 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
741 gensock.set_value(gencores,0,56,1,1)
742 # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
743 source_port,destination_port = flows[FLOWSIZE]
744 gensock.set_random(gencores,0,34,source_port,2)
745 gensock.set_random(gencores,0,36,destination_port,2)
746 gensock.start(genstatcores)
747 time.sleep(WARMUPTIME)
748 gensock.stop(genstatcores)
749 gensock.set_value(gencores,0,56,50,1)
757 global packet_size_list
759 global flow_size_list
762 global required_number_of_test_machines
763 # To generate a desired number of flows, PROX will randomize the bits in source and destination ports, as specified by the bit masks in the flows variable.
765 1: ['1000000000000000','1000000000000000'],\
766 2: ['1000000000000000','100000000000000X'],\
767 4: ['100000000000000X','100000000000000X'],\
768 8: ['100000000000000X','10000000000000XX'],\
769 16: ['10000000000000XX','10000000000000XX'],\
770 32: ['10000000000000XX','1000000000000XXX'],\
771 64: ['1000000000000XXX','1000000000000XXX'],\
772 128: ['1000000000000XXX','100000000000XXXX'],\
773 256: ['100000000000XXXX','100000000000XXXX'],\
774 512: ['100000000000XXXX','10000000000XXXXX'],\
775 1024: ['10000000000XXXXX','10000000000XXXXX'],\
776 2048: ['10000000000XXXXX','1000000000XXXXXX'],\
777 4096: ['1000000000XXXXXX','1000000000XXXXXX'],\
778 8192: ['1000000000XXXXXX','100000000XXXXXXX'],\
779 16384: ['100000000XXXXXXX','100000000XXXXXXX'],\
780 32768: ['100000000XXXXXXX','10000000XXXXXXXX'],\
781 65536: ['10000000XXXXXXXX','10000000XXXXXXXX'],\
782 131072: ['10000000XXXXXXXX','1000000XXXXXXXXX'],\
783 262144: ['1000000XXXXXXXXX','1000000XXXXXXXXX'],\
784 524288: ['1000000XXXXXXXXX','100000XXXXXXXXXX'],\
785 1048576:['100000XXXXXXXXXX','100000XXXXXXXXXX'],}
803 data_file = 'RUN{}.{}.csv'.format(env,test_file)
804 data_csv_file = open(data_file,'w')
805 testconfig = ConfigParser.RawConfigParser()
806 testconfig.read(test_file)
807 required_number_of_test_machines = testconfig.get('DEFAULT', 'total_number_of_test_machines')
808 TestName = testconfig.get('DEFAULT', 'name')
809 if testconfig.has_option('DEFAULT', 'PushGateway'):
810 PushGateway = testconfig.get('DEFAULT', 'PushGateway')
811 log.info('Measurements will be pushed to %s'%PushGateway)
814 config = ConfigParser.RawConfigParser()
816 machine_map = ConfigParser.RawConfigParser()
817 machine_map.read(machine_map_file)
818 key = config.get('ssh', 'key')
819 user = config.get('ssh', 'user')
820 total_number_of_machines = config.get('rapid', 'total_number_of_machines')
821 if int(required_number_of_test_machines) > int(total_number_of_machines):
822 log.exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
823 raise Exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
824 for vm in range(1, int(total_number_of_machines)+1):
825 vmAdminIP.append(config.get('M%d'%vm, 'admin_ip'))
826 vmDPmac.append(config.get('M%d'%vm, 'dp_mac'))
827 vmDPIP.append(config.get('M%d'%vm, 'dp_ip'))
828 ip = vmDPIP[-1].split('.')
829 hexDPIP.append(hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2))
831 for vm in range(1, int(required_number_of_test_machines)+1):
832 machine_index.append(int(machine_map.get('TestM%d'%vm, 'machine_index'))-1)
833 prox_socket.append(testconfig.getboolean('TestM%d'%vm, 'prox_socket'))
834 for vm in range(1, int(required_number_of_test_machines)+1):
835 if prox_socket[vm-1]:
836 prox_launch_exit.append(testconfig.getboolean('TestM%d'%vm, 'prox_launch_exit'))
837 config_file.append(testconfig.get('TestM%d'%vm, 'config_file'))
838 # Looking for all task definitions in the PROX cfg files. Constructing a list of all tasks used
839 textfile = open (config_file[-1], 'r')
840 filetext = textfile.read()
842 tasks_for_this_cfg = set(re.findall("task\s*=\s*(\d+)",filetext))
843 with open('{}_{}_parameters{}.lua'.format(env,test_file,vm), "w") as f:
844 f.write('name="%s"\n'% testconfig.get('TestM%d'%vm, 'name'))
845 f.write('local_ip="%s"\n'% vmDPIP[machine_index[vm-1]])
846 f.write('local_hex_ip="%s"\n'% hexDPIP[machine_index[vm-1]])
847 if testconfig.has_option('TestM%d'%vm, 'cores'):
848 cores.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'cores')))
849 f.write('cores="%s"\n'% ','.join(map(str, cores[-1])))
852 if testconfig.has_option('TestM%d'%vm, 'ports'):
853 ports.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'ports')))
854 f.write('ports="%s"\n'% ','.join(map(str, ports[-1])))
857 if re.match('(l2){0,1}gen(_bare){0,1}.*\.cfg',config_file[-1]):
858 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
859 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
860 genstatcores = gencores + latcores
861 gentasks = tasks_for_this_cfg
862 auto_start.append(False)
863 mach_type.append('gen')
864 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
865 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
866 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
867 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
868 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
869 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
870 elif re.match('(l2){0,1}gen_gw.*\.cfg',config_file[-1]):
871 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
872 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
873 genstatcores = gencores + latcores
874 gentasks = tasks_for_this_cfg
875 auto_start.append(False)
876 mach_type.append('gen')
877 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
878 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
879 gwVMindex = int(testconfig.get('TestM%d'%vm, 'gw_vm')) -1
880 f.write('gw_ip="%s"\n'% vmDPIP[machine_index[gwVMindex]])
881 f.write('gw_hex_ip="%s"\n'% hexDPIP[machine_index[gwVMindex]])
882 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
883 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
884 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
885 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
886 elif re.match('(l2){0,1}swap.*\.cfg',config_file[-1]):
887 sutstatcores = cores[-1]
888 auto_start.append(True)
889 mach_type.append('sut')
890 elif re.match('secgw1.*\.cfg',config_file[-1]):
891 auto_start.append(True)
892 mach_type.append('none')
893 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
894 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
895 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
896 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
897 elif re.match('secgw2.*\.cfg',config_file[-1]):
898 sutstatcores = cores[-1]
899 auto_start.append(True)
900 mach_type.append('sut')
902 auto_start.append(True)
903 mach_type.append('none')
905 tasks = tasks_for_this_cfg.union(tasks)
906 log.debug("Tasks detected in all PROX config files %r"%tasks)
907 #####################################################################################
909 log.debug ('exit cleanup')
910 for index, sock in enumerate(socks):
911 if socks_control[index]:
913 for client in clients:
918 atexit.register(exit_handler)
920 for vm in range(0, int(required_number_of_test_machines)):
922 clients.append(prox_ctrl(vmAdminIP[machine_index[vm]], key,user))
923 connect_client(clients[-1])
924 # Creating script to bind the right network interface to the poll mode driver
925 devbindfile = '{}_{}_devbindvm{}.sh'.format(env,test_file, vm+1)
926 with open(devbindfile, "w") as f:
927 newText= 'link="$(ip -o link | grep '+vmDPmac[machine_index[vm]]+' |cut -d":" -f 2)"\n'
929 newText= 'if [ -n "$link" ];\n'
933 newText= ' echo Need to bind\n'
935 newText= ' sudo ' + rundir + '/dpdk/usertools/dpdk-devbind.py --force --bind igb_uio $('+rundir+'/dpdk/usertools/dpdk-devbind.py --status |grep $link | cut -d" " -f 1)\n'
939 newText= ' echo Assuming port is already bound to DPDK\n'
945 st = os.stat(devbindfile)
946 os.chmod(devbindfile, st.st_mode | stat.S_IEXEC)
947 clients[-1].scp_put('./%s'%devbindfile, rundir+'/devbind.sh')
948 cmd = 'sudo ' + rundir+ '/devbind.sh'
949 clients[-1].run_cmd(cmd)
950 log.debug("devbind.sh running on VM%d"%(vm+1))
951 clients[-1].scp_put('./%s'%config_file[vm], rundir+'/%s'%config_file[vm])
952 clients[-1].scp_put('./{}_{}_parameters{}.lua'.format(env,test_file, vm+1), rundir + '/parameters.lua')
954 if prox_launch_exit[vm]:
955 log.debug("Starting PROX on VM%d"%(vm+1))
957 cmd = 'sudo ' +rundir + '/prox/build/prox -t -o cli -f ' + rundir + '/%s'%config_file[vm]
959 cmd = 'sudo ' +rundir + '/prox/build/prox -e -t -o cli -f ' + rundir + '/%s'%config_file[vm]
960 clients[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1))
961 socks_control.append(prox_launch_exit[vm])
962 socks.append(connect_socket(clients[-1]))
963 sock_type.append(mach_type[vm])
965 def get_BinarySearchParams() :
966 global DROP_RATE_TRESHOLD
967 global LAT_AVG_TRESHOLD
968 global LAT_MAX_TRESHOLD
971 DROP_RATE_TRESHOLD = float(testconfig.get('BinarySearchParams', 'drop_rate_threshold'))
972 LAT_AVG_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_avg_threshold'))
973 LAT_MAX_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_max_threshold'))
974 ACCURACY = float(testconfig.get('BinarySearchParams', 'accuracy'))
975 STARTSPEED = float(testconfig.get('BinarySearchParams', 'startspeed'))
979 ####################################################
981 # Best to run the flow test at the end since otherwise the tests coming after might be influenced by the big number of entries in the switch flow tables
982 ####################################################
983 gensock_index = sock_type.index('gen') if 'gen' in sock_type else -1
984 sutsock_index = sock_type.index('sut') if 'sut' in sock_type else -1
985 number_of_tests = testconfig.get('DEFAULT', 'number_of_tests')
986 for test_nr in range(1, int(number_of_tests)+1):
987 test=testconfig.get('test%d'%test_nr,'test')
989 if test == 'flowsizetest':
990 get_BinarySearchParams()
991 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
992 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
993 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
994 elif test == 'fixed_rate':
995 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
996 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
997 STARTSPEED = float(testconfig.get('test%d'%test_nr, 'speed'))
998 run_fixed_rate(socks[gensock_index],socks[sutsock_index])
999 elif test == 'corestats':
1000 run_core_stats(socks)
1001 elif test == 'portstats':
1002 run_port_stats(socks)
1003 elif test == 'impairtest':
1004 get_BinarySearchParams()
1005 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1006 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1007 run_impairtest(socks[gensock_index],socks[sutsock_index])
1008 elif test == 'irqtest':
1010 elif test == 'warmuptest':
1011 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1012 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1013 WARMUPSPEED = int(testconfig.get('test%d'%test_nr, 'warmupspeed'))
1014 WARMUPTIME = int(testconfig.get('test%d'%test_nr, 'warmuptime'))
1015 run_warmuptest(socks[gensock_index])
1016 ####################################################