5520d139b7d5f2b7fb55b27f9eddea54ed3007cf
[samplevnf.git] / VNFs / DPPD-PROX / helper-scripts / rapid / runrapid.py
1 #!/usr/bin/python
2
3 ##
4 ## Copyright (c) 2010-2020 Intel Corporation
5 ##
6 ## Licensed under the Apache License, Version 2.0 (the "License");
7 ## you may not use this file except in compliance with the License.
8 ## You may obtain a copy of the License at
9 ##
10 ##     http://www.apache.org/licenses/LICENSE-2.0
11 ##
12 ## Unless required by applicable law or agreed to in writing, software
13 ## distributed under the License is distributed on an "AS IS" BASIS,
14 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 ## See the License for the specific language governing permissions and
16 ## limitations under the License.
17 ##
18
19 from __future__ import print_function
20
21 import os
22 import stat
23 import sys
24 import time
25 import subprocess
26 import getopt
27 import re
28 import logging
29 from logging.handlers import RotatingFileHandler
30 from logging import handlers
31 from prox_ctrl import prox_ctrl
32 import ConfigParser
33 import ast
34 import atexit
35 import csv
36 import requests
37 from math import ceil
38 inf = float("inf")
39
40 version="20.03.23"
41 env = "rapid.env" #Default string for environment
42 test_file = "basicrapid.test" #Default string for test
43 machine_map_file = "machine.map" #Default string for machine map file
44 loglevel="DEBUG" # sets log level for writing to file
45 screenloglevel="INFO" # sets log level for writing to screen
46 runtime=10 # time in seconds for 1 test run
47 configonly = False # IF True, the system will upload all the necessary config fiels to the VMs, but not start PROX and the actual testing
48 #rundir = "/home/centos" # Directory where to find the tools in the machines running PROX
49 rundir = "~" # Directory where to find the tools in the machines running PROX
50
51 def usage():
52         print("usage: runrapid    [--version] [-v]")
53         print("                   [--env ENVIRONMENT_NAME]")
54         print("                   [--test TEST_NAME]")
55         print("                   [--map MACHINE_MAP_FILE]")
56         print("                   [--runtime TIME_FOR_TEST]")
57         print("                   [--configonly False|True]")
58         print("                   [--log DEBUG|INFO|WARNING|ERROR|CRITICAL]")
59         print("                   [-h] [--help]")
60         print("")
61         print("Command-line interface to runrapid")
62         print("")
63         print("optional arguments:")
64         print("  -v,  --version                 Show program's version number and exit")
65         print("  --env ENVIRONMENT_NAME         Parameters will be read from ENVIRONMENT_NAME. Default is %s."%env)
66         print("  --test TEST_NAME               Test cases will be read from TEST_NAME. Default is %s."%test_file)
67         print("  --map MACHINE_MAP_FILE Machine mapping will be read from MACHINE_MAP_FILE. Default is %s."%machine_map_file)
68         print("  --runtime                      Specify time in seconds for 1 test run")
69         print("  --configonly                   If this option is specified, only upload all config files to the VMs, do not run the tests")
70         print("  --log                          Specify logging level for log file output, default is DEBUG")
71         print("  --screenlog                    Specify logging level for screen output, default is INFO")
72         print("  -h, --help                     Show help message and exit.")
73         print("")
74
75 try:
76         opts, args = getopt.getopt(sys.argv[1:], "vh", ["version","help", "env=", "test=", "map=", "runtime=","configonly","log=","screenlog="])
77 except getopt.GetoptError as err:
78         print("===========================================")
79         print(str(err))
80         print("===========================================")
81         usage()
82         sys.exit(2)
83 if args:
84         usage()
85         sys.exit(2)
86 for opt, arg in opts:
87         if opt in ["-h", "--help"]:
88                 usage()
89                 sys.exit()
90         if opt in ["-v", "--version"]:
91                 print("Rapid Automated Performance Indication for Dataplane "+version)
92                 sys.exit()
93         if opt in ["--env"]:
94                 env = arg
95         if opt in ["--test"]:
96                 test_file = arg
97         if opt in ["--map"]:
98                 machine_map_file = arg
99         if opt in ["--runtime"]:
100                 runtime = arg
101         if opt in ["--configonly"]:
102                 configonly = True
103                 print('No actual runs, only uploading configuration files')
104         if opt in ["--log"]:
105                 loglevel = arg
106                 print ("Log level: "+ loglevel)
107         if opt in ["--screenlog"]:
108                 screenloglevel = arg
109                 print ("Screen Log level: "+ screenloglevel)
110
111 print ("Using '"+env+"' as name for the environment")
112 print ("Using '"+test_file+"' for test case definition")
113 print ("Using '"+machine_map_file+"' for machine mapping")
114 print ("Runtime: "+ str(runtime))
115
116 class bcolors:
117         HEADER = '\033[95m'
118         OKBLUE = '\033[94m'
119         OKGREEN = '\033[92m'
120         WARNING = '\033[93m'
121         FAIL = '\033[91m'
122         ENDC = '\033[0m'
123         BOLD = '\033[1m'
124         UNDERLINE = '\033[4m'
125         FLASH = '\033[5m'
126
127 # create formatters
128 screen_formatter = logging.Formatter("%(message)s")
129 file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
130
131 # get a top-level logger,
132 # set its log level,
133 # BUT PREVENT IT from propagating messages to the root logger
134 #
135 log = logging.getLogger()
136 numeric_level = getattr(logging, loglevel.upper(), None)
137 if not isinstance(numeric_level, int):
138         raise ValueError('Invalid log level: %s' % loglevel)
139 log.setLevel(numeric_level)
140 log.propagate = 0
141
142 # create a console handler
143 # and set its log level to the command-line option 
144
145 console_handler = logging.StreamHandler(sys.stdout)
146 #console_handler.setLevel(logging.INFO)
147 numeric_screenlevel = getattr(logging, screenloglevel.upper(), None)
148 if not isinstance(numeric_screenlevel, int):
149         raise ValueError('Invalid screenlog level: %s' % screenloglevel)
150 console_handler.setLevel(numeric_screenlevel)
151 console_handler.setFormatter(screen_formatter)
152
153 # create a file handler
154 # and set its log level
155 #
156 log_file = 'RUN{}.{}.log'.format(env,test_file)
157 file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10)
158 #file_handler = log.handlers.TimedRotatingFileHandler(log_file, 'D', 1, 5)
159 file_handler.setLevel(numeric_level)
160 file_handler.setFormatter(file_formatter)
161
162 # add handlers to the logger
163 #
164 log.addHandler(file_handler)
165 log.addHandler(console_handler)
166
167 # Check if log exists and should therefore be rolled
168 needRoll = os.path.isfile(log_file)
169
170
171 # This is a stale log, so roll it
172 if needRoll:    
173         # Add timestamp
174         log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime())
175
176         # Roll over on application start
177         log.handlers[0].doRollover()
178
179 # Add timestamp
180 log.debug('\n---------\nLog started on %s.\n---------\n' % time.asctime())
181
182 log.debug("runrapid.py version: "+version)
183 #========================================================================
184 def connect_socket(client):
185         attempts = 1
186         log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
187         sock = None
188         while True:
189                 sock = client.prox_sock()
190                 if sock is not None:
191                         break
192                 attempts += 1
193                 if attempts > 20:
194                         log.exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
195                         raise Exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
196                 time.sleep(2)
197                 log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
198         log.info("Connected to PROX on %s" % client.ip())
199         return sock
200
201 def connect_client(client):
202         attempts = 1
203         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
204         while True:
205                 try:
206                         client.connect()
207                         break
208                 except RuntimeWarning, ex:
209                         attempts += 1
210                         if attempts > 20:
211                                 log.exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
212                                 raise Exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
213                         time.sleep(2)
214                         log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
215         log.debug("Connected to VM on %s" % client.ip())
216
217 def report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,tx,rx,tot_drop,elapsed_time,speed_prefix='',lat_avg_prefix='',lat_perc_prefix='',lat_max_prefix='',abs_drop_rate_prefix='',drop_rate_prefix=''):
218         if flow_number < 0:
219                 flow_number_str = '| ({:>4}) |'.format(abs(flow_number))
220         else:
221                 flow_number_str = '|{:>7} |'.format(flow_number)
222         if pps_req_tx is None:
223                 pps_req_tx_str = '{0: >14}'.format('   NA     |')
224         else:
225                 pps_req_tx_str = '{:>7.3f} Mpps |'.format(pps_req_tx)
226         if pps_tx is None:
227                 pps_tx_str = '{0: >14}'.format('   NA     |')
228         else:
229                 pps_tx_str = '{:>7.3f} Mpps |'.format(pps_tx) 
230         if pps_sut_tx is None:
231                 pps_sut_tx_str = '{0: >14}'.format('   NA     |')
232         else:
233                 pps_sut_tx_str = '{:>7.3f} Mpps |'.format(pps_sut_tx)
234         if pps_rx is None:
235                 pps_rx_str = '{0: >25}'.format('NA        |')
236         else:
237                 pps_rx_str = bcolors.OKBLUE + '{:>4.1f} Gb/s |{:7.3f} Mpps {}|'.format(get_speed(pps_rx,size),pps_rx,bcolors.ENDC)
238         if tot_drop is None:
239                 tot_drop_str = ' |       NA  | '
240         else:
241                 tot_drop_str = ' | {:>9.0f} | '.format(tot_drop)
242         if lat_perc is None:
243                 lat_perc_str = ' |{:^10.10}|'.format('NA')
244         elif lat_perc_max == True:
245                 lat_perc_str = '|>{}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC) 
246         else:
247                 lat_perc_str = '| {}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC) 
248         if elapsed_time is None:
249                 elapsed_time_str = ' NA |'
250         else:
251                 elapsed_time_str = '{:>3.0f} |'.format(elapsed_time)
252         return(flow_number_str + '{:>5.1f}'.format(speed) + '% '+speed_prefix +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+ pps_req_tx_str + pps_tx_str + bcolors.ENDC + pps_sut_tx_str + pps_rx_str +lat_avg_prefix+ ' {:>6.0f}'.format(lat_avg)+' us'+lat_perc_str+lat_max_prefix+'{:>6.0f}'.format(lat_max)+' us | ' + '{:>9.0f}'.format(tx) + ' | {:>9.0f}'.format(rx) + ' | '+ abs_drop_rate_prefix+ '{:>9.0f}'.format(tx-rx) + tot_drop_str +drop_rate_prefix+ '{:>5.2f}'.format(float(tx-rx)/tx)  +bcolors.ENDC+' |' + elapsed_time_str)
253                 
254 def run_iteration(gensock, sutsock, requested_duration,flow_number,size,speed):
255         r = 0;
256         sleep_time = 2
257         while (r < TST009_MAXr):
258                 time.sleep(sleep_time)
259                 # Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
260                 t1_rx, t1_non_dp_rx, t1_tx, t1_non_dp_tx, t1_drop, t1_tx_fail, t1_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
261                 t1_dp_rx = t1_rx - t1_non_dp_rx
262                 t1_dp_tx = t1_tx - t1_non_dp_tx
263                 gensock.start(gencores)
264                 time.sleep(2) ## Needs to be 2 seconds since this 1 sec is the time that PROX uses to refresh the stats. Note that this can be changed in PROX!! Don't do it.
265                 if sutsock!='none':
266                         t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
267                 t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
268                 tx = t2_tx - t1_tx
269                 dp_tx =  tx - (t2_non_dp_tx - t1_non_dp_tx )
270                 dp_rx =  t2_rx - t1_rx - (t2_non_dp_rx - t1_non_dp_rx) 
271                 tot_dp_drop = dp_tx - dp_rx
272                 if tx == 0:
273                         log.critical("TX = 0. Test interrupted since no packet has been sent.")
274                         raise Exception("TX = 0")
275                 if dp_tx == 0:
276                         log.critical("Only non-dataplane packets (e.g. ARP) sent. Test interrupted since no packet has been sent.")
277                         raise Exception("Only non-dataplane packets (e.g. ARP) sent")
278                 # Ask PROX to calibrate the bucket size once we have a PROX function to do this.
279                 # Measure latency statistics per second
280                 lat_min, lat_max, lat_avg, used_avg, t2_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
281                 lat_samples = sum(buckets)
282                 sample_count = 0
283                 for sample_percentile, bucket in enumerate(buckets,start=1):
284                         sample_count += bucket
285                         if sample_count > (lat_samples * LAT_PERCENTILE):
286                                 break
287                 percentile_max = (sample_percentile == len(buckets))
288                 sample_percentile = sample_percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
289                 if test == 'fixed_rate':
290                         log.info(report_result(flow_number,size,speed,None,None,None,None,lat_avg,sample_percentile,percentile_max,lat_max, dp_tx, dp_rx , None, None))
291                 tot_rx = tot_non_dp_rx = tot_tx = tot_non_dp_tx = tot_drop = 0
292                 lat_avg = used_avg = 0
293                 buckets_total = [0] * 128
294                 tot_lat_samples = 0
295                 tot_lat_measurement_duration = float(0)
296                 tot_core_measurement_duration = float(0)
297                 tot_sut_core_measurement_duration = float(0)
298                 tot_sut_rx = tot_sut_non_dp_rx = tot_sut_tx = tot_sut_non_dp_tx = tot_sut_drop = tot_sut_tx_fail = tot_sut_tsc = 0
299                 lat_avail = core_avail = sut_avail = False
300                 while (tot_core_measurement_duration - float(requested_duration) <= 0.1) or (tot_lat_measurement_duration - float(requested_duration) <= 0.1):
301                         time.sleep(0.5)
302                         lat_min_sample, lat_max_sample, lat_avg_sample, used_sample, t3_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
303                         # Get statistics after some execution time
304                         if t3_lat_tsc != t2_lat_tsc:
305                                 single_lat_measurement_duration = (t3_lat_tsc - t2_lat_tsc) * 1.0 / lat_hz  # time difference between the 2 measurements, expressed in seconds.
306                                 # A second has passed in between to lat_stats requests. Hence we need to process the results
307                                 tot_lat_measurement_duration = tot_lat_measurement_duration + single_lat_measurement_duration
308                                 if lat_min > lat_min_sample:
309                                         lat_min = lat_min_sample
310                                 if lat_max < lat_max_sample:
311                                         lat_max = lat_max_sample
312                                 lat_avg = lat_avg + lat_avg_sample * single_lat_measurement_duration # Sometimes, There is more than 1 second between 2 lat_stats. Hence we will take the latest measurement
313                                 used_avg = used_avg + used_sample * single_lat_measurement_duration  # and give it more weigth.
314                                 lat_samples = sum(buckets)
315                                 tot_lat_samples += lat_samples
316                                 sample_count = 0
317                                 for sample_percentile, bucket in enumerate(buckets,start=1):
318                                         sample_count += bucket
319                                         if sample_count > lat_samples * LAT_PERCENTILE:
320                                                 break
321                                 percentile_max = (sample_percentile == len(buckets))
322                                 sample_percentile = sample_percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
323                                 buckets_total = [buckets_total[i] + buckets[i] for i in range(len(buckets_total))] 
324                                 t2_lat_tsc = t3_lat_tsc
325                                 lat_avail = True
326                         t3_rx, t3_non_dp_rx, t3_tx, t3_non_dp_tx, t3_drop, t3_tx_fail, t3_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks)
327                         if t3_tsc != t2_tsc:
328                                 single_core_measurement_duration = (t3_tsc - t2_tsc) * 1.0 / tsc_hz  # time difference between the 2 measurements, expressed in seconds.
329                                 tot_core_measurement_duration = tot_core_measurement_duration + single_core_measurement_duration
330                                 delta_rx = t3_rx - t2_rx
331                                 tot_rx += delta_rx
332                                 delta_non_dp_rx = t3_non_dp_rx - t2_non_dp_rx
333                                 tot_non_dp_rx += delta_non_dp_rx
334                                 delta_tx = t3_tx - t2_tx
335                                 tot_tx += delta_tx
336                                 delta_non_dp_tx = t3_non_dp_tx - t2_non_dp_tx
337                                 tot_non_dp_tx += delta_non_dp_tx
338                                 delta_dp_tx = delta_tx -delta_non_dp_tx
339                                 delta_dp_rx = delta_rx -delta_non_dp_rx
340                                 delta_dp_drop = delta_dp_tx - delta_dp_rx
341                                 tot_dp_drop += delta_dp_drop
342                                 delta_drop = t3_drop - t2_drop
343                                 tot_drop += delta_drop
344                                 t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc = t3_rx, t3_non_dp_rx, t3_tx, t3_non_dp_tx, t3_drop, t3_tx_fail, t3_tsc
345                                 core_avail = True
346                         if sutsock!='none':
347                                 t3_sut_rx, t3_sut_non_dp_rx, t3_sut_tx, t3_sut_non_dp_tx, t3_sut_drop, t3_sut_tx_fail, t3_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks)
348                                 if t3_sut_tsc != t2_sut_tsc:
349                                         single_sut_core_measurement_duration = (t3_sut_tsc - t2_sut_tsc) * 1.0 / tsc_hz  # time difference between the 2 measurements, expressed in seconds.
350                                         tot_sut_core_measurement_duration = tot_sut_core_measurement_duration + single_sut_core_measurement_duration
351                                         tot_sut_rx += t3_sut_rx - t2_sut_rx
352                                         tot_sut_non_dp_rx += t3_sut_non_dp_rx - t2_sut_non_dp_rx
353                                         delta_sut_tx = t3_sut_tx - t2_sut_tx
354                                         tot_sut_tx += delta_sut_tx
355                                         delta_sut_non_dp_tx = t3_sut_non_dp_tx - t2_sut_non_dp_tx
356                                         tot_sut_non_dp_tx += delta_sut_non_dp_tx 
357                                         t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc = t3_sut_rx, t3_sut_non_dp_rx, t3_sut_tx, t3_sut_non_dp_tx, t3_sut_drop, t3_sut_tx_fail, t3_sut_tsc
358                                         sut_avail = True
359                         if test == 'fixed_rate':
360                                 if lat_avail == core_avail == True:
361                                         lat_avail = core_avail = False
362                                         pps_req_tx = (delta_tx + delta_drop - delta_rx)/single_core_measurement_duration/1000000
363                                         pps_tx = delta_tx/single_core_measurement_duration/1000000
364                                         if sutsock!='none' and sut_avail:
365                                                 pps_sut_tx = delta_sut_tx/single_sut_core_measurement_duration/1000000
366                                                 sut_avail = False
367                                         else:
368                                                 pps_sut_tx = None
369                                         pps_rx = delta_rx/single_core_measurement_duration/1000000
370                                         log.info(report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg_sample,sample_percentile,percentile_max,lat_max_sample,delta_dp_tx,delta_dp_rx,tot_dp_drop,single_core_measurement_duration))
371                 #Stop generating
372                 gensock.stop(gencores)
373                 r += 1
374                 lat_avg = lat_avg / float(tot_lat_measurement_duration)
375                 used_avg = used_avg / float(tot_lat_measurement_duration)
376                 t4_tsc = t2_tsc
377                 while t4_tsc == t2_tsc:
378                         t4_rx, t4_non_dp_rx, t4_tx, t4_non_dp_tx, t4_drop, t4_tx_fail, t4_tsc, abs_tsc_hz = gensock.core_stats(genstatcores,gentasks)
379                 if test == 'fixed_rate':
380                         t4_lat_tsc = t2_lat_tsc
381                         while t4_lat_tsc == t2_lat_tsc:
382                                 lat_min_sample, lat_max_sample, lat_avg_sample, used_sample, t4_lat_tsc, lat_hz, buckets = gensock.lat_stats(latcores)
383                         sample_count = 0
384                         lat_samples = sum(buckets)
385                         for percentile, bucket in enumerate(buckets,start=1):
386                                 sample_count += bucket
387                                 if sample_count > lat_samples * LAT_PERCENTILE:
388                                         break
389                         percentile_max = (percentile == len(buckets))
390                         percentile = percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
391                         lat_max = lat_max_sample
392                         lat_avg = lat_avg_sample
393                         delta_rx = t4_rx - t2_rx
394                         delta_non_dp_rx = t4_non_dp_rx - t2_non_dp_rx
395                         delta_tx = t4_tx - t2_tx
396                         delta_non_dp_tx = t4_non_dp_tx - t2_non_dp_tx
397                         delta_dp_tx = delta_tx -delta_non_dp_tx
398                         delta_dp_rx = delta_rx -delta_non_dp_rx
399                         dp_tx = delta_dp_tx
400                         dp_rx = delta_dp_rx
401                         tot_dp_drop += delta_dp_tx - delta_dp_rx
402                         pps_req_tx = None
403                         pps_tx = None
404                         pps_sut_tx = None
405                         pps_rx = None
406                         drop_rate = 100.0*(dp_tx-dp_rx)/dp_tx
407                         tot_core_measurement_duration = None
408                         break ## Not really needed since the while loop will stop when evaluating the value of r
409                 else:
410                         sample_count = 0
411                         for percentile, bucket in enumerate(buckets_total,start=1):
412                                 sample_count += bucket
413                                 if sample_count > tot_lat_samples * LAT_PERCENTILE:
414                                         break
415                         percentile_max = (percentile == len(buckets_total))
416                         percentile = percentile *  float(2 ** BUCKET_SIZE_EXP) / (float(lat_hz)/float(10**6))
417                         pps_req_tx = (tot_tx + tot_drop - tot_rx)/tot_core_measurement_duration/1000000.0 # tot_drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
418                         pps_tx = tot_tx/tot_core_measurement_duration/1000000.0 # tot_tx is all generated packets actually accepted by the interface
419                         pps_rx = tot_rx/tot_core_measurement_duration/1000000.0 # tot_rx is all packets received by the nop task = all packets received in the gen VM
420                         if sutsock!='none' and sut_avail:
421                                 pps_sut_tx = tot_sut_tx / tot_sut_core_measurement_duration / 1000000.0
422                         else:
423                                 pps_sut_tx = None
424                         dp_tx = (t4_tx - t1_tx) - (t4_non_dp_tx - t1_non_dp_tx)
425                         dp_rx = (t4_rx - t1_rx) - (t4_non_dp_rx - t1_non_dp_rx)
426                         tot_dp_drop = dp_tx - dp_rx
427                         drop_rate = 100.0*tot_dp_drop/dp_tx
428                         if ((drop_rate < DROP_RATE_TRESHOLD) or (tot_dp_drop == DROP_RATE_TRESHOLD ==0) or (tot_dp_drop > TST009_MAXz)):
429                                 break
430         return(pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,percentile,percentile_max,lat_max,dp_tx,dp_rx,tot_dp_drop,(t4_tx_fail - t1_tx_fail),drop_rate,lat_min,used_avg,r,tot_core_measurement_duration)
431
432 def new_speed(speed,size,success):
433         if test == 'fixed_rate':
434                 return (STARTSPEED)
435         elif TST009:
436                 global TST009_m
437                 global TST009_L
438                 global TST009_R
439                 if success:
440                         TST009_L = TST009_m + 1
441                 else:
442                         TST009_R = max(TST009_m - 1, TST009_L)
443                 TST009_m = int ((TST009_L + TST009_R)/2)
444                 return (get_percentageof10Gbps(TST009_S[TST009_m],size))
445         else:
446                 global minspeed
447                 global maxspeed
448                 if success:
449                         minspeed = speed
450                 else:
451                         maxspeed = speed
452                 return ((minspeed + maxspeed)/2.0)
453
454 def get_start_speed_and_init(size):
455         if test == 'fixed_rate':
456                 return (STARTSPEED)
457         elif TST009:
458                 global TST009_L
459                 global TST009_R
460                 global TST009_m
461                 TST009_L = 0
462                 TST009_R = TST009_n - 1
463                 TST009_m = int((TST009_L + TST009_R) / 2)
464                 return (get_percentageof10Gbps(TST009_S[TST009_m],size))
465         else:
466                 global minspeed
467                 global maxspeed
468                 minspeed = 0
469                 maxspeed = STARTSPEED 
470                 return (STARTSPEED)
471
472 def resolution_achieved():
473         if test == 'fixed_rate':
474                 return (True)
475         elif TST009:
476                 return (TST009_L == TST009_R)
477         else:
478                 return ((maxspeed - minspeed) <= ACCURACY)
479
480 def get_percentageof10Gbps(pps_speed,size):
481         # speed is given in pps, returning % of 10Gb/s
482         return (pps_speed / 1000000.0 * 0.08 * (size+24))
483
484 def get_pps(speed,size):
485         # speed is given in % of 10Gb/s, returning Mpps
486         return (speed * 100.0 / (8*(size+24)))
487
488 def get_speed(packet_speed,size):
489         # return speed in Gb/s
490         return (packet_speed / 1000.0 * (8*(size+24)))
491
492 def set_background_flows(source_port,destination_port):
493         for sock in background_gen_socks:
494                 sock.set_random(gencores,0,34,source_port,2)
495                 sock.set_random(gencores,0,36,destination_port,2)
496
497 def set_background_speed(speed):
498         for sock in background_gen_socks:
499                 sock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
500
501 def start_background_traffic():
502         for sock in background_gen_socks:
503                 sock.start(gencores+latcores)
504
505 def stop_background_traffic():
506         for sock in background_gen_socks:
507                 sock.stop(gencores+latcores)
508
509 def run_flow_size_test(gensock,sutsock):
510         global fieldnames
511         global writer
512         #fieldnames = ['Flows','PacketSize','Gbps','Mpps','AvgLatency','MaxLatency','PacketsDropped','PacketDropRate']
513         fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Sent','Received','Lost','LostTotal']
514         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
515         writer.writeheader()
516         gensock.start(latcores)
517         for size in packet_size_list:
518                 size = size-4
519                 gensock.set_size(gencores,0,size) # This is setting the frame size
520                 gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
521                 gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
522                 # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
523                 if background_gen_socks:
524                         backgroundinfo = '{}Running {} x background traffic not represented in the table{}'.format(bcolors.FLASH,len(background_gen_socks),bcolors.ENDC)
525                 else:
526                         backgroundinfo = '{}{}'.format(bcolors.FLASH,bcolors.ENDC)
527                 log.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
528                 log.info('| UDP, {:>5} bytes, different number of flows by randomizing SRC & DST UDP port. {:116.116}|'.format((size+4),backgroundinfo))
529                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
530                 log.info('| Flows  | Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core           | Avg. Lat.|{:.0f} Pcentil| Max. Lat.|   Sent    |  Received |    Lost   | Total Lost|L.Ratio|Time|'.format(LAT_PERCENTILE*100))
531                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
532                 for flow_number in flow_size_list:
533                         attempts = 0
534                         gensock.reset_stats()
535                         if sutsock!='none':
536                                 sutsock.reset_stats()
537                         source_port,destination_port = flows[flow_number]
538                         gensock.set_random(gencores,0,34,source_port,2)
539                         gensock.set_random(gencores,0,36,destination_port,2)
540                         set_background_flows(source_port,destination_port)
541                         endspeed = None
542                         speed = get_start_speed_and_init(size)
543                         while True:
544                                 attempts += 1
545                                 endwarning = False
546                                 print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
547                                 sys.stdout.flush()
548                                 # Start generating packets at requested speed (in % of a 10Gb/s link)
549                                 gensock.speed(speed / len(gencores) / len (gentasks), gencores, gentasks)
550                                 set_background_speed(speed)
551                                 start_background_traffic()
552                                 ##time.sleep(1)
553                                 # Get statistics now that the generation is stable and initial ARP messages are dealt with
554                                 pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc , lat_perc_max, lat_max, abs_tx,abs_rx,abs_dropped, abs_tx_fail, drop_rate, lat_min, lat_used, r, actual_duration = run_iteration(gensock,sutsock,float(runtime),flow_number,size,speed)
555                                 stop_background_traffic()
556                                 if r > 1:
557                                         retry_warning = bcolors.WARNING + ' {:1} retries needed'.format(r) +  bcolors.ENDC
558                                 else:
559                                         retry_warning = ''
560                                 # Drop rate is expressed in percentage. lat_used is a ratio (0 to 1). The sum of these 2 should be 100%.
561                                 # If the sum is lower than 95, it means that more than 5% of the latency measurements where dropped for accuracy reasons.
562                                 if (drop_rate + lat_used * 100) < 95:
563                                         lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
564                                 else:
565                                         lat_warning = ''
566                                 if test == 'fixed_rate':
567                                         endspeed = speed
568                                         endpps_req_tx = None
569                                         endpps_tx = None
570                                         endpps_sut_tx = None
571                                         endpps_rx = None
572                                         endlat_avg = lat_avg
573                                         endlat_perc = lat_perc
574                                         endlat_perc_max = lat_perc_max
575                                         endlat_max = lat_max
576                                         endabs_dropped = abs_dropped
577                                         enddrop_rate = drop_rate
578                                         endabs_tx = abs_tx
579                                         endabs_rx = abs_rx
580                                         if lat_warning or retry_warning:
581                                                 endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning)
582                                         success = True
583                                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
584                                 # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold
585                                 # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
586                                 # This can be specified by putting 0 in the .test file
587                                 elif ((drop_rate < DROP_RATE_TRESHOLD) or (abs_dropped==DROP_RATE_TRESHOLD==0)) and (lat_avg< LAT_AVG_TRESHOLD) and (lat_perc< LAT_PERC_TRESHOLD) and (lat_max < LAT_MAX_TRESHOLD):
588                                         if ((get_pps(speed,size) - pps_tx)/get_pps(speed,size))>0.01:
589                                                 speed_prefix = bcolors.WARNING
590                                                 if abs_tx_fail > 0:
591                                                         gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
592                                                 else:
593                                                         gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(get_pps(speed,size), pps_tx) + bcolors.ENDC
594                                         else:
595                                                 speed_prefix = bcolors.ENDC
596                                                 gen_warning = ''
597                                         endspeed = speed
598                                         endspeed_prefix = speed_prefix
599                                         endpps_req_tx = pps_req_tx
600                                         endpps_tx = pps_tx
601                                         endpps_sut_tx = pps_sut_tx
602                                         endpps_rx = pps_rx
603                                         endlat_avg = lat_avg
604                                         endlat_perc = lat_perc
605                                         endlat_perc_max = lat_perc_max
606                                         endlat_max = lat_max
607                                         endabs_dropped = None
608                                         enddrop_rate = drop_rate
609                                         endabs_tx = abs_tx
610                                         endabs_rx = abs_rx
611                                         if lat_warning or gen_warning or retry_warning:
612                                                 endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning + gen_warning)
613                                         success = True
614                                         success_message=' SUCCESS'
615                                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
616                                         log.debug(report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
617                                 else:
618                                         success_message=' FAILED'
619                                         gen_warning = ''
620                                         abs_drop_rate_prefix = bcolors.ENDC
621                                         if ((abs_dropped>0) and (DROP_RATE_TRESHOLD ==0)):
622                                                 abs_drop_rate_prefix = bcolors.FAIL
623                                         if (drop_rate < DROP_RATE_TRESHOLD):
624                                                 drop_rate_prefix = bcolors.ENDC
625                                         else:
626                                                 drop_rate_prefix = bcolors.FAIL
627                                         if (lat_avg< LAT_AVG_TRESHOLD):
628                                                 lat_avg_prefix = bcolors.ENDC
629                                         else:
630                                                 lat_avg_prefix = bcolors.FAIL
631                                         if (lat_perc< LAT_PERC_TRESHOLD):
632                                                 lat_perc_prefix = bcolors.ENDC
633                                         else:
634                                                 lat_perc_prefix = bcolors.FAIL
635                                         if (lat_max< LAT_MAX_TRESHOLD):
636                                                 lat_max_prefix = bcolors.ENDC
637                                         else:
638                                                 lat_max_prefix = bcolors.FAIL
639                                         if (((get_pps(speed,size) - pps_tx)/get_pps(speed,size))<0.001):
640                                                 speed_prefix = bcolors.ENDC
641                                         else:
642                                                 speed_prefix = bcolors.FAIL
643                                         success = False 
644                                         log.debug(report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
645                                 speed = new_speed(speed, size, success)
646                                 if resolution_achieved():
647                                         break
648                         if endspeed is not  None:
649                                 speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
650                                 log.info(report_result(flow_number,size,endspeed,endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_perc,endlat_perc_max,endlat_max,endabs_tx,endabs_rx,endabs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix))
651                                 if endwarning:
652                                         log.info (endwarning)
653                                 log.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
654                                 writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':get_pps(endspeed,size),'GeneratedPPS':endpps_req_tx,'SentPPS':endpps_tx,'ForwardedPPS':endpps_sut_tx,'ReceivedPPS':endpps_rx,'AvgLatencyUSEC':endlat_avg,'MaxLatencyUSEC':endlat_max,'Sent':endabs_tx,'Received':endabs_rx,'Lost':endabs_dropped,'LostTotal':endabs_dropped})
655                                 if PushGateway:
656                                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
657                                         if endabs_dropped == None:
658                                                 ead = 0
659                                         else:
660                                                 ead = endabs_dropped
661                                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nSent {}\nReceived {}\nLost {}\nLostTotal {}\n'.format(flow_number,size+4,get_pps(endspeed,size),endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_max,endabs_tx,endabs_rx,ead,ead)
662                                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
663                                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
664                                         if (response.status_code != 202) and (response.status_code != 200):
665                                                 log.info('Cannot send metrics to {}'.format(URL))
666                                                 log.info(DATA)
667                         else:
668                                 log.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
669         gensock.stop(latcores)
670
671 def run_core_stats(socks):
672         fieldnames = ['PROXID','Time','Received','Sent','NonDPReceived','NonDPSent','Delta','NonDPDelta','Dropped']
673         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
674         writer.writeheader()
675         log.info("+------------------------------------------------------------------------------------------------------------------+")
676         log.info("| Measuring core statistics on 1 or more PROX instances                                                            |")
677         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
678         log.info("| PROX ID   |    Time   |    RX      |     TX     | non DP RX  | non DP TX  |   TX - RX  | nonDP TX-RX|  DROP TOT  |")
679         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
680         for sock in socks:
681                 sock.reset_stats()
682         duration = float(runtime)
683         tot_drop = []
684         old_rx = []; old_non_dp_rx = []; old_tx = []; old_non_dp_tx = []; old_drop = []; old_tx_fail = []; old_tsc = []
685         new_rx = []; new_non_dp_rx = []; new_tx = []; new_non_dp_tx = []; new_drop = []; new_tx_fail = []; new_tsc = []
686         sockets_to_go = len (socks)
687         for i,sock in enumerate(socks,start=0):
688                 tot_drop.append(0)
689                 old_rx.append(0); old_non_dp_rx.append(0); old_tx.append(0); old_non_dp_tx.append(0); old_drop.append(0); old_tx_fail.append(0); old_tsc.append(0)
690                 old_rx[-1], old_non_dp_rx[-1], old_tx[-1], old_non_dp_tx[-1], old_drop[-1], old_tx_fail[-1], old_tsc[-1], tsc_hz = sock.core_stats(cores[i],tasks)
691                 new_rx.append(0); new_non_dp_rx.append(0); new_tx.append(0); new_non_dp_tx.append(0); new_drop.append(0); new_tx_fail.append(0); new_tsc.append(0)
692         while (duration > 0):
693                 time.sleep(0.5)
694                 # Get statistics after some execution time
695                 for i,sock in enumerate(socks,start=0):
696                         new_rx[i], new_non_dp_rx[i], new_tx[i], new_non_dp_tx[i], new_drop[i], new_tx_fail[i], new_tsc[i], tsc_hz = sock.core_stats(cores[i],tasks)
697                         drop = new_drop[i]-old_drop[i]
698                         rx = new_rx[i] - old_rx[i]
699                         tx = new_tx[i] - old_tx[i]
700                         non_dp_rx = new_non_dp_rx[i] - old_non_dp_rx[i]
701                         non_dp_tx = new_non_dp_tx[i] - old_non_dp_tx[i]
702                         tsc = new_tsc[i] - old_tsc[i]
703                         if tsc == 0 :
704                                 continue
705                         sockets_to_go -= 1
706                         old_drop[i] = new_drop[i]
707                         old_rx[i] = new_rx[i]
708                         old_tx[i] = new_tx[i]
709                         old_non_dp_rx[i] = new_non_dp_rx[i]
710                         old_non_dp_tx[i] = new_non_dp_tx[i]
711                         old_tsc[i] = new_tsc[i]
712                         tot_drop[i] = tot_drop[i] + tx - rx
713                         log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(non_dp_rx)+' | '+'{:>10.0f}'.format(non_dp_tx)+' | ' + '{:>10.0f}'.format(tx-rx) + ' | '+ '{:>10.0f}'.format(non_dp_tx-non_dp_rx) + ' | '+'{:>10.0f}'.format(tot_drop[i]) +' |')
714                         writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NonDPReceived':non_dp_rx,'NonDPSent':non_dp_tx,'Delta':tx-rx,'NonDPDelta':non_dp_tx-non_dp_rx,'Dropped':tot_drop[i]})
715                         if PushGateway:
716                                 URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
717                                 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNonDPReceived {}\nNonDPSent {}\nDelta {}\nNonDPDelta {}\nDropped {}\n'.format(i,duration,rx,tx,non_dp_rx,non_dp_tx,tx-rx,non_dp_tx-non_dp_rx,tot_drop[i])
718                                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
719                                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
720                                 if (response.status_code != 202) and (response.status_code != 200):
721                                         log.info('Cannot send metrics to {}'.format(URL))
722                                         log.info(DATA)
723                         if sockets_to_go == 0:
724                                 duration = duration - 1
725                                 sockets_to_go = len (socks)
726         log.info("+-----------+-----------+------------+------------+------------+------------+------------+------------+------------+")
727
728 def run_port_stats(socks):
729         fieldnames = ['PROXID','Time','Received','Sent','NoMbufs','iErrMiss']
730         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
731         writer.writeheader()
732         log.info("+---------------------------------------------------------------------------+")
733         log.info("| Measuring port statistics on 1 or more PROX instances                     |")
734         log.info("+-----------+-----------+------------+------------+------------+------------+")
735         log.info("| PROX ID   |    Time   |    RX      |     TX     | no MBUFS   | ierr&imiss |")
736         log.info("+-----------+-----------+------------+------------+------------+------------+")
737         for sock in socks:
738                 sock.reset_stats()
739         duration = float(runtime)
740         old_rx = []; old_tx = []; old_no_mbufs = []; old_errors = []; old_tsc = []
741         new_rx = []; new_tx = []; new_no_mbufs = []; new_errors = []; new_tsc = []
742         sockets_to_go = len (socks)
743         for i,sock in enumerate(socks,start=0):
744                 old_rx.append(0); old_tx.append(0); old_no_mbufs.append(0); old_errors.append(0); old_tsc.append(0)
745                 old_rx[-1], old_tx[-1], old_no_mbufs[-1], old_errors[-1], old_tsc[-1] = sock.multi_port_stats(ports[i])
746                 new_rx.append(0); new_tx.append(0); new_no_mbufs.append(0); new_errors.append(0); new_tsc.append(0)
747         while (duration > 0):
748                 time.sleep(0.5)
749                 # Get statistics after some execution time
750                 for i,sock in enumerate(socks,start=0):
751                         new_rx[i], new_tx[i], new_no_mbufs[i], new_errors[i], new_tsc[i] = sock.multi_port_stats(ports[i])
752                         rx = new_rx[i] - old_rx[i]
753                         tx = new_tx[i] - old_tx[i]
754                         no_mbufs = new_no_mbufs[i] - old_no_mbufs[i]
755                         errors = new_errors[i] - old_errors[i]
756                         tsc = new_tsc[i] - old_tsc[i]
757                         if tsc == 0 :
758                                 continue
759                         sockets_to_go -= 1
760                         old_rx[i] = new_rx[i]
761                         old_tx[i] = new_tx[i]
762                         old_no_mbufs[i] = new_no_mbufs[i]
763                         old_errors[i] = new_errors[i]
764                         old_tsc[i] = new_tsc[i]
765                         log.info('|{:>10.0f}'.format(i)+ ' |{:>10.0f}'.format(duration)+' | ' + '{:>10.0f}'.format(rx) + ' | ' +'{:>10.0f}'.format(tx) + ' | '+'{:>10.0f}'.format(no_mbufs)+' | '+'{:>10.0f}'.format(errors)+' |')
766                         writer.writerow({'PROXID':i,'Time':duration,'Received':rx,'Sent':tx,'NoMbufs':no_mbufs,'iErrMiss':errors})
767                         if PushGateway:
768                                 URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env + str(i)
769                                 DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNoMbufs {}\niErrMiss {}\n'.format(i,duration,rx,tx,no_mbufs,errors)
770                                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
771                                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
772                                 if (response.status_code != 202) and (response.status_code != 200):
773                                         log.info('Cannot send metrics to {}'.format(URL))
774                                         log.info(DATA)
775                         if sockets_to_go == 0:
776                                 duration = duration - 1
777                                 sockets_to_go = len (socks)
778         log.info("+-----------+-----------+------------+------------+------------+------------+")
779
780 def run_irqtest(socks):
781         log.info("+----------------------------------------------------------------------------------------------------------------------------")
782         log.info("| Measuring time probably spent dealing with an interrupt. Interrupting DPDK cores for more than 50us might be problematic   ")
783         log.info("| and result in packet loss. The first row shows the interrupted time buckets: first number is the bucket between 0us and    ")
784         log.info("| that number expressed in us and so on. The numbers in the other rows show how many times per second, the program was       ")
785         log.info("| interrupted for a time as specified by its bucket. '0' is printed when there are no interrupts in this bucket throughout   ")
786         log.info("| the duration of the test. 0.00 means there were interrupts in this bucket but very few. Due to rounding this shows as 0.00 ") 
787         log.info("+----------------------------------------------------------------------------------------------------------------------------")
788         sys.stdout.flush()
789         for sock_index,sock in enumerate(socks,start=0):
790                 buckets=socks[sock_index].show_irq_buckets(1)
791                 print('Measurement ongoing ... ',end='\r')
792                 socks[sock_index].stop(cores[sock_index])
793                 old_irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)] 
794                 irq = [[0 for x in range(len(buckets)+1)] for y in range(len(cores[sock_index])+1)]
795                 irq[0][0] = 'bucket us' 
796                 for j,bucket in enumerate(buckets,start=1):
797                         irq[0][j] = '<'+ bucket
798                 irq[0][-1] = '>'+ buckets [-2]
799                 socks[sock_index].start(cores[sock_index])
800                 time.sleep(2)
801                 for j,bucket in enumerate(buckets,start=1):
802                         for i,irqcore in enumerate(cores[sock_index],start=1):
803                                 old_irq[i][j] = socks[sock_index].irq_stats(irqcore,j-1)
804                 time.sleep(float(runtime))
805                 socks[sock_index].stop(cores[sock_index])
806                 for i,irqcore in enumerate(cores[sock_index],start=1):
807                         irq[i][0]='core %s '%irqcore
808                         for j,bucket in enumerate(buckets,start=1):
809                                 diff =  socks[sock_index].irq_stats(irqcore,j-1) - old_irq[i][j]
810                                 if diff == 0:
811                                         irq[i][j] = '0'
812                                 else:
813                                         irq[i][j] = str(round(diff/float(runtime), 2))
814                 log.info('Results for PROX instance %s'%sock_index)
815                 for row in irq:
816                         log.info(''.join(['{:>12}'.format(item) for item in row]))
817
818 def run_impairtest(gensock,sutsock):
819         fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Dropped','DropRate']
820         writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
821         writer.writeheader()
822         size=PACKETSIZE-4
823         log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
824         log.info("| Generator is sending UDP ("+'{:>5}'.format(FLOWSIZE)+" flow) packets ("+ '{:>5}'.format(size+4) +" bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test    |")
825         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
826         log.info("| Test   |  Speed requested   | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
827         log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
828         attempts = 0
829         gensock.set_size(gencores,0,size) # This is setting the frame size
830         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
831         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
832         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
833         source_port,destination_port = flows[FLOWSIZE]
834         gensock.set_random(gencores,0,34,source_port,2)
835         gensock.set_random(gencores,0,36,destination_port,2)
836         gensock.start(latcores)
837         speed = STARTSPEED
838         gensock.speed(speed / len(gencores) / len(gentasks), gencores, gentasks)
839         while True:
840                 attempts += 1
841                 print('Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
842                 sys.stdout.flush()
843                 time.sleep(1)
844                 # Get statistics now that the generation is stable and NO ARP messages any more
845                 pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg, lat_perc, lat_perc_max, lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used, r, actual_duration = run_iteration(gensock,sutsock,runtime)
846                 drop_rate = 100.0*abs_dropped/abs_tx
847                 if lat_used < 0.95:
848                         lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
849                 else:
850                         lat_warning = ''
851                 log.info('|{:>7}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | '+ '{:>9.3f}'.format(pps_tx) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9.3f}'.format(pps_rx)+' Mpps | '+ '{:>9.0f}'.format(lat_avg)+' us   | '+ '{:>9.0f}'.format(lat_max)+' us   | '+ '{:>14d}'.format(abs_dropped)+ ' |''{:>9.2f}'.format(drop_rate)+ '%  |'+lat_warning)
852                 writer.writerow({'Flows':FLOWSIZE,'PacketSize':(size+4),'RequestedPPS':get_pps(speed,size),'GeneratedPPS':pps_req_tx,'SentPPS':pps_tx,'ForwardedPPS':pps_sut_tx_str,'ReceivedPPS':pps_rx,'AvgLatencyUSEC':lat_avg,'MaxLatencyUSEC':lat_max,'Dropped':abs_dropped,'DropRate':drop_rate})
853                 if PushGateway:
854                         URL     = PushGateway + '/metrics/job/' + TestName + '/instance/' + env
855                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nDropped {}\nDropRate {}\n'.format(FLOWSIZE,size+4,get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,drop_rate)
856                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
857                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
858                         if (response.status_code != 202) and (response.status_code != 200):
859                                 log.info('Cannot send metrics to {}'.format(URL))
860                                 log.info(DATA)
861
862 def run_warmuptest(gensock):
863 # Running at low speed to make sure the ARP messages can get through.
864 # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
865 # Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
866 # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
867         gensock.speed(WARMUPSPEED / len(gencores) /len (gentasks), gencores, gentasks)
868         size=PACKETSIZE-4
869         gensock.set_size(gencores,0,size) # This is setting the frame size
870         gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
871         gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
872         gensock.set_value(gencores,0,56,1,1)
873         # This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
874         source_port,destination_port = flows[FLOWSIZE]
875         gensock.set_random(gencores,0,34,source_port,2)
876         gensock.set_random(gencores,0,36,destination_port,2)
877         gensock.start(genstatcores)
878         time.sleep(WARMUPTIME)
879         gensock.stop(genstatcores)
880         gensock.set_value(gencores,0,56,50,1)
881         time.sleep(WARMUPTIME)
882
883 def exit_handler():
884         log.debug ('exit cleanup')
885         for index, sock in enumerate(socks):
886                 if socks_control[index]:
887                         sock.quit()
888         for client in clients:
889                 client.close()
890         data_csv_file.close
891         sys.exit(0)
892
893 def get_BinarySearchParams() :
894         global  DROP_RATE_TRESHOLD
895         global  LAT_AVG_TRESHOLD
896         global  LAT_PERC_TRESHOLD
897         global  LAT_MAX_TRESHOLD
898         global  ACCURACY
899         global  STARTSPEED
900         global  TST009
901         global  TST009_MAXr
902         global  TST009_MAXz
903         DROP_RATE_TRESHOLD = float(testconfig.get('BinarySearchParams', 'drop_rate_threshold'))
904         LAT_AVG_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_avg_threshold'))
905         LAT_PERC_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_perc_threshold'))
906         LAT_MAX_TRESHOLD = float(testconfig.get('BinarySearchParams', 'lat_max_threshold'))
907         ACCURACY = float(testconfig.get('BinarySearchParams', 'accuracy'))
908         STARTSPEED = float(testconfig.get('BinarySearchParams', 'startspeed'))
909         TST009_MAXr = 1
910         TST009_MAXz = inf
911         TST009 = False
912         
913 def get_FixedRateParams() :
914         global  DROP_RATE_TRESHOLD
915         global  LAT_AVG_TRESHOLD
916         global  LAT_PERC_TRESHOLD
917         global  LAT_MAX_TRESHOLD
918         global  flow_size_list
919         global  packet_size_list
920         global  STARTSPEED
921         global  TST009
922         global  TST009_MAXr
923         global  TST009_MAXz
924         DROP_RATE_TRESHOLD = inf
925         LAT_AVG_TRESHOLD = inf
926         LAT_PERC_TRESHOLD = inf
927         LAT_MAX_TRESHOLD = inf
928         TST009_MAXr = 1
929         TST009_MAXz = inf
930         TST009 = False
931         packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
932         flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
933         STARTSPEED = float(testconfig.get('test%d'%test_nr, 'speed'))
934         
935 def get_TST009SearchParams() :
936         global  DROP_RATE_TRESHOLD
937         global  LAT_AVG_TRESHOLD
938         global  LAT_PERC_TRESHOLD
939         global  LAT_MAX_TRESHOLD
940         global  TST009
941         global  TST009_MAXr
942         global  TST009_MAXz
943         global  TST009_MAXFramesAllIngress
944         global  TST009_StepSize
945         global  TST009_n
946         global  TST009_L
947         global  TST009_R
948         global  TST009_S
949         if testconfig.has_option('TST009SearchParams', 'drop_rate_threshold'):
950                 DROP_RATE_TRESHOLD = float(testconfig.get('TST009SearchParams', 'drop_rate_threshold'))
951         else:
952                 DROP_RATE_TRESHOLD = 0
953         LAT_AVG_TRESHOLD = inf
954         LAT_PERC_TRESHOLD = inf
955         LAT_MAX_TRESHOLD = inf
956         TST009_MAXr = float(testconfig.get('TST009SearchParams', 'MAXr'))
957         TST009_MAXz = float(testconfig.get('TST009SearchParams', 'MAXz'))
958         TST009_MAXFramesAllIngress = int(testconfig.get('TST009SearchParams', 'MAXFramesPerSecondAllIngress'))
959         TST009_StepSize = int(testconfig.get('TST009SearchParams', 'StepSize'))
960         TST009_n = int(ceil(TST009_MAXFramesAllIngress / TST009_StepSize))
961         TST009 = True
962         TST009_L = 0
963         TST009_R = TST009_n - 1
964         for m in range(0, TST009_n):
965                 TST009_S.append((m+1) * TST009_StepSize)
966 # To generate a desired number of flows, PROX will randomize the bits in source and destination ports, as specified by the bit masks in the flows variable. 
967 flows={\
968 1:      ['1000000000000000','1000000000000000'],\
969 2:      ['1000000000000000','100000000000000X'],\
970 4:      ['100000000000000X','100000000000000X'],\
971 8:      ['100000000000000X','10000000000000XX'],\
972 16:     ['10000000000000XX','10000000000000XX'],\
973 32:     ['10000000000000XX','1000000000000XXX'],\
974 64:     ['1000000000000XXX','1000000000000XXX'],\
975 128:    ['1000000000000XXX','100000000000XXXX'],\
976 256:    ['100000000000XXXX','100000000000XXXX'],\
977 512:    ['100000000000XXXX','10000000000XXXXX'],\
978 1024:   ['10000000000XXXXX','10000000000XXXXX'],\
979 2048:   ['10000000000XXXXX','1000000000XXXXXX'],\
980 4096:   ['1000000000XXXXXX','1000000000XXXXXX'],\
981 8192:   ['1000000000XXXXXX','100000000XXXXXXX'],\
982 16384:  ['100000000XXXXXXX','100000000XXXXXXX'],\
983 32768:  ['100000000XXXXXXX','10000000XXXXXXXX'],\
984 65536:  ['10000000XXXXXXXX','10000000XXXXXXXX'],\
985 131072: ['10000000XXXXXXXX','1000000XXXXXXXXX'],\
986 262144: ['1000000XXXXXXXXX','1000000XXXXXXXXX'],\
987 524288: ['1000000XXXXXXXXX','100000XXXXXXXXXX'],\
988 1048576:['100000XXXXXXXXXX','100000XXXXXXXXXX'],}
989 clients =[]
990 socks =[]
991 socks_control =[]
992 vmDPIP =[]
993 vmAdminIP =[]
994 vmDPmac =[]
995 hexDPIP =[]
996 vmDPPCIDEV =[]
997 config_file =[]
998 prox_socket =[]
999 prox_launch_exit =[]
1000 mach_type =[]
1001 sock_type =[]
1002 monitor =[]
1003 sock_monitor =[]
1004 cores = []
1005 ports = []
1006 tasks = {}
1007 TST009_S = []
1008
1009 data_file = 'RUN{}.{}.csv'.format(env,test_file)
1010 data_csv_file = open(data_file,'w')
1011 testconfig = ConfigParser.RawConfigParser()
1012 testconfig.read(test_file)
1013 required_number_of_test_machines = testconfig.get('DEFAULT', 'total_number_of_test_machines')
1014 TestName = testconfig.get('DEFAULT', 'name')
1015 if testconfig.has_option('DEFAULT', 'PushGateway'):
1016         PushGateway = testconfig.get('DEFAULT', 'PushGateway')
1017         log.info('Measurements will be pushed to %s'%PushGateway)
1018 else:
1019         PushGateway = None
1020 if testconfig.has_option('DEFAULT', 'lat_percentile'):
1021         LAT_PERCENTILE = float(testconfig.get('DEFAULT', 'lat_percentile')) /100.0
1022 else:
1023         LAT_PERCENTILE = 0.99
1024 log.info('Latency percentile measured at {:.0f}%'.format(LAT_PERCENTILE*100))
1025 config = ConfigParser.RawConfigParser()
1026 config.read(env)
1027 machine_map = ConfigParser.RawConfigParser()
1028 machine_map.read(machine_map_file)
1029 vim_type = config.get('Varia', 'vim')
1030 key = config.get('ssh', 'key')
1031 user = config.get('ssh', 'user')
1032 total_number_of_machines = config.get('rapid', 'total_number_of_machines')
1033 if int(required_number_of_test_machines) > int(total_number_of_machines):
1034         log.exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
1035         raise Exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_test_machines,total_number_of_machines))
1036 for vm in range(1, int(total_number_of_machines)+1):
1037         vmAdminIP.append(config.get('M%d'%vm, 'admin_ip'))
1038         vmDPmac.append(config.get('M%d'%vm, 'dp_mac'))
1039         vmDPIP.append(config.get('M%d'%vm, 'dp_ip'))
1040         ip = vmDPIP[-1].split('.')
1041         hexDPIP.append(hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2))
1042         if (vim_type == "kubernetes"):
1043                 vmDPPCIDEV.append(config.get('M%d'%vm, 'dp_pci_dev'))
1044 machine_index = []
1045 atexit.register(exit_handler)
1046 for vm in range(1, int(required_number_of_test_machines)+1):
1047         machine_index.append(int(machine_map.get('TestM%d'%vm, 'machine_index'))-1)
1048         prox_socket.append(testconfig.getboolean('TestM%d'%vm, 'prox_socket'))
1049 for vm in range(1, int(required_number_of_test_machines)+1):
1050         if prox_socket[vm-1]:
1051                 prox_launch_exit.append(testconfig.getboolean('TestM%d'%vm, 'prox_launch_exit'))
1052                 config_file.append(testconfig.get('TestM%d'%vm, 'config_file'))
1053                 # Looking for all task definitions in the PROX cfg files. Constructing a list of all tasks used
1054                 textfile =  open (config_file[-1], 'r')
1055                 filetext = textfile.read()
1056                 textfile.close()
1057                 tasks_for_this_cfg = set(re.findall("task\s*=\s*(\d+)",filetext))
1058                 with open('{}_{}_parameters{}.lua'.format(env,test_file,vm), "w") as f:
1059                         f.write('name="%s"\n'% testconfig.get('TestM%d'%vm, 'name'))
1060                         f.write('local_ip="%s"\n'% vmDPIP[machine_index[vm-1]])
1061                         f.write('local_hex_ip="%s"\n'% hexDPIP[machine_index[vm-1]])
1062                         if (vim_type == "kubernetes"):
1063                                 f.write("eal=\"--socket-mem=512,0 --file-prefix %s-%s-%s --pci-whitelist %s\"\n" % (env, test_file, vm, vmDPPCIDEV[machine_index[vm-1]]))
1064                         else:
1065                                 f.write("eal=\"\"\n")
1066                         if testconfig.has_option('TestM%d'%vm, 'cores'):
1067                                 cores.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'cores')))
1068                                 f.write('cores="%s"\n'% ','.join(map(str, cores[-1])))
1069                         else:
1070                                 cores.append(None)
1071                         if testconfig.has_option('TestM%d'%vm, 'ports'):
1072                                 ports.append(ast.literal_eval(testconfig.get('TestM%d'%vm, 'ports')))
1073                                 f.write('ports="%s"\n'% ','.join(map(str, ports[-1])))
1074                         else:
1075                                 ports.append(None)
1076                         if testconfig.has_option('TestM%d'%vm, 'monitor'):
1077                                 monitor.append(testconfig.getboolean('TestM%d'%vm, 'monitor'))
1078                         else:
1079                                 monitor.append(True)
1080                         if testconfig.has_option('TestM%d'%vm, 'gencores'):
1081                                 # This must be a generator VM
1082                                 gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores'))
1083                                 latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores'))
1084                                 genstatcores = gencores + latcores
1085                                 gentasks = tasks_for_this_cfg
1086                                 mach_type.append('gen')
1087                                 f.write('gencores="%s"\n'% ','.join(map(str, gencores)))
1088                                 f.write('latcores="%s"\n'% ','.join(map(str, latcores)))
1089                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
1090                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
1091                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
1092                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
1093                                 if testconfig.has_option('TestM%d'%vm, 'gw_vm'):
1094                                         gwVMindex = int(testconfig.get('TestM%d'%vm, 'gw_vm')) -1
1095                                         f.write('gw_ip="%s"\n'% vmDPIP[machine_index[gwVMindex]])
1096                                         f.write('gw_hex_ip="%s"\n'% hexDPIP[machine_index[gwVMindex]])
1097                                 if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'):
1098                                         BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp'))
1099                                 else:
1100                                         BUCKET_SIZE_EXP = 11
1101                                 f.write('bucket_size_exp="%s"\n'% BUCKET_SIZE_EXP)
1102                                 if testconfig.has_option('TestM%d'%vm, 'heartbeat'):
1103                                         heartbeat = int(testconfig.get('TestM%d'%vm, 'heartbeat'))
1104                                 else:
1105                                         heartbeat = 60
1106                                 f.write('heartbeat="%s"\n'% heartbeat)
1107                         elif testconfig.has_option('TestM%d'%vm, 'dest_vm'):
1108                                 # This must be a machine acting as a GW that we will not monitor
1109                                 mach_type.append('none')
1110                                 destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1
1111                                 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]])
1112                                 f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]])
1113                                 f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' '))
1114                         elif re.findall('mode\s*=\s*(swap|esp_dec)',filetext,re.MULTILINE):
1115                                 sutstatcores = cores[-1]
1116                                 mach_type.append('sut')
1117                         else:
1118                                 mach_type.append('none')
1119                 f.close
1120                 tasks = tasks_for_this_cfg.union(tasks)
1121 log.debug("Tasks detected in all PROX config files %r"%tasks)
1122 #####################################################################################
1123 for vm in range(0, int(required_number_of_test_machines)):
1124         if prox_socket[vm]:
1125                 clients.append(prox_ctrl(vmAdminIP[machine_index[vm]], key,user))
1126                 connect_client(clients[-1])
1127                 if (vim_type == "OpenStack"):
1128 # Creating script to bind the right network interface to the poll mode driver
1129                         devbindfile = '{}_{}_devbindvm{}.sh'.format(env,test_file, vm+1)
1130                         with open(devbindfile, "w") as f:
1131                                 newText= 'link="$(ip -o link | grep '+vmDPmac[machine_index[vm]]+' |cut -d":" -f 2)"\n'
1132                                 f.write(newText)
1133                                 newText= 'if [ -n "$link" ];\n'
1134                                 f.write(newText)
1135                                 newText= 'then\n'
1136                                 f.write(newText)
1137                                 newText= '        echo Need to bind\n'
1138                                 f.write(newText)
1139                                 newText= '        sudo ' + rundir + '/dpdk/usertools/dpdk-devbind.py --force --bind igb_uio $('+rundir+'/dpdk/usertools/dpdk-devbind.py --status |grep  $link | cut -d" " -f 1)\n'
1140                                 f.write(newText)
1141                                 newText= 'else\n'
1142                                 f.write(newText)
1143                                 newText= '       echo Assuming port is already bound to DPDK\n'
1144                                 f.write(newText)
1145                                 newText= 'fi\n'
1146                                 f.write(newText)
1147                                 newText= 'exit 0\n'
1148                                 f.write(newText)
1149                         st = os.stat(devbindfile)
1150                         os.chmod(devbindfile, st.st_mode | stat.S_IEXEC)
1151                         clients[-1].scp_put('./%s'%devbindfile, rundir+'/devbind.sh')
1152                         cmd = 'sudo ' + rundir+ '/devbind.sh'
1153                         clients[-1].run_cmd(cmd)
1154                         log.debug("devbind.sh running on VM%d"%(vm+1))
1155
1156                 clients[-1].scp_put('./%s'%config_file[vm], rundir+'/%s'%config_file[vm])
1157                 clients[-1].scp_put('./{}_{}_parameters{}.lua'.format(env,test_file, vm+1), rundir + '/parameters.lua')
1158                 if not configonly:
1159                         if prox_launch_exit[vm]:
1160                                 if mach_type[vm] == 'gen':
1161                                         cmd = 'sudo ' + rundir + '/prox -e -t -o cli -f ' + rundir + '/%s'%config_file[vm]
1162                                 else:
1163                                         cmd = 'sudo ' + rundir + '/prox -t -o cli -f ' + rundir + '/%s'%config_file[vm]
1164                                 log.debug("Starting PROX on VM{}: {}".format((vm+1),cmd))
1165                                 clients[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1))
1166                         socks_control.append(prox_launch_exit[vm])
1167                         socks.append(connect_socket(clients[-1]))
1168                         sock_type.append(mach_type[vm])
1169                         sock_monitor.append(monitor[vm])
1170 monitor_gen = monitor_sut = False
1171 background_gen_socks =[]
1172 for index, sock in enumerate(socks, start=0):
1173         if sock_type[index] == 'gen':
1174                 if sock_monitor[index]:
1175                         if monitor_gen:
1176                                 log.exception("Can only monitor 1 generator")
1177                                 raise Exception("Can only monitor 1 generator")
1178                         else:
1179                                 monitor_gen = True
1180                                 gensock_index = index
1181                 else:
1182                         background_gen_socks.append(sock)
1183         elif sock_type[index] == 'sut' and sock_monitor[index]:
1184                 if monitor_sut:
1185                         log.exception("Can only monitor 1 sut")
1186                         raise Exception("Can only monitor 1 sut")
1187                 else:
1188                         monitor_sut = True
1189                         sutsock_index = index
1190 if configonly:
1191         sys.exit()
1192 ####################################################
1193 # Run test cases
1194 # Best to run the flow test at the end since otherwise the tests coming after might be influenced by the big number of entries in the switch flow tables
1195 ####################################################
1196 number_of_tests = testconfig.get('DEFAULT', 'number_of_tests')
1197 for test_nr in range(1, int(number_of_tests)+1):
1198         test=testconfig.get('test%d'%test_nr,'test')
1199         log.info(test)
1200         if test == 'flowsizetest':
1201                 get_BinarySearchParams()
1202                 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1203                 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1204                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1205         elif test == 'TST009test':
1206                 # This test implements some of the testing as defined in https://docbox.etsi.org/ISG/NFV/open/Publications_pdf/Specs-Reports/NFV-TST%20009v3.2.1%20-%20GS%20-%20NFVI_Benchmarks.pdf
1207                 get_TST009SearchParams()
1208                 packet_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'packetsizes'))
1209                 flow_size_list = ast.literal_eval(testconfig.get('test%d'%test_nr, 'flows'))
1210                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1211         elif test == 'fixed_rate':
1212                 get_FixedRateParams()
1213                 run_flow_size_test(socks[gensock_index],socks[sutsock_index])
1214         elif test == 'corestats':
1215                 run_core_stats(socks)
1216         elif test == 'portstats':
1217                 run_port_stats(socks)
1218         elif test == 'impairtest':
1219                 get_BinarySearchParams()
1220                 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1221                 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1222                 run_impairtest(socks[gensock_index],socks[sutsock_index])
1223         elif test == 'irqtest':
1224                 run_irqtest(socks)
1225         elif test == 'warmuptest':
1226                 PACKETSIZE = int(testconfig.get('test%d'%test_nr, 'packetsize'))
1227                 FLOWSIZE = int(testconfig.get('test%d'%test_nr, 'flowsize'))
1228                 WARMUPSPEED = int(testconfig.get('test%d'%test_nr, 'warmupspeed'))
1229                 WARMUPTIME = int(testconfig.get('test%d'%test_nr, 'warmuptime'))
1230                 run_warmuptest(socks[gensock_index])
1231 ####################################################