4 ## Copyright (c) 2020 Intel Corporation
6 ## Licensed under the Apache License, Version 2.0 (the "License");
7 ## you may not use this file except in compliance with the License.
8 ## You may obtain a copy of the License at
11 ## http://www.apache.org/licenses/LICENSE-2.0
13 ## Unless required by applicable law or agreed to in writing, software
14 ## distributed under the License is distributed on an "AS IS" BASIS,
15 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 ## See the License for the specific language governing permissions and
17 ## limitations under the License.
22 from statistics import mean
23 from past.utils import old_div
24 from rapid_log import RapidLog
25 from rapid_log import bcolors
26 from rapid_test import RapidTest
29 class FlowSizeTest(RapidTest):
31 Class to manage the flowsizetesting
33 def __init__(self, test_param, lat_percentile, runtime, testname,
34 environment_file, gen_machine, sut_machine, background_machines):
35 super().__init__(test_param, runtime, testname, environment_file)
36 self.gen_machine = gen_machine
37 self.sut_machine = sut_machine
38 self.background_machines = background_machines
39 self.test['lat_percentile'] = lat_percentile
40 if self.test['test'] == 'TST009test':
41 # This test implements some of the testing as defined in
42 # https://docbox.etsi.org/ISG/NFV/open/Publications_pdf/Specs-Reports/NFV-TST%20009v3.2.1%20-%20GS%20-%20NFVI_Benchmarks.pdf
43 self.test['TST009_n'] = int(ceil(old_div(
44 self.test['maxframespersecondallingress'],
45 self.test['stepsize'])))
46 self.test['TST009'] = True
47 self.test['TST009_L'] = 0
48 self.test['TST009_R'] = self.test['TST009_n'] - 1
49 self.test['TST009_S']= []
50 for m in range(0, self.test['TST009_n']):
51 self.test['TST009_S'].append((m+1) * self.test['stepsize'])
52 elif self.test['test'] == 'fixed_rate':
53 for key in['drop_rate_threshold','lat_avg_threshold',
54 'lat_perc_threshold','lat_max_threshold']:
57 def new_speed(self, speed,size,success):
58 if self.test['test'] == 'fixed_rate':
59 return (self.test['startspeed'])
60 elif self.test['test'] == 'increment_till_fail':
61 return (speed + self.test['step'])
62 elif 'TST009' in self.test.keys():
64 self.test['TST009_L'] = self.test['TST009_m'] + 1
66 self.test['TST009_R'] = max(self.test['TST009_m'] - 1, self.test['TST009_L'])
67 self.test['TST009_m'] = int (old_div((self.test['TST009_L'] + self.test['TST009_R']),2))
68 return (self.get_percentageof10Gbps(self.test['TST009_S'][self.test['TST009_m']],size))
71 self.test['minspeed'] = speed
73 self.test['maxspeed'] = speed
74 return (old_div((self.test['minspeed'] + self.test['maxspeed']),2.0))
76 def get_start_speed_and_init(self, size):
77 if self.test['test'] == 'fixed_rate':
78 return (self.test['startspeed'])
79 elif self.test['test'] == 'increment_till_fail':
80 return (self.test['startspeed'])
81 elif 'TST009' in self.test.keys():
82 self.test['TST009_L'] = 0
83 self.test['TST009_R'] = self.test['TST009_n'] - 1
84 self.test['TST009_m'] = int(old_div((self.test['TST009_L'] + self.test['TST009_R']), 2))
85 return (self.get_percentageof10Gbps(self.test['TST009_S'][self.test['TST009_m']],size))
87 self.test['minspeed'] = 0
88 self.test['maxspeed'] = self.test['startspeed']
89 return (self.test['startspeed'])
91 def resolution_achieved(self):
92 if self.test['test'] == 'fixed_rate':
94 elif 'TST009' in self.test.keys():
95 return (self.test['TST009_L'] == self.test['TST009_R'])
97 return ((self.test['maxspeed'] - self.test['minspeed']) <= self.test['accuracy'])
102 # #fieldnames = ['Flows','PacketSize','Gbps','Mpps','AvgLatency','MaxLatency','PacketsDropped','PacketDropRate']
103 # fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Sent','Received','Lost','LostTotal']
104 # writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
105 # writer.writeheader()
106 self.gen_machine.start_latency_cores()
108 for imix in self.test['imixs']:
110 self.gen_machine.set_udp_packet_size(imix)
111 if self.background_machines:
112 backgroundinfo = '{}Running {} x background traffic not represented in the table{}'.format(bcolors.FLASH,len(self.background_machines),bcolors.ENDC)
114 backgroundinfo = '{}{}'.format(bcolors.FLASH,bcolors.ENDC)
115 self.set_background_size(self.background_machines, imix)
116 RapidLog.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
117 RapidLog.info('| UDP, {:>5} bytes, different number of flows by randomizing SRC & DST UDP port. {:116.116}|'.format(size, backgroundinfo))
118 RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
119 RapidLog.info('| Flows | Speed requested | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core | Avg. Lat.|{:.0f} Pcentil| Max. Lat.| Sent | Received | Lost | Total Lost|L.Ratio|Time|'.format(self.test['lat_percentile']*100))
120 RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
121 for flow_number in self.test['flows']:
123 self.gen_machine.reset_stats()
125 self.sut_machine.reset_stats()
126 flow_number = self.gen_machine.set_flows(flow_number)
127 self.set_background_flows(self.background_machines, flow_number)
129 speed = self.get_start_speed_and_init(size)
130 self.record_start_time()
134 print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '% ',end='\r')
136 # Start generating packets at requested speed (in % of a 10Gb/s link)
137 self.gen_machine.set_generator_speed(speed)
138 self.set_background_speed(self.background_machines, speed)
139 self.start_background_traffic(self.background_machines)
140 # Get statistics now that the generation is stable and initial ARP messages are dealt with
141 pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc , lat_perc_max, lat_max, abs_tx,abs_rx,abs_dropped, abs_tx_fail, drop_rate, lat_min, lat_used, r, actual_duration, avg_bg_rate, bucket_size, buckets = self.run_iteration(float(self.test['runtime']),flow_number,size,speed)
142 self.stop_background_traffic(self.background_machines)
144 retry_warning = bcolors.WARNING + ' {:1} retries needed'.format(r) + bcolors.ENDC
147 # Drop rate is expressed in percentage. lat_used is a ratio (0 to 1). The sum of these 2 should be 100%.
148 # If the sum is lower than 95, it means that more than 5% of the latency measurements where dropped for accuracy reasons.
149 if (drop_rate + lat_used * 100) < 95:
150 lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) + bcolors.ENDC
153 if self.test['test'] == 'fixed_rate':
160 endlat_perc = lat_perc
161 endlat_perc_max = lat_perc_max
164 endabs_dropped = abs_dropped
165 enddrop_rate = drop_rate
168 endavg_bg_rate = avg_bg_rate
169 if lat_warning or retry_warning:
170 endwarning = '| | {:177.177} |'.format(retry_warning + lat_warning)
172 TestPassed = False # fixed rate testing cannot be True, it is just reporting numbers every second
173 speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
174 # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold
175 # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
176 # This can be specified by putting 0 in the .test file
177 elif ((drop_rate < self.test['drop_rate_threshold']) or (abs_dropped==self.test['drop_rate_threshold']==0)) and (lat_avg< self.test['lat_avg_threshold']) and (lat_perc< self.test['lat_perc_threshold']) and (lat_max < self.test['lat_max_threshold']):
178 if (old_div((self.get_pps(speed,size) - pps_tx),self.get_pps(speed,size)))>0.01:
179 speed_prefix = bcolors.WARNING
181 gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(self.get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
183 gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(self.get_pps(speed,size), pps_tx) + bcolors.ENDC
185 speed_prefix = bcolors.ENDC
188 endspeed_prefix = speed_prefix
189 endpps_req_tx = pps_req_tx
191 endpps_sut_tx = pps_sut_tx
194 endlat_perc = lat_perc
195 endlat_perc_max = lat_perc_max
198 endabs_dropped = None
199 enddrop_rate = drop_rate
202 endavg_bg_rate = avg_bg_rate
203 if lat_warning or gen_warning or retry_warning:
204 endwarning = '| | {:186.186} |'.format(retry_warning + lat_warning + gen_warning)
206 success_message=' SUCCESS'
207 speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
208 RapidLog.debug(self.report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
210 success_message=' FAILED'
211 abs_drop_rate_prefix = bcolors.ENDC
212 if ((abs_dropped>0) and (self.test['drop_rate_threshold'] ==0)):
213 abs_drop_rate_prefix = bcolors.FAIL
214 if (drop_rate < self.test['drop_rate_threshold']):
215 drop_rate_prefix = bcolors.ENDC
217 drop_rate_prefix = bcolors.FAIL
218 if (lat_avg< self.test['lat_avg_threshold']):
219 lat_avg_prefix = bcolors.ENDC
221 lat_avg_prefix = bcolors.FAIL
222 if (lat_perc< self.test['lat_perc_threshold']):
223 lat_perc_prefix = bcolors.ENDC
225 lat_perc_prefix = bcolors.FAIL
226 if (lat_max< self.test['lat_max_threshold']):
227 lat_max_prefix = bcolors.ENDC
229 lat_max_prefix = bcolors.FAIL
230 if ((old_div((self.get_pps(speed,size) - pps_tx),self.get_pps(speed,size)))<0.001):
231 speed_prefix = bcolors.ENDC
233 speed_prefix = bcolors.FAIL
235 RapidLog.debug(self.report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning)
236 speed = self.new_speed(speed, size, success)
237 if self.test['test'] == 'increment_till_fail':
240 elif self.resolution_achieved():
242 self.record_stop_time()
243 if endspeed is not None:
244 if TestPassed and (endpps_rx < self.test['pass_threshold']):
246 speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
247 RapidLog.info(self.report_result(flow_number,size,endspeed,endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_perc,endlat_perc_max,endlat_max,endabs_tx,endabs_rx,endabs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix))
249 tot_avg_rx_rate = endpps_rx + (endavg_bg_rate * len(self.background_machines))
250 endtotaltrafficrate = '| | Total amount of traffic received by all generators during this test: {:>4.3f} Gb/s {:7.3f} Mpps {} |'.format(RapidTest.get_speed(tot_avg_rx_rate,size) , tot_avg_rx_rate, ' '*84)
251 RapidLog.info (endtotaltrafficrate)
253 RapidLog.info (endwarning)
254 RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
255 if self.test['test'] != 'fixed_rate':
256 variables = {'test': self.test['testname'],
257 'environment_file': self.test['environment_file'],
258 'start_date': self.start,
259 'stop_date': self.stop,
260 'Flows': flow_number,
262 'RequestedSpeed': RapidTest.get_pps(speed,size),
263 'CoreGenerated': endpps_req_tx,
264 'SentByNIC': endpps_tx,
265 'FwdBySUT': endpps_sut_tx,
266 'RevByCore': endpps_rx,
267 'AvgLatency': endlat_avg,
268 'PCTLatency': endlat_perc,
269 'MaxLatency': endlat_max,
270 'PacketsSent': endabs_tx,
271 'PacketsReceived': endabs_rx,
272 'PacketsLost': abs_dropped,
273 'bucket_size': bucket_size,
274 'buckets': endbuckets}
275 self.post_data('rapid_flowsizetest', variables)
277 RapidLog.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
278 self.gen_machine.stop_latency_cores()