VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py

   1 #!/usr/bin/python
   2
   3 ##
   4 ## Copyright (c) 2020 Intel Corporation
   5 ##
   6 ## Licensed under the Apache License, Version 2.0 (the "License");
   7 ## you may not use this file except in compliance with the License.
   8 ## You may obtain a copy of the License at
   9 ##
  10 ##
  11 ##     http://www.apache.org/licenses/LICENSE-2.0
  12 ##
  13 ## Unless required by applicable law or agreed to in writing, software
  14 ## distributed under the License is distributed on an "AS IS" BASIS,
  15 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 ## See the License for the specific language governing permissions and
  17 ## limitations under the License.
  18 ##
  19 import sys
  20 import time
  21 from math import ceil
  22 from statistics import mean
  23 from past.utils import old_div
  24 from rapid_log import RapidLog
  25 from rapid_log import bcolors
  26 from rapid_test import RapidTest
  27 inf = float("inf")
  28
  29 class FlowSizeTest(RapidTest):
  30     """
  31     Class to manage the flowsizetesting
  32     """
  33     def __init__(self, test_param, lat_percentile, runtime, testname,
  34             environment_file, gen_machine, sut_machine, background_machines):
  35         super().__init__(test_param, runtime, testname, environment_file)
  36         self.gen_machine = gen_machine
  37         self.sut_machine = sut_machine
  38         self.background_machines = background_machines
  39         self.test['lat_percentile'] = lat_percentile
  40         if self.test['test'] == 'TST009test':
  41             # This test implements some of the testing as defined in
  42             # https://docbox.etsi.org/ISG/NFV/open/Publications_pdf/Specs-Reports/NFV-TST%20009v3.2.1%20-%20GS%20-%20NFVI_Benchmarks.pdf
  43             self.test['TST009_n'] = int(ceil(old_div(
  44                 self.test['maxframespersecondallingress'],
  45                 self.test['stepsize'])))
  46             self.test['TST009'] = True
  47             self.test['TST009_L'] = 0
  48             self.test['TST009_R'] = self.test['TST009_n'] - 1
  49             self.test['TST009_S']= []
  50             for m in range(0, self.test['TST009_n']):
  51                 self.test['TST009_S'].append((m+1) * self.test['stepsize'])
  52             self.test['lat_avg_threshold'] = inf
  53             self.test['lat_perc_threshold'] = inf
  54             self.test['lat_max_threshold'] = inf
  55         elif self.test['test'] == 'fixed_rate':
  56             for key in['drop_rate_threshold','lat_avg_threshold',
  57                     'lat_perc_threshold','lat_max_threshold']:
  58                 self.test[key] = inf
  59
  60     def new_speed(self, speed,size,success):
  61         if self.test['test'] == 'fixed_rate':
  62             return (self.test['startspeed'])
  63         elif self.test['test'] == 'increment_till_fail':
  64             return (speed + self.test['step'])
  65         elif 'TST009' in self.test.keys():
  66             if success:
  67                 self.test['TST009_L'] = self.test['TST009_m'] + 1
  68             else:
  69                 self.test['TST009_R'] = max(self.test['TST009_m'] - 1, self.test['TST009_L'])
  70             self.test['TST009_m'] = int (old_div((self.test['TST009_L'] + self.test['TST009_R']),2))
  71             return (self.get_percentageof10Gbps(self.test['TST009_S'][self.test['TST009_m']],size))
  72         else:
  73             if success:
  74                 self.test['minspeed'] = speed
  75             else:
  76                 self.test['maxspeed'] = speed
  77             return (old_div((self.test['minspeed'] + self.test['maxspeed']),2.0))
  78
  79     def get_start_speed_and_init(self, size):
  80         if self.test['test'] == 'fixed_rate':
  81             return (self.test['startspeed'])
  82         elif self.test['test'] == 'increment_till_fail':
  83             return (self.test['startspeed'])
  84         elif 'TST009' in self.test.keys():
  85             self.test['TST009_L'] = 0
  86             self.test['TST009_R'] = self.test['TST009_n'] - 1
  87             self.test['TST009_m'] = int(old_div((self.test['TST009_L'] + self.test['TST009_R']), 2))
  88             return (self.get_percentageof10Gbps(self.test['TST009_S'][self.test['TST009_m']],size))
  89         else:
  90             self.test['minspeed'] = 0
  91             self.test['maxspeed'] = self.test['startspeed']
  92             return (self.test['startspeed'])
  93
  94     def resolution_achieved(self):
  95         if self.test['test'] == 'fixed_rate':
  96             return (True)
  97         elif 'TST009' in self.test.keys():
  98             return (self.test['TST009_L'] == self.test['TST009_R'])
  99         else:
 100             return ((self.test['maxspeed'] - self.test['minspeed']) <= self.test['accuracy'])
 101
 102     def run(self):
 103     #    global fieldnames
 104     #    global writer
 105     #    #fieldnames = ['Flows','PacketSize','Gbps','Mpps','AvgLatency','MaxLatency','PacketsDropped','PacketDropRate']
 106     #    fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Sent','Received','Lost','LostTotal']
 107     #    writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 108     #    writer.writeheader()
 109         self.gen_machine.start_latency_cores()
 110         TestPassed = True
 111         for imix in self.test['imixs']:
 112             size = mean(imix)
 113             self.gen_machine.set_udp_packet_size(imix)
 114             if self.background_machines:
 115                 backgroundinfo = '{}Running {} x background traffic not represented in the table{}'.format(bcolors.FLASH,len(self.background_machines),bcolors.ENDC)
 116             else:
 117                 backgroundinfo = '{}{}'.format(bcolors.FLASH,bcolors.ENDC)
 118             self.set_background_size(self.background_machines, imix)
 119             RapidLog.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 120             RapidLog.info('| UDP, {:>5} bytes, different number of flows by randomizing SRC & DST UDP port. {:116.116}|'.format(size, backgroundinfo))
 121             RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 122             RapidLog.info('| Flows  | Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core           | Avg. Lat.|{:.0f} Pcentil| Max. Lat.|   Sent    |  Received |    Lost   | Total Lost|L.Ratio|Time|'.format(self.test['lat_percentile']*100))
 123             RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 124             for flow_number in self.test['flows']:
 125                 attempts = 0
 126                 self.gen_machine.reset_stats()
 127                 if self.sut_machine:
 128                     self.sut_machine.reset_stats()
 129                 flow_number = self.gen_machine.set_flows(flow_number)
 130                 self.set_background_flows(self.background_machines, flow_number)
 131                 endspeed = None
 132                 speed = self.get_start_speed_and_init(size)
 133                 self.record_start_time()
 134                 while True:
 135                     attempts += 1
 136                     endwarning = False
 137                     print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 138                     sys.stdout.flush()
 139                     # Start generating packets at requested speed (in % of a 10Gb/s link)
 140                     self.gen_machine.set_generator_speed(speed)
 141                     self.set_background_speed(self.background_machines, speed)
 142                     self.start_background_traffic(self.background_machines)
 143                     # Get statistics now that the generation is stable and initial ARP messages are dealt with
 144                     pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc , lat_perc_max, lat_max, abs_tx,abs_rx,abs_dropped, abs_tx_fail, drop_rate, lat_min, lat_used, r, actual_duration = self.run_iteration(float(self.test['runtime']),flow_number,size,speed)
 145                     self.stop_background_traffic(self.background_machines)
 146                     if r > 1:
 147                         retry_warning = bcolors.WARNING + ' {:1} retries needed'.format(r) +  bcolors.ENDC
 148                     else:
 149                         retry_warning = ''
 150                     # Drop rate is expressed in percentage. lat_used is a ratio (0 to 1). The sum of these 2 should be 100%.
 151                     # If the sum is lower than 95, it means that more than 5% of the latency measurements where dropped for accuracy reasons.
 152                     if (drop_rate + lat_used * 100) < 95:
 153                         lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 154                     else:
 155                         lat_warning = ''
 156                     if self.test['test'] == 'fixed_rate':
 157                         endspeed = speed
 158                         endpps_req_tx = None
 159                         endpps_tx = None
 160                         endpps_sut_tx = None
 161                         endpps_rx = None
 162                         endlat_avg = lat_avg
 163                         endlat_perc = lat_perc
 164                         endlat_perc_max = lat_perc_max
 165                         endlat_max = lat_max
 166                         endabs_dropped = abs_dropped
 167                         enddrop_rate = drop_rate
 168                         endabs_tx = abs_tx
 169                         endabs_rx = abs_rx
 170                         if lat_warning or retry_warning:
 171                             endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning)
 172                         success = True
 173                         TestPassed = False # fixed rate testing cannot be True, it is just reporting numbers every second
 174                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 175                     # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold
 176                     # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
 177                     # This can be specified by putting 0 in the .test file
 178                     elif ((drop_rate < self.test['drop_rate_threshold']) or (abs_dropped==self.test['drop_rate_threshold']==0)) and (lat_avg< self.test['lat_avg_threshold']) and (lat_perc< self.test['lat_perc_threshold']) and (lat_max < self.test['lat_max_threshold']):
 179                         if (old_div((self.get_pps(speed,size) - pps_tx),self.get_pps(speed,size)))>0.01:
 180                             speed_prefix = bcolors.WARNING
 181                             if abs_tx_fail > 0:
 182                                 gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(self.get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
 183                             else:
 184                                 gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(self.get_pps(speed,size), pps_tx) + bcolors.ENDC
 185                         else:
 186                             speed_prefix = bcolors.ENDC
 187                             gen_warning = ''
 188                         endspeed = speed
 189                         endspeed_prefix = speed_prefix
 190                         endpps_req_tx = pps_req_tx
 191                         endpps_tx = pps_tx
 192                         endpps_sut_tx = pps_sut_tx
 193                         endpps_rx = pps_rx
 194                         endlat_avg = lat_avg
 195                         endlat_perc = lat_perc
 196                         endlat_perc_max = lat_perc_max
 197                         endlat_max = lat_max
 198                         endabs_dropped = None
 199                         enddrop_rate = drop_rate
 200                         endabs_tx = abs_tx
 201                         endabs_rx = abs_rx
 202                         if lat_warning or gen_warning or retry_warning:
 203                             endwarning = '|        | {:186.186} |'.format(retry_warning + lat_warning + gen_warning)
 204                         success = True
 205                         success_message=' SUCCESS'
 206                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 207                         RapidLog.debug(self.report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
 208                     else:
 209                         success_message=' FAILED'
 210                         abs_drop_rate_prefix = bcolors.ENDC
 211                         if ((abs_dropped>0) and (self.test['drop_rate_threshold'] ==0)):
 212                             abs_drop_rate_prefix = bcolors.FAIL
 213                         if (drop_rate < self.test['drop_rate_threshold']):
 214                             drop_rate_prefix = bcolors.ENDC
 215                         else:
 216                             drop_rate_prefix = bcolors.FAIL
 217                         if (lat_avg< self.test['lat_avg_threshold']):
 218                             lat_avg_prefix = bcolors.ENDC
 219                         else:
 220                             lat_avg_prefix = bcolors.FAIL
 221                         if (lat_perc< self.test['lat_perc_threshold']):
 222                             lat_perc_prefix = bcolors.ENDC
 223                         else:
 224                             lat_perc_prefix = bcolors.FAIL
 225                         if (lat_max< self.test['lat_max_threshold']):
 226                             lat_max_prefix = bcolors.ENDC
 227                         else:
 228                             lat_max_prefix = bcolors.FAIL
 229                         if ((old_div((self.get_pps(speed,size) - pps_tx),self.get_pps(speed,size)))<0.001):
 230                             speed_prefix = bcolors.ENDC
 231                         else:
 232                             speed_prefix = bcolors.FAIL
 233                         success = False
 234                         RapidLog.debug(self.report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning)
 235                     speed = self.new_speed(speed, size, success)
 236                     if self.test['test'] == 'increment_till_fail':
 237                         if not success:
 238                             break
 239                     elif self.resolution_achieved():
 240                         break
 241                 self.record_stop_time()
 242                 if endspeed is not None:
 243                     if TestPassed and (endpps_rx < self.test['pass_threshold']):
 244                         TestPassed = False
 245                     speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 246                     RapidLog.info(self.report_result(flow_number,size,endspeed,endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_perc,endlat_perc_max,endlat_max,endabs_tx,endabs_rx,endabs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix))
 247                     if endwarning:
 248                         RapidLog.info (endwarning)
 249                     RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 250                     if self.test['test'] != 'fixed_rate':
 251                         variables = {'test': self.test['testname'],
 252                                 'environment_file': self.test['environment_file'],
 253                                 'start_date': self.start,
 254                                 'stop_date': self.stop,
 255                                 'Flows': flow_number,
 256                                 'Size': size,
 257                                 'RequestedSpeed': RapidTest.get_pps(speed,size),
 258                                 'CoreGenerated': endpps_req_tx,
 259                                 'SentByNIC': endpps_tx,
 260                                 'FwdBySUT': endpps_sut_tx,
 261                                 'RevByCore': endpps_rx,
 262                                 'AvgLatency': endlat_avg,
 263                                 'PCTLatency': endlat_perc,
 264                                 'MaxLatency': endlat_max,
 265                                 'PacketsSent': endabs_tx,
 266                                 'PacketsReceived': endabs_rx,
 267                                 'PacketsLost': abs_dropped}
 268                         self.post_data('rapid_flowsizetest', variables)
 269                 else:
 270                     RapidLog.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
 271         self.gen_machine.stop_latency_cores()
 272         return (TestPassed)
 273