VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py

   1 #!/usr/bin/python
   2
   3 ##
   4 ## Copyright (c) 2020 Intel Corporation
   5 ##
   6 ## Licensed under the Apache License, Version 2.0 (the "License");
   7 ## you may not use this file except in compliance with the License.
   8 ## You may obtain a copy of the License at
   9 ##
  10 ##
  11 ##     http://www.apache.org/licenses/LICENSE-2.0
  12 ##
  13 ## Unless required by applicable law or agreed to in writing, software
  14 ## distributed under the License is distributed on an "AS IS" BASIS,
  15 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 ## See the License for the specific language governing permissions and
  17 ## limitations under the License.
  18 ##
  19 import sys
  20 import time
  21 from math import ceil
  22 from statistics import mean
  23 from past.utils import old_div
  24 from rapid_log import RapidLog
  25 from rapid_log import bcolors
  26 from rapid_test import RapidTest
  27 inf = float("inf")
  28
  29 class FlowSizeTest(RapidTest):
  30     """
  31     Class to manage the flowsizetesting
  32     """
  33     def __init__(self, test_param, lat_percentile, runtime, testname,
  34             environment_file, gen_machine, sut_machine, background_machines):
  35         super().__init__(test_param, runtime, testname, environment_file)
  36         self.gen_machine = gen_machine
  37         self.sut_machine = sut_machine
  38         self.background_machines = background_machines
  39         self.test['lat_percentile'] = lat_percentile
  40         if self.test['test'] == 'TST009test':
  41             # This test implements some of the testing as defined in
  42             # https://docbox.etsi.org/ISG/NFV/open/Publications_pdf/Specs-Reports/NFV-TST%20009v3.2.1%20-%20GS%20-%20NFVI_Benchmarks.pdf
  43             self.test['TST009_n'] = int(ceil(old_div(
  44                 self.test['maxframespersecondallingress'],
  45                 self.test['stepsize'])))
  46             self.test['TST009'] = True
  47             self.test['TST009_L'] = 0
  48             self.test['TST009_R'] = self.test['TST009_n'] - 1
  49             self.test['TST009_S']= []
  50             for m in range(0, self.test['TST009_n']):
  51                 self.test['TST009_S'].append((m+1) * self.test['stepsize'])
  52         elif self.test['test'] == 'fixed_rate':
  53             for key in['drop_rate_threshold','lat_avg_threshold',
  54                     'lat_perc_threshold','lat_max_threshold']:
  55                 self.test[key] = inf
  56
  57     def new_speed(self, speed,size,success):
  58         if self.test['test'] == 'fixed_rate':
  59             return (self.test['startspeed'])
  60         elif self.test['test'] == 'increment_till_fail':
  61             return (speed + self.test['step'])
  62         elif 'TST009' in self.test.keys():
  63             if success:
  64                 self.test['TST009_L'] = self.test['TST009_m'] + 1
  65             else:
  66                 self.test['TST009_R'] = max(self.test['TST009_m'] - 1, self.test['TST009_L'])
  67             self.test['TST009_m'] = int (old_div((self.test['TST009_L'] + self.test['TST009_R']),2))
  68             return (self.get_percentageof10Gbps(self.test['TST009_S'][self.test['TST009_m']],size))
  69         else:
  70             if success:
  71                 self.test['minspeed'] = speed
  72             else:
  73                 self.test['maxspeed'] = speed
  74             return (old_div((self.test['minspeed'] + self.test['maxspeed']),2.0))
  75
  76     def get_start_speed_and_init(self, size):
  77         if self.test['test'] == 'fixed_rate':
  78             return (self.test['startspeed'])
  79         elif self.test['test'] == 'increment_till_fail':
  80             return (self.test['startspeed'])
  81         elif 'TST009' in self.test.keys():
  82             self.test['TST009_L'] = 0
  83             self.test['TST009_R'] = self.test['TST009_n'] - 1
  84             self.test['TST009_m'] = int(old_div((self.test['TST009_L'] + self.test['TST009_R']), 2))
  85             return (self.get_percentageof10Gbps(self.test['TST009_S'][self.test['TST009_m']],size))
  86         else:
  87             self.test['minspeed'] = 0
  88             self.test['maxspeed'] = self.test['startspeed']
  89             return (self.test['startspeed'])
  90
  91     def resolution_achieved(self):
  92         if self.test['test'] == 'fixed_rate':
  93             return (True)
  94         elif 'TST009' in self.test.keys():
  95             return (self.test['TST009_L'] == self.test['TST009_R'])
  96         else:
  97             return ((self.test['maxspeed'] - self.test['minspeed']) <= self.test['accuracy'])
  98
  99     def run(self):
 100         result_details = {'Details': 'Nothing'}
 101         self.gen_machine.start_latency_cores()
 102         TestPassed = True
 103         for imix in self.test['imixs']:
 104             size = mean(imix)
 105             self.gen_machine.set_udp_packet_size(imix)
 106             if self.background_machines:
 107                 backgroundinfo = '{}Running {} x background traffic not represented in the table{}'.format(bcolors.FLASH,len(self.background_machines),bcolors.ENDC)
 108             else:
 109                 backgroundinfo = '{}{}'.format(bcolors.FLASH,bcolors.ENDC)
 110             self.set_background_size(self.background_machines, imix)
 111             RapidLog.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 112             RapidLog.info('| UDP, {:>5} bytes, different number of flows by randomizing SRC & DST UDP port. {:116.116}|'.format(size, backgroundinfo))
 113             RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 114             RapidLog.info('| Flows  | Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core           | Avg. Lat.|{:.0f} Pcentil| Max. Lat.|   Sent    |  Received |    Lost   | Total Lost|L.Ratio|Time|'.format(self.test['lat_percentile']*100))
 115             RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 116             for flow_number in self.test['flows']:
 117                 attempts = 0
 118                 self.gen_machine.reset_stats()
 119                 if self.sut_machine:
 120                     self.sut_machine.reset_stats()
 121                 flow_number = self.gen_machine.set_flows(flow_number)
 122                 self.set_background_flows(self.background_machines, flow_number)
 123                 endspeed = None
 124                 speed = self.get_start_speed_and_init(size)
 125                 self.record_start_time()
 126                 while True:
 127                     attempts += 1
 128                     endwarning = False
 129                     print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 130                     sys.stdout.flush()
 131                     # Start generating packets at requested speed (in % of a 10Gb/s link)
 132                     self.gen_machine.set_generator_speed(speed)
 133                     self.set_background_speed(self.background_machines, speed)
 134                     self.start_background_traffic(self.background_machines)
 135                     # Get statistics now that the generation is stable and initial ARP messages are dealt with
 136                     pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc , lat_perc_max, lat_max, abs_tx,abs_rx,abs_dropped, abs_tx_fail, drop_rate, lat_min, lat_used, r, actual_duration, avg_bg_rate, bucket_size, buckets = self.run_iteration(float(self.test['runtime']),flow_number,size,speed)
 137                     self.stop_background_traffic(self.background_machines)
 138                     if r > 1:
 139                         retry_warning = bcolors.WARNING + ' {:1} retries needed'.format(r) +  bcolors.ENDC
 140                     else:
 141                         retry_warning = ''
 142                     # Drop rate is expressed in percentage. lat_used is a ratio (0 to 1). The sum of these 2 should be 100%.
 143                     # If the sum is lower than 95, it means that more than 5% of the latency measurements where dropped for accuracy reasons.
 144                     if (drop_rate + lat_used * 100) < 95:
 145                         lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 146                     else:
 147                         lat_warning = ''
 148                     if self.test['test'] == 'fixed_rate':
 149                         endspeed = speed
 150                         endpps_req_tx = None
 151                         endpps_tx = None
 152                         endpps_sut_tx = None
 153                         endpps_rx = None
 154                         endlat_avg = lat_avg
 155                         endlat_perc = lat_perc
 156                         endlat_perc_max = lat_perc_max
 157                         endlat_max = lat_max
 158                         endbuckets = buckets
 159                         endabs_dropped = abs_dropped
 160                         enddrop_rate = drop_rate
 161                         endabs_tx = abs_tx
 162                         endabs_rx = abs_rx
 163                         endavg_bg_rate = avg_bg_rate
 164                         if lat_warning or retry_warning:
 165                             endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning)
 166                         success = True
 167                         TestPassed = False # fixed rate testing cannot be True, it is just reporting numbers every second
 168                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 169                     # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold
 170                     # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
 171                     # This can be specified by putting 0 in the .test file
 172                     elif ((drop_rate < self.test['drop_rate_threshold']) or (abs_dropped==self.test['drop_rate_threshold']==0)) and (lat_avg< self.test['lat_avg_threshold']) and (lat_perc< self.test['lat_perc_threshold']) and (lat_max < self.test['lat_max_threshold']):
 173                         if (old_div((self.get_pps(speed,size) - pps_tx),self.get_pps(speed,size)))>0.01:
 174                             speed_prefix = bcolors.WARNING
 175                             if abs_tx_fail > 0:
 176                                 gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(self.get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
 177                             else:
 178                                 gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(self.get_pps(speed,size), pps_tx) + bcolors.ENDC
 179                         else:
 180                             speed_prefix = bcolors.ENDC
 181                             gen_warning = ''
 182                         endspeed = speed
 183                         endspeed_prefix = speed_prefix
 184                         endpps_req_tx = pps_req_tx
 185                         endpps_tx = pps_tx
 186                         endpps_sut_tx = pps_sut_tx
 187                         endpps_rx = pps_rx
 188                         endlat_avg = lat_avg
 189                         endlat_perc = lat_perc
 190                         endlat_perc_max = lat_perc_max
 191                         endlat_max = lat_max
 192                         endbuckets = buckets
 193                         endabs_dropped = None
 194                         enddrop_rate = drop_rate
 195                         endabs_tx = abs_tx
 196                         endabs_rx = abs_rx
 197                         endavg_bg_rate = avg_bg_rate
 198                         if lat_warning or gen_warning or retry_warning:
 199                             endwarning = '|        | {:186.186} |'.format(retry_warning + lat_warning + gen_warning)
 200                         success = True
 201                         success_message=' SUCCESS'
 202                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 203                         RapidLog.debug(self.report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
 204                     else:
 205                         success_message=' FAILED'
 206                         abs_drop_rate_prefix = bcolors.ENDC
 207                         if ((abs_dropped>0) and (self.test['drop_rate_threshold'] ==0)):
 208                             abs_drop_rate_prefix = bcolors.FAIL
 209                         if (drop_rate < self.test['drop_rate_threshold']):
 210                             drop_rate_prefix = bcolors.ENDC
 211                         else:
 212                             drop_rate_prefix = bcolors.FAIL
 213                         if (lat_avg< self.test['lat_avg_threshold']):
 214                             lat_avg_prefix = bcolors.ENDC
 215                         else:
 216                             lat_avg_prefix = bcolors.FAIL
 217                         if (lat_perc< self.test['lat_perc_threshold']):
 218                             lat_perc_prefix = bcolors.ENDC
 219                         else:
 220                             lat_perc_prefix = bcolors.FAIL
 221                         if (lat_max< self.test['lat_max_threshold']):
 222                             lat_max_prefix = bcolors.ENDC
 223                         else:
 224                             lat_max_prefix = bcolors.FAIL
 225                         if ((old_div((self.get_pps(speed,size) - pps_tx),self.get_pps(speed,size)))<0.001):
 226                             speed_prefix = bcolors.ENDC
 227                         else:
 228                             speed_prefix = bcolors.FAIL
 229                         success = False
 230                         RapidLog.debug(self.report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning)
 231                     speed = self.new_speed(speed, size, success)
 232                     if self.test['test'] == 'increment_till_fail':
 233                         if not success:
 234                             break
 235                     elif self.resolution_achieved():
 236                         break
 237                 self.record_stop_time()
 238                 if endspeed is not None:
 239                     if TestPassed and (endpps_rx < self.test['pass_threshold']):
 240                         TestPassed = False
 241                     speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 242                     RapidLog.info(self.report_result(flow_number,size,endspeed,endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_perc,endlat_perc_max,endlat_max,endabs_tx,endabs_rx,endabs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix))
 243                     if endavg_bg_rate:
 244                         tot_avg_rx_rate = endpps_rx + (endavg_bg_rate * len(self.background_machines))
 245                         endtotaltrafficrate = '|        | Total amount of traffic received by all generators during this test: {:>4.3f} Gb/s {:7.3f} Mpps {} |'.format(RapidTest.get_speed(tot_avg_rx_rate,size) , tot_avg_rx_rate, ' '*84)
 246                         RapidLog.info (endtotaltrafficrate)
 247                     if endwarning:
 248                         RapidLog.info (endwarning)
 249                     RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 250                     if self.test['test'] != 'fixed_rate':
 251                         result_details = {'test': self.test['testname'],
 252                                 'environment_file': self.test['environment_file'],
 253                                 'start_date': self.start,
 254                                 'stop_date': self.stop,
 255                                 'Flows': flow_number,
 256                                 'Size': size,
 257                                 'RequestedSpeed': RapidTest.get_pps(endspeed,size),
 258                                 'CoreGenerated': endpps_req_tx,
 259                                 'SentByNIC': endpps_tx,
 260                                 'FwdBySUT': endpps_sut_tx,
 261                                 'RevByCore': endpps_rx,
 262                                 'AvgLatency': endlat_avg,
 263                                 'PCTLatency': endlat_perc,
 264                                 'MaxLatency': endlat_max,
 265                                 'PacketsSent': endabs_tx,
 266                                 'PacketsReceived': endabs_rx,
 267                                 'PacketsLost': endabs_dropped,
 268                                 'bucket_size': bucket_size,
 269                                 'buckets': endbuckets}
 270                         result_details = self.post_data('rapid_flowsizetest', result_details)
 271                 else:
 272                     RapidLog.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
 273         self.gen_machine.stop_latency_cores()
 274         return (TestPassed, result_details)