VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py

   1 #!/usr/bin/python
   2
   3 ##
   4 ## Copyright (c) 2020 Intel Corporation
   5 ##
   6 ## Licensed under the Apache License, Version 2.0 (the "License");
   7 ## you may not use this file except in compliance with the License.
   8 ## You may obtain a copy of the License at
   9 ##
  10 ##
  11 ##     http://www.apache.org/licenses/LICENSE-2.0
  12 ##
  13 ## Unless required by applicable law or agreed to in writing, software
  14 ## distributed under the License is distributed on an "AS IS" BASIS,
  15 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 ## See the License for the specific language governing permissions and
  17 ## limitations under the License.
  18 ##
  19 import sys
  20 import time
  21 import copy
  22 from math import ceil
  23 from statistics import mean
  24 from past.utils import old_div
  25 from rapid_log import RapidLog
  26 from rapid_log import bcolors
  27 from rapid_test import RapidTest
  28 inf = float("inf")
  29
  30 class FlowSizeTest(RapidTest):
  31     """
  32     Class to manage the flowsizetesting
  33     """
  34     def __init__(self, test_param, lat_percentile, runtime, testname,
  35             environment_file, gen_machine, sut_machine, background_machines, sleep_time):
  36         super().__init__(test_param, runtime, testname, environment_file)
  37         self.gen_machine = gen_machine
  38         self.sut_machine = sut_machine
  39         self.background_machines = background_machines
  40         self.test['lat_percentile'] = lat_percentile
  41         self.test['sleep_time'] = sleep_time
  42         if self.test['test'] == 'TST009test':
  43             # This test implements some of the testing as defined in
  44             # https://docbox.etsi.org/ISG/NFV/open/Publications_pdf/Specs-Reports/NFV-TST%20009v3.2.1%20-%20GS%20-%20NFVI_Benchmarks.pdf
  45             self.test['TST009_n'] = int(ceil(old_div(
  46                 self.test['maxframespersecondallingress'],
  47                 self.test['stepsize'])))
  48             self.test['TST009'] = True
  49             self.test['TST009_L'] = 0
  50             self.test['TST009_R'] = self.test['TST009_n'] - 1
  51             self.test['TST009_S']= []
  52             for m in range(0, self.test['TST009_n']):
  53                 self.test['TST009_S'].append((m+1) * self.test['stepsize'])
  54         elif self.test['test'] == 'fixed_rate':
  55             for key in['drop_rate_threshold','lat_avg_threshold',
  56                     'lat_perc_threshold','lat_max_threshold','mis_ordered_threshold']:
  57                 self.test[key] = inf
  58
  59     def new_speed(self, speed,size,success):
  60         if self.test['test'] == 'fixed_rate':
  61             return (self.test['startspeed'])
  62         elif self.test['test'] == 'increment_till_fail':
  63             return (speed + self.test['step'])
  64         elif 'TST009' in self.test.keys():
  65             if success:
  66                 self.test['TST009_L'] = self.test['TST009_m'] + 1
  67             else:
  68                 self.test['TST009_R'] = max(self.test['TST009_m'] - 1,
  69                         self.test['TST009_L'])
  70             self.test['TST009_m'] = int (old_div((self.test['TST009_L'] +
  71                 self.test['TST009_R']),2))
  72             return (self.get_percentageof10Gbps(self.test['TST009_S'][self.test['TST009_m']],size))
  73         else:
  74             if success:
  75                 self.test['minspeed'] = speed
  76             else:
  77                 self.test['maxspeed'] = speed
  78             return (old_div((self.test['minspeed'] + self.test['maxspeed']),2.0))
  79
  80     def get_start_speed_and_init(self, size):
  81         if self.test['test'] == 'fixed_rate':
  82             return (self.test['startspeed'])
  83         elif self.test['test'] == 'increment_till_fail':
  84             return (self.test['startspeed'])
  85         elif 'TST009' in self.test.keys():
  86             self.test['TST009_L'] = 0
  87             self.test['TST009_R'] = self.test['TST009_n'] - 1
  88             self.test['TST009_m'] = int(old_div((self.test['TST009_L'] +
  89                 self.test['TST009_R']), 2))
  90             return (self.get_percentageof10Gbps(self.test['TST009_S'][self.test['TST009_m']],size))
  91         else:
  92             self.test['minspeed'] = 0
  93             self.test['maxspeed'] = self.test['startspeed']
  94             return (self.test['startspeed'])
  95
  96     def resolution_achieved(self):
  97         if self.test['test'] == 'fixed_rate':
  98             return (True)
  99         elif 'TST009' in self.test.keys():
 100             return (self.test['TST009_L'] == self.test['TST009_R'])
 101         else:
 102             return ((self.test['maxspeed'] - self.test['minspeed']) <= self.test['accuracy'])
 103
 104     def warm_up(self):
 105         # Running at low speed to make sure the ARP messages can get through.
 106         # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
 107         # Note however that if we would run the test steps during a very long time, the ARP would expire in the switch.
 108         # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
 109         imix = self.test['warmupimix']
 110         FLOWSIZE = self.test['warmupflowsize']
 111         WARMUPSPEED = self.test['warmupspeed']
 112         WARMUPTIME = self.test['warmuptime']
 113
 114         if WARMUPTIME == 0:
 115             RapidLog.info(("Not Warming up"))
 116             return
 117
 118         RapidLog.info(("Warming up during {} seconds..., packet size = {},"
 119             " flows = {}, speed = {}").format(WARMUPTIME, imix, FLOWSIZE,
 120                 WARMUPSPEED))
 121         self.gen_machine.set_generator_speed(WARMUPSPEED)
 122         self.set_background_speed(self.background_machines, WARMUPSPEED)
 123         self.gen_machine.set_udp_packet_size(imix)
 124         self.set_background_size(self.background_machines, imix)
 125         if FLOWSIZE:
 126             _ = self.gen_machine.set_flows(FLOWSIZE)
 127             self.set_background_flows(self.background_machines, FLOWSIZE)
 128         self.gen_machine.start()
 129         self.start_background_traffic(self.background_machines)
 130         time.sleep(WARMUPTIME)
 131         self.stop_background_traffic(self.background_machines)
 132         self.gen_machine.stop()
 133
 134     def run(self):
 135         result_details = {'Details': 'Nothing'}
 136         TestResult = 0
 137         end_data = {}
 138         iteration_prefix = {}
 139         self.warm_up()
 140         for imix in self.test['imixs']:
 141             size = mean(imix)
 142             self.gen_machine.set_udp_packet_size(imix)
 143             if self.background_machines:
 144                 backgroundinfo = ('{}Running {} x background traffic not '
 145                     'represented in the table{}').format(bcolors.FLASH,
 146                             len(self.background_machines),bcolors.ENDC)
 147             else:
 148                 backgroundinfo = '{}{}'.format(bcolors.FLASH,bcolors.ENDC)
 149             self.set_background_size(self.background_machines, imix)
 150             RapidLog.info('+' + '-' * 200 + '+')
 151             RapidLog.info(("| UDP, {:>5} bytes, different number of flows by "
 152                 "randomizing SRC & DST UDP port. {:128.128}|").
 153                 format(round(size), backgroundinfo))
 154             RapidLog.info('+' + '-' * 8 + '+' + '-' * 18 + '+' + '-' * 13 +
 155                     '+' + '-' * 13 + '+' + '-' * 13 + '+' + '-' * 24 + '+' +
 156                     '-' * 10 + '+' + '-' * 10 + '+' + '-' * 10 + '+' + '-' * 11
 157                     + '+' + '-' * 11 + '+' + '-' * 11 + '+'  + '-' * 11 +  '+'
 158                     + '-' * 7 + '+' + '-' * 11 + '+' + '-' * 4 + '+')
 159             RapidLog.info(('| Flows  | Speed requested  | Gen by core | Sent by'
 160                 ' NIC | Fwrd by SUT | Rec. by core           | Avg. Lat.|{:.0f}'
 161                 ' Pcentil| Max. Lat.|   Sent    |  Received |    Lost   | Total'
 162                 ' Lost|L.Ratio|Mis-ordered|Time').format(self.test['lat_percentile']*100))
 163             RapidLog.info('+' + '-' * 8 + '+' + '-' * 18 + '+' + '-' * 13 +
 164                     '+' + '-' * 13 + '+' + '-' * 13 + '+' + '-' * 24 + '+' +
 165                     '-' * 10 + '+' + '-' * 10 + '+' + '-' * 10 + '+' + '-' * 11
 166                     + '+' + '-' * 11 + '+' + '-' * 11 + '+'  + '-' * 11 +  '+'
 167                     + '-' * 7 + '+' + '-' * 11 + '+' + '-' * 4 + '+')
 168             for flow_number in self.test['flows']:
 169                 attempts = 0
 170                 self.gen_machine.reset_stats()
 171                 if self.sut_machine:
 172                     self.sut_machine.reset_stats()
 173                 if flow_number != 0:
 174                     flow_number = self.gen_machine.set_flows(flow_number)
 175                     self.set_background_flows(self.background_machines, flow_number)
 176                 end_data['speed'] = None
 177                 speed = self.get_start_speed_and_init(size)
 178                 while True:
 179                     attempts += 1
 180                     endwarning = False
 181                     print('{} flows: Measurement ongoing at speed: {}%'.format(
 182                         str(flow_number), str(round(speed, 2))), end='     \r')
 183                     sys.stdout.flush()
 184                     iteration_data = self.run_iteration(
 185                             float(self.test['runtime']),flow_number,size,speed)
 186                     if iteration_data['r'] > 1:
 187                         retry_warning = '{} {:1} retries needed{}'.format(
 188                                 bcolors.WARNING, iteration_data['r'],
 189                                 bcolors.ENDC)
 190                     else:
 191                         retry_warning = ''
 192                     # Drop rate is expressed in percentage. lat_used is a ratio
 193                     # (0 to 1). The sum of these 2 should be 100%.
 194                     # If the sum is lower than 95, it means that more than 5%
 195                     # of the latency measurements where dropped for accuracy
 196                     # reasons.
 197                     if (iteration_data['drop_rate'] +
 198                             iteration_data['lat_used'] * 100) < 95:
 199                         lat_warning = ('{} Latency accuracy issue?: {:>3.0f}%'
 200                             '{}').format(bcolors.WARNING,
 201                                     iteration_data['lat_used'] * 100,
 202                                     bcolors.ENDC)
 203                     else:
 204                         lat_warning = ''
 205                     iteration_prefix = {'speed' : bcolors.ENDC,
 206                             'lat_avg' : bcolors.ENDC,
 207                             'lat_perc' : bcolors.ENDC,
 208                             'lat_max' : bcolors.ENDC,
 209                             'abs_drop_rate' : bcolors.ENDC,
 210                             'mis_ordered' : bcolors.ENDC,
 211                             'drop_rate' : bcolors.ENDC}
 212                     if self.test['test'] == 'fixed_rate':
 213                         end_data = copy.deepcopy(iteration_data)
 214                         end_prefix = copy.deepcopy(iteration_prefix)
 215                         if lat_warning or retry_warning:
 216                             endwarning = '|        | {:177.177} |'.format(
 217                                     retry_warning + lat_warning)
 218                         success = True
 219                         # TestResult = TestResult + iteration_data['pps_rx']
 220                         # fixed rate testing result is strange: we just report
 221                         # the pps received
 222                     # The following if statement is testing if we pass the
 223                     # success criteria of a certain drop rate, average latency
 224                     # and maximum latency below the threshold.
 225                     # The drop rate success can be achieved in 2 ways: either
 226                     # the drop rate is below a treshold, either we want that no
 227                     # packet has been lost during the test.
 228                     # This can be specified by putting 0 in the .test file
 229                     elif ((iteration_data['drop_rate'] < self.test['drop_rate_threshold']) or (iteration_data['abs_dropped']==self.test['drop_rate_threshold']==0)) and (iteration_data['lat_avg']< self.test['lat_avg_threshold']) and (iteration_data['lat_perc']< self.test['lat_perc_threshold']) and (iteration_data['lat_max'] < self.test['lat_max_threshold'] and iteration_data['mis_ordered'] <= self.test['mis_ordered_threshold']):
 230                         if (old_div((self.get_pps(speed,size) - iteration_data['pps_tx']),self.get_pps(speed,size)))>0.01:
 231                             iteration_prefix['speed'] = bcolors.WARNING
 232                             if iteration_data['abs_tx_fail'] > 0:
 233                                 gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(self.get_pps(speed,size), iteration_data['pps_tx'], iteration_data['abs_tx_fail']) + bcolors.ENDC
 234                             else:
 235                                 gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(self.get_pps(speed,size), iteration_data['pps_tx']) + bcolors.ENDC
 236                         else:
 237                             iteration_prefix['speed'] = bcolors.ENDC
 238                             gen_warning = ''
 239                         end_data = copy.deepcopy(iteration_data)
 240                         end_prefix = copy.deepcopy(iteration_prefix)
 241                         if lat_warning or gen_warning or retry_warning:
 242                             endwarning = '|        | {:186.186} |'.format(retry_warning + lat_warning + gen_warning)
 243                         success = True
 244                         success_message=' SUCCESS'
 245                         RapidLog.debug(self.report_result(-attempts, size,
 246                             iteration_data, iteration_prefix) + success_message +
 247                             retry_warning + lat_warning + gen_warning)
 248                     else:
 249                         success_message=' FAILED'
 250                         if ((iteration_data['abs_dropped']>0) and (self.test['drop_rate_threshold'] ==0)):
 251                             iteration_prefix['abs_drop_rate'] = bcolors.FAIL
 252                         if (iteration_data['drop_rate'] <= self.test['drop_rate_threshold']):
 253                             iteration_prefix['drop_rate'] = bcolors.ENDC
 254                         else:
 255                             iteration_prefix['drop_rate'] = bcolors.FAIL
 256                         if (iteration_data['lat_avg']< self.test['lat_avg_threshold']):
 257                             iteration_prefix['lat_avg'] = bcolors.ENDC
 258                         else:
 259                             iteration_prefix['lat_avg'] = bcolors.FAIL
 260                         if (iteration_data['lat_perc']< self.test['lat_perc_threshold']):
 261                             iteration_prefix['lat_perc'] = bcolors.ENDC
 262                         else:
 263                             iteration_prefix['lat_perc'] = bcolors.FAIL
 264                         if (iteration_data['lat_max']< self.test['lat_max_threshold']):
 265                             iteration_prefix['lat_max'] = bcolors.ENDC
 266                         else:
 267                             iteration_prefix['lat_max'] = bcolors.FAIL
 268                         if ((old_div((self.get_pps(speed,size) - iteration_data['pps_tx']),self.get_pps(speed,size)))<0.001):
 269                             iteration_prefix['speed'] = bcolors.ENDC
 270                         else:
 271                             iteration_prefix['speed'] = bcolors.FAIL
 272                         if (iteration_data['mis_ordered']< self.test['mis_ordered_threshold']):
 273                             iteration_prefix['mis_ordered'] = bcolors.ENDC
 274                         else:
 275                             iteration_prefix['mis_ordered'] = bcolors.FAIL
 276
 277                         success = False
 278                         RapidLog.debug(self.report_result(-attempts, size,
 279                             iteration_data, iteration_prefix) +
 280                             success_message + retry_warning + lat_warning)
 281                     speed = self.new_speed(speed, size, success)
 282                     if self.test['test'] == 'increment_till_fail':
 283                         if not success:
 284                             break
 285                     elif self.resolution_achieved():
 286                         break
 287                 if end_data['speed'] is None:
 288                     end_data = iteration_data
 289                     end_prefix = iteration_prefix
 290                     RapidLog.info('|{:>7} | {:<177} |'.format("FAILED","Speed 0 or close to 0, data for last failed step below:"))
 291                 RapidLog.info(self.report_result(flow_number, size,
 292                     end_data, end_prefix))
 293                 if end_data['avg_bg_rate']:
 294                     tot_avg_rx_rate = end_data['pps_rx'] + (end_data['avg_bg_rate'] * len(self.background_machines))
 295                     endtotaltrafficrate = '|        | Total amount of traffic received by all generators during this test: {:>4.3f} Gb/s {:7.3f} Mpps {} |'.format(RapidTest.get_speed(tot_avg_rx_rate,size) , tot_avg_rx_rate, ' '*84)
 296                     RapidLog.info (endtotaltrafficrate)
 297                 if endwarning:
 298                     RapidLog.info (endwarning)
 299                 if self.test['test'] != 'fixed_rate':
 300                     TestResult = TestResult + end_data['pps_rx']
 301                     end_data['test'] = self.test['testname']
 302                     end_data['environment_file'] = self.test['environment_file']
 303                     end_data['Flows'] = flow_number
 304                     end_data['Size'] = size
 305                     end_data['RequestedSpeed'] = RapidTest.get_pps(end_data['speed'] ,size)
 306                     result_details = self.post_data(end_data)
 307                     RapidLog.debug(result_details)
 308                 RapidLog.info('+' + '-' * 8 + '+' + '-' * 18 + '+' + '-' * 13 +
 309                     '+' + '-' * 13 + '+' + '-' * 13 + '+' + '-' * 24 + '+' +
 310                     '-' * 10 + '+' + '-' * 10 + '+' + '-' * 10 + '+' + '-' * 11
 311                     + '+' + '-' * 11 + '+' + '-' * 11 + '+'  + '-' * 11 +  '+'
 312                     + '+' + '-' * 11 + '+'
 313                     + '-' * 7 + '+' + '-' * 4 + '+')
 314         return (TestResult, result_details)