VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py

   1 #!/usr/bin/python
   2
   3 ##
   4 ## Copyright (c) 2010-2020 Intel Corporation
   5 ##
   6 ## Licensed under the Apache License, Version 2.0 (the "License");
   7 ## you may not use this file except in compliance with the License.
   8 ## You may obtain a copy of the License at
   9 ##
  10 ##
  11 ##     http://www.apache.org/licenses/LICENSE-2.0
  12 ##
  13 ## Unless required by applicable law or agreed to in writing, software
  14 ## distributed under the License is distributed on an "AS IS" BASIS,
  15 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 ## See the License for the specific language governing permissions and
  17 ## limitations under the License.
  18 ##
  19
  20 import sys
  21 import time
  22 from math import ceil
  23 from statistics import mean
  24 from past.utils import old_div
  25 from rapid_log import RapidLog
  26 from rapid_log import bcolors
  27 from rapid_test import RapidTest
  28 inf = float("inf")
  29
  30 class FlowSizeTest(RapidTest):
  31     """
  32     Class to manage the flowsizetesting
  33     """
  34     def __init__(self, test_param, lat_percentile, runtime, pushgateway,
  35             environment_file, gen_machine, sut_machine, background_machines):
  36         self.test = test_param
  37         self.gen_machine = gen_machine
  38         self.sut_machine = sut_machine
  39         self.background_machines = background_machines
  40         self.test['lat_percentile'] = lat_percentile
  41         self.test['runtime'] = runtime
  42         self.test['pushgateway'] = pushgateway
  43         self.test['environment_file'] = environment_file
  44         if 'maxr' not in self.test.keys():
  45             self.test['maxr'] = 1
  46         if 'maxz' not in self.test.keys():
  47             self.test['maxz'] = inf
  48         if self.test['test'] == 'TST009test':
  49             # This test implements some of the testing as defined in https://docbox.etsi.org/ISG/NFV/open/Publications_pdf/Specs-Reports/NFV-TST%20009v3.2.1%20-%20GS%20-%20NFVI_Benchmarks.pdf
  50             self.test['TST009_n'] = int(ceil(old_div(self.test['maxframespersecondallingress'], self.test['stepsize'])))
  51             self.test['TST009'] = True
  52             self.test['TST009_L'] = 0
  53             self.test['TST009_R'] = self.test['TST009_n'] - 1
  54             self.test['TST009_S']= []
  55             for m in range(0, self.test['TST009_n']):
  56                 self.test['TST009_S'].append((m+1) * self.test['stepsize'])
  57             self.test['lat_avg_threshold'] = inf
  58             self.test['lat_perc_threshold'] = inf
  59             self.test['lat_max_threshold'] = inf
  60         elif self.test['test'] == 'fixed_rate':
  61             for key in['drop_rate_threshold','lat_avg_threshold','lat_perc_threshold','lat_max_threshold']:
  62                 self.test[key] = inf
  63
  64     def new_speed(self, speed,size,success):
  65         if self.test['test'] == 'fixed_rate':
  66             return (self.test['startspeed'])
  67         elif 'TST009' in self.test.keys():
  68             if success:
  69                 self.test['TST009_L'] = self.test['TST009_m'] + 1
  70             else:
  71                 self.test['TST009_R'] = max(self.test['TST009_m'] - 1, self.test['TST009_L'])
  72             self.test['TST009_m'] = int (old_div((self.test['TST009_L'] + self.test['TST009_R']),2))
  73             return (self.get_percentageof10Gbps(self.test['TST009_S'][self.test['TST009_m']],size))
  74         else:
  75             if success:
  76                 self.test['minspeed'] = speed
  77             else:
  78                 self.test['maxspeed'] = speed
  79             return (old_div((self.test['minspeed'] + self.test['maxspeed']),2.0))
  80
  81     def get_start_speed_and_init(self, size):
  82         if self.test['test'] == 'fixed_rate':
  83             return (self.test['startspeed'])
  84         elif 'TST009' in self.test.keys():
  85             self.test['TST009_L'] = 0
  86             self.test['TST009_R'] = self.test['TST009_n'] - 1
  87             self.test['TST009_m'] = int(old_div((self.test['TST009_L'] + self.test['TST009_R']), 2))
  88             return (self.get_percentageof10Gbps(self.test['TST009_S'][self.test['TST009_m']],size))
  89         else:
  90             self.test['minspeed'] = 0
  91             self.test['maxspeed'] = self.test['startspeed']
  92             return (self.test['startspeed'])
  93
  94     def resolution_achieved(self):
  95         if self.test['test'] == 'fixed_rate':
  96             return (True)
  97         elif 'TST009' in self.test.keys():
  98             return (self.test['TST009_L'] == self.test['TST009_R'])
  99         else:
 100             return ((self.test['maxspeed'] - self.test['minspeed']) <= self.test['accuracy'])
 101
 102     def run(self):
 103     #    global fieldnames
 104     #    global writer
 105     #    #fieldnames = ['Flows','PacketSize','Gbps','Mpps','AvgLatency','MaxLatency','PacketsDropped','PacketDropRate']
 106     #    fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Sent','Received','Lost','LostTotal']
 107     #    writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 108     #    writer.writeheader()
 109         self.gen_machine.start_latency_cores()
 110         TestPassed = True
 111         for imix in self.test['imixs']:
 112             size = mean(imix)
 113             self.gen_machine.set_udp_packet_size(imix)
 114             if self.background_machines:
 115                 backgroundinfo = '{}Running {} x background traffic not represented in the table{}'.format(bcolors.FLASH,len(self.background_machines),bcolors.ENDC)
 116             else:
 117                 backgroundinfo = '{}{}'.format(bcolors.FLASH,bcolors.ENDC)
 118             self.set_background_size(self.background_machines, imix)
 119             RapidLog.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 120             RapidLog.info('| UDP, {:>5} bytes, different number of flows by randomizing SRC & DST UDP port. {:116.116}|'.format(size, backgroundinfo))
 121             RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 122             RapidLog.info('| Flows  | Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core           | Avg. Lat.|{:.0f} Pcentil| Max. Lat.|   Sent    |  Received |    Lost   | Total Lost|L.Ratio|Time|'.format(self.test['lat_percentile']*100))
 123             RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 124             for flow_number in self.test['flows']:
 125                 attempts = 0
 126                 self.gen_machine.reset_stats()
 127                 if self.sut_machine:
 128                     self.sut_machine.reset_stats()
 129                 self.gen_machine.set_flows(flow_number)
 130                 self.set_background_flows(self.background_machines, flow_number)
 131                 endspeed = None
 132                 speed = self.get_start_speed_and_init(size)
 133                 while True:
 134                     attempts += 1
 135                     endwarning = False
 136                     print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 137                     sys.stdout.flush()
 138                     # Start generating packets at requested speed (in % of a 10Gb/s link)
 139                     self.gen_machine.set_generator_speed(speed)
 140                     self.set_background_speed(self.background_machines, speed)
 141                     self.start_background_traffic(self.background_machines)
 142                     # Get statistics now that the generation is stable and initial ARP messages are dealt with
 143                     pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc , lat_perc_max, lat_max, abs_tx,abs_rx,abs_dropped, abs_tx_fail, drop_rate, lat_min, lat_used, r, actual_duration = self.run_iteration(float(self.test['runtime']),flow_number,size,speed)
 144                     self.stop_background_traffic(self.background_machines)
 145                     if r > 1:
 146                         retry_warning = bcolors.WARNING + ' {:1} retries needed'.format(r) +  bcolors.ENDC
 147                     else:
 148                         retry_warning = ''
 149                     # Drop rate is expressed in percentage. lat_used is a ratio (0 to 1). The sum of these 2 should be 100%.
 150                     # If the sum is lower than 95, it means that more than 5% of the latency measurements where dropped for accuracy reasons.
 151                     if (drop_rate + lat_used * 100) < 95:
 152                         lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 153                     else:
 154                         lat_warning = ''
 155                     if self.test['test'] == 'fixed_rate':
 156                         endspeed = speed
 157                         endpps_req_tx = None
 158                         endpps_tx = None
 159                         endpps_sut_tx = None
 160                         endpps_rx = None
 161                         endlat_avg = lat_avg
 162                         endlat_perc = lat_perc
 163                         endlat_perc_max = lat_perc_max
 164                         endlat_max = lat_max
 165                         endabs_dropped = abs_dropped
 166                         enddrop_rate = drop_rate
 167                         endabs_tx = abs_tx
 168                         endabs_rx = abs_rx
 169                         if lat_warning or retry_warning:
 170                             endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning)
 171                         success = True
 172                         TestPassed = False # fixed rate testing cannot be True, it is just reported numbers every second
 173                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 174                     # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold
 175                     # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
 176                     # This can be specified by putting 0 in the .test file
 177                     elif ((drop_rate < self.test['drop_rate_threshold']) or (abs_dropped==self.test['drop_rate_threshold']==0)) and (lat_avg< self.test['lat_avg_threshold']) and (lat_perc< self.test['lat_perc_threshold']) and (lat_max < self.test['lat_max_threshold']):
 178                         if (old_div((self.get_pps(speed,size) - pps_tx),self.get_pps(speed,size)))>0.01:
 179                             speed_prefix = bcolors.WARNING
 180                             if abs_tx_fail > 0:
 181                                 gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(self.get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
 182                             else:
 183                                 gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(self.get_pps(speed,size), pps_tx) + bcolors.ENDC
 184                         else:
 185                             speed_prefix = bcolors.ENDC
 186                             gen_warning = ''
 187                         endspeed = speed
 188                         endspeed_prefix = speed_prefix
 189                         endpps_req_tx = pps_req_tx
 190                         endpps_tx = pps_tx
 191                         endpps_sut_tx = pps_sut_tx
 192                         endpps_rx = pps_rx
 193                         endlat_avg = lat_avg
 194                         endlat_perc = lat_perc
 195                         endlat_perc_max = lat_perc_max
 196                         endlat_max = lat_max
 197                         endabs_dropped = None
 198                         enddrop_rate = drop_rate
 199                         endabs_tx = abs_tx
 200                         endabs_rx = abs_rx
 201                         if lat_warning or gen_warning or retry_warning:
 202                             endwarning = '|        | {:186.186} |'.format(retry_warning + lat_warning + gen_warning)
 203                         success = True
 204                         success_message=' SUCCESS'
 205                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 206                         RapidLog.debug(self.report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
 207                     else:
 208                         success_message=' FAILED'
 209                         abs_drop_rate_prefix = bcolors.ENDC
 210                         if ((abs_dropped>0) and (self.test['drop_rate_threshold'] ==0)):
 211                             abs_drop_rate_prefix = bcolors.FAIL
 212                         if (drop_rate < self.test['drop_rate_threshold']):
 213                             drop_rate_prefix = bcolors.ENDC
 214                         else:
 215                             drop_rate_prefix = bcolors.FAIL
 216                         if (lat_avg< self.test['lat_avg_threshold']):
 217                             lat_avg_prefix = bcolors.ENDC
 218                         else:
 219                             lat_avg_prefix = bcolors.FAIL
 220                         if (lat_perc< self.test['lat_perc_threshold']):
 221                             lat_perc_prefix = bcolors.ENDC
 222                         else:
 223                             lat_perc_prefix = bcolors.FAIL
 224                         if (lat_max< self.test['lat_max_threshold']):
 225                             lat_max_prefix = bcolors.ENDC
 226                         else:
 227                             lat_max_prefix = bcolors.FAIL
 228                         if ((old_div((self.get_pps(speed,size) - pps_tx),self.get_pps(speed,size)))<0.001):
 229                             speed_prefix = bcolors.ENDC
 230                         else:
 231                             speed_prefix = bcolors.FAIL
 232                         success = False
 233                         RapidLog.debug(self.report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning)
 234                     speed = self.new_speed(speed, size, success)
 235                     if self.resolution_achieved():
 236                         break
 237                 if endspeed is not None:
 238                     if TestPassed and (endpps_rx < self.test['pass_threshold']):
 239                         TestPassed = False
 240                     speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 241                     RapidLog.info(self.report_result(flow_number,size,endspeed,endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_perc,endlat_perc_max,endlat_max,endabs_tx,endabs_rx,endabs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix))
 242                     if endwarning:
 243                         RapidLog.info (endwarning)
 244                     RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 245     #                writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':self.get_pps(endspeed,size),'GeneratedPPS':endpps_req_tx,'SentPPS':endpps_tx,'ForwardedPPS':endpps_sut_tx,'ReceivedPPS':endpps_rx,'AvgLatencyUSEC':endlat_avg,'MaxLatencyUSEC':endlat_max,'Sent':endabs_tx,'Received':endabs_rx,'Lost':endabs_dropped,'LostTotal':endabs_dropped})
 246                     if self.test['pushgateway']:
 247                         URL = self.test['pushgateway'] + '/metrics/job/' + self.test['test']+ '/instance/' + self.test['environment_file']
 248                         if endabs_dropped == None:
 249                             ead = 0
 250                         else:
 251                             ead = endabs_dropped
 252                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nSent {}\nReceived {}\nLost {}\nLostTotal {}\n'.format(flow_number,size+4,self.get_pps(endspeed,size),endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_max,endabs_tx,endabs_rx,ead,ead)
 253                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 254                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 255                         if (response.status_code != 202) and (response.status_code != 200):
 256                             RapidLog.info('Cannot send metrics to {}'.format(URL))
 257                             RapidLog.info(DATA)
 258                 else:
 259                     RapidLog.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
 260         self.gen_machine.stop_latency_cores()
 261         return (TestPassed)