VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py

   1 #!/usr/bin/python
   2
   3 ##
   4 ## Copyright (c) 2020 Intel Corporation
   5 ##
   6 ## Licensed under the Apache License, Version 2.0 (the "License");
   7 ## you may not use this file except in compliance with the License.
   8 ## You may obtain a copy of the License at
   9 ##
  10 ##
  11 ##     http://www.apache.org/licenses/LICENSE-2.0
  12 ##
  13 ## Unless required by applicable law or agreed to in writing, software
  14 ## distributed under the License is distributed on an "AS IS" BASIS,
  15 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 ## See the License for the specific language governing permissions and
  17 ## limitations under the License.
  18 ##
  19
  20 import sys
  21 import time
  22 import requests
  23 from math import ceil
  24 from statistics import mean
  25 from past.utils import old_div
  26 from rapid_log import RapidLog
  27 from rapid_log import bcolors
  28 from rapid_test import RapidTest
  29 inf = float("inf")
  30
  31 class FlowSizeTest(RapidTest):
  32     """
  33     Class to manage the flowsizetesting
  34     """
  35     def __init__(self, test_param, lat_percentile, runtime, pushgateway,
  36             environment_file, gen_machine, sut_machine, background_machines):
  37         super().__init__(test_param, runtime, pushgateway, environment_file)
  38         self.gen_machine = gen_machine
  39         self.sut_machine = sut_machine
  40         self.background_machines = background_machines
  41         self.test['lat_percentile'] = lat_percentile
  42         if self.test['test'] == 'TST009test':
  43             # This test implements some of the testing as defined in
  44             # https://docbox.etsi.org/ISG/NFV/open/Publications_pdf/Specs-Reports/NFV-TST%20009v3.2.1%20-%20GS%20-%20NFVI_Benchmarks.pdf
  45             self.test['TST009_n'] = int(ceil(old_div(
  46                 self.test['maxframespersecondallingress'],
  47                 self.test['stepsize'])))
  48             self.test['TST009'] = True
  49             self.test['TST009_L'] = 0
  50             self.test['TST009_R'] = self.test['TST009_n'] - 1
  51             self.test['TST009_S']= []
  52             for m in range(0, self.test['TST009_n']):
  53                 self.test['TST009_S'].append((m+1) * self.test['stepsize'])
  54             self.test['lat_avg_threshold'] = inf
  55             self.test['lat_perc_threshold'] = inf
  56             self.test['lat_max_threshold'] = inf
  57         elif self.test['test'] == 'fixed_rate':
  58             for key in['drop_rate_threshold','lat_avg_threshold',
  59                     'lat_perc_threshold','lat_max_threshold']:
  60                 self.test[key] = inf
  61
  62     def new_speed(self, speed,size,success):
  63         if self.test['test'] == 'fixed_rate':
  64             return (self.test['startspeed'])
  65         elif 'TST009' in self.test.keys():
  66             if success:
  67                 self.test['TST009_L'] = self.test['TST009_m'] + 1
  68             else:
  69                 self.test['TST009_R'] = max(self.test['TST009_m'] - 1, self.test['TST009_L'])
  70             self.test['TST009_m'] = int (old_div((self.test['TST009_L'] + self.test['TST009_R']),2))
  71             return (self.get_percentageof10Gbps(self.test['TST009_S'][self.test['TST009_m']],size))
  72         else:
  73             if success:
  74                 self.test['minspeed'] = speed
  75             else:
  76                 self.test['maxspeed'] = speed
  77             return (old_div((self.test['minspeed'] + self.test['maxspeed']),2.0))
  78
  79     def get_start_speed_and_init(self, size):
  80         if self.test['test'] == 'fixed_rate':
  81             return (self.test['startspeed'])
  82         elif 'TST009' in self.test.keys():
  83             self.test['TST009_L'] = 0
  84             self.test['TST009_R'] = self.test['TST009_n'] - 1
  85             self.test['TST009_m'] = int(old_div((self.test['TST009_L'] + self.test['TST009_R']), 2))
  86             return (self.get_percentageof10Gbps(self.test['TST009_S'][self.test['TST009_m']],size))
  87         else:
  88             self.test['minspeed'] = 0
  89             self.test['maxspeed'] = self.test['startspeed']
  90             return (self.test['startspeed'])
  91
  92     def resolution_achieved(self):
  93         if self.test['test'] == 'fixed_rate':
  94             return (True)
  95         elif 'TST009' in self.test.keys():
  96             return (self.test['TST009_L'] == self.test['TST009_R'])
  97         else:
  98             return ((self.test['maxspeed'] - self.test['minspeed']) <= self.test['accuracy'])
  99
 100     def run(self):
 101     #    global fieldnames
 102     #    global writer
 103     #    #fieldnames = ['Flows','PacketSize','Gbps','Mpps','AvgLatency','MaxLatency','PacketsDropped','PacketDropRate']
 104     #    fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Sent','Received','Lost','LostTotal']
 105     #    writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
 106     #    writer.writeheader()
 107         self.gen_machine.start_latency_cores()
 108         TestPassed = True
 109         for imix in self.test['imixs']:
 110             size = mean(imix)
 111             self.gen_machine.set_udp_packet_size(imix)
 112             if self.background_machines:
 113                 backgroundinfo = '{}Running {} x background traffic not represented in the table{}'.format(bcolors.FLASH,len(self.background_machines),bcolors.ENDC)
 114             else:
 115                 backgroundinfo = '{}{}'.format(bcolors.FLASH,bcolors.ENDC)
 116             self.set_background_size(self.background_machines, imix)
 117             RapidLog.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
 118             RapidLog.info('| UDP, {:>5} bytes, different number of flows by randomizing SRC & DST UDP port. {:116.116}|'.format(size, backgroundinfo))
 119             RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 120             RapidLog.info('| Flows  | Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core           | Avg. Lat.|{:.0f} Pcentil| Max. Lat.|   Sent    |  Received |    Lost   | Total Lost|L.Ratio|Time|'.format(self.test['lat_percentile']*100))
 121             RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 122             for flow_number in self.test['flows']:
 123                 attempts = 0
 124                 self.gen_machine.reset_stats()
 125                 if self.sut_machine:
 126                     self.sut_machine.reset_stats()
 127                 flow_number = self.gen_machine.set_flows(flow_number)
 128                 self.set_background_flows(self.background_machines, flow_number)
 129                 endspeed = None
 130                 speed = self.get_start_speed_and_init(size)
 131                 while True:
 132                     attempts += 1
 133                     endwarning = False
 134                     print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 135                     sys.stdout.flush()
 136                     # Start generating packets at requested speed (in % of a 10Gb/s link)
 137                     self.gen_machine.set_generator_speed(speed)
 138                     self.set_background_speed(self.background_machines, speed)
 139                     self.start_background_traffic(self.background_machines)
 140                     # Get statistics now that the generation is stable and initial ARP messages are dealt with
 141                     pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc , lat_perc_max, lat_max, abs_tx,abs_rx,abs_dropped, abs_tx_fail, drop_rate, lat_min, lat_used, r, actual_duration = self.run_iteration(float(self.test['runtime']),flow_number,size,speed)
 142                     self.stop_background_traffic(self.background_machines)
 143                     if r > 1:
 144                         retry_warning = bcolors.WARNING + ' {:1} retries needed'.format(r) +  bcolors.ENDC
 145                     else:
 146                         retry_warning = ''
 147                     # Drop rate is expressed in percentage. lat_used is a ratio (0 to 1). The sum of these 2 should be 100%.
 148                     # If the sum is lower than 95, it means that more than 5% of the latency measurements where dropped for accuracy reasons.
 149                     if (drop_rate + lat_used * 100) < 95:
 150                         lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
 151                     else:
 152                         lat_warning = ''
 153                     if self.test['test'] == 'fixed_rate':
 154                         endspeed = speed
 155                         endpps_req_tx = None
 156                         endpps_tx = None
 157                         endpps_sut_tx = None
 158                         endpps_rx = None
 159                         endlat_avg = lat_avg
 160                         endlat_perc = lat_perc
 161                         endlat_perc_max = lat_perc_max
 162                         endlat_max = lat_max
 163                         endabs_dropped = abs_dropped
 164                         enddrop_rate = drop_rate
 165                         endabs_tx = abs_tx
 166                         endabs_rx = abs_rx
 167                         if lat_warning or retry_warning:
 168                             endwarning = '|        | {:177.177} |'.format(retry_warning + lat_warning)
 169                         success = True
 170                         TestPassed = False # fixed rate testing cannot be True, it is just reported numbers every second
 171                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 172                     # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold
 173                     # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test
 174                     # This can be specified by putting 0 in the .test file
 175                     elif ((drop_rate < self.test['drop_rate_threshold']) or (abs_dropped==self.test['drop_rate_threshold']==0)) and (lat_avg< self.test['lat_avg_threshold']) and (lat_perc< self.test['lat_perc_threshold']) and (lat_max < self.test['lat_max_threshold']):
 176                         if (old_div((self.get_pps(speed,size) - pps_tx),self.get_pps(speed,size)))>0.01:
 177                             speed_prefix = bcolors.WARNING
 178                             if abs_tx_fail > 0:
 179                                 gen_warning = bcolors.WARNING + ' Network limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps - {} failed to be transmitted'.format(self.get_pps(speed,size), pps_tx, abs_tx_fail) + bcolors.ENDC
 180                             else:
 181                                 gen_warning = bcolors.WARNING + ' Generator limit?: requesting {:<.3f} Mpps and getting {:<.3f} Mpps'.format(self.get_pps(speed,size), pps_tx) + bcolors.ENDC
 182                         else:
 183                             speed_prefix = bcolors.ENDC
 184                             gen_warning = ''
 185                         endspeed = speed
 186                         endspeed_prefix = speed_prefix
 187                         endpps_req_tx = pps_req_tx
 188                         endpps_tx = pps_tx
 189                         endpps_sut_tx = pps_sut_tx
 190                         endpps_rx = pps_rx
 191                         endlat_avg = lat_avg
 192                         endlat_perc = lat_perc
 193                         endlat_perc_max = lat_perc_max
 194                         endlat_max = lat_max
 195                         endabs_dropped = None
 196                         enddrop_rate = drop_rate
 197                         endabs_tx = abs_tx
 198                         endabs_rx = abs_rx
 199                         if lat_warning or gen_warning or retry_warning:
 200                             endwarning = '|        | {:186.186} |'.format(retry_warning + lat_warning + gen_warning)
 201                         success = True
 202                         success_message=' SUCCESS'
 203                         speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 204                         RapidLog.debug(self.report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning + gen_warning)
 205                     else:
 206                         success_message=' FAILED'
 207                         abs_drop_rate_prefix = bcolors.ENDC
 208                         if ((abs_dropped>0) and (self.test['drop_rate_threshold'] ==0)):
 209                             abs_drop_rate_prefix = bcolors.FAIL
 210                         if (drop_rate < self.test['drop_rate_threshold']):
 211                             drop_rate_prefix = bcolors.ENDC
 212                         else:
 213                             drop_rate_prefix = bcolors.FAIL
 214                         if (lat_avg< self.test['lat_avg_threshold']):
 215                             lat_avg_prefix = bcolors.ENDC
 216                         else:
 217                             lat_avg_prefix = bcolors.FAIL
 218                         if (lat_perc< self.test['lat_perc_threshold']):
 219                             lat_perc_prefix = bcolors.ENDC
 220                         else:
 221                             lat_perc_prefix = bcolors.FAIL
 222                         if (lat_max< self.test['lat_max_threshold']):
 223                             lat_max_prefix = bcolors.ENDC
 224                         else:
 225                             lat_max_prefix = bcolors.FAIL
 226                         if ((old_div((self.get_pps(speed,size) - pps_tx),self.get_pps(speed,size)))<0.001):
 227                             speed_prefix = bcolors.ENDC
 228                         else:
 229                             speed_prefix = bcolors.FAIL
 230                         success = False
 231                         RapidLog.debug(self.report_result(-attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)+ success_message + retry_warning + lat_warning)
 232                     speed = self.new_speed(speed, size, success)
 233                     if self.resolution_achieved():
 234                         break
 235                 if endspeed is not None:
 236                     if TestPassed and (endpps_rx < self.test['pass_threshold']):
 237                         TestPassed = False
 238                     speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC
 239                     RapidLog.info(self.report_result(flow_number,size,endspeed,endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_perc,endlat_perc_max,endlat_max,endabs_tx,endabs_rx,endabs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix))
 240                     if endwarning:
 241                         RapidLog.info (endwarning)
 242                     RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
 243     #                writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':self.get_pps(endspeed,size),'GeneratedPPS':endpps_req_tx,'SentPPS':endpps_tx,'ForwardedPPS':endpps_sut_tx,'ReceivedPPS':endpps_rx,'AvgLatencyUSEC':endlat_avg,'MaxLatencyUSEC':endlat_max,'Sent':endabs_tx,'Received':endabs_rx,'Lost':endabs_dropped,'LostTotal':endabs_dropped})
 244                     if self.test['pushgateway']:
 245                         URL = self.test['pushgateway'] + self.test['test']+ '/instance/' + self.test['environment_file']
 246                         if endabs_dropped == None:
 247                             ead = 0
 248                         else:
 249                             ead = endabs_dropped
 250                         DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nSent {}\nReceived {}\nLost {}\nLostTotal {}\n'.format(flow_number,size+4,self.get_pps(endspeed,size),endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_max,endabs_tx,endabs_rx,ead,ead)
 251                         HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
 252                         response = requests.post(url=URL, data=DATA,headers=HEADERS)
 253                         if (response.status_code != 202) and (response.status_code != 200):
 254                             RapidLog.info('Cannot send metrics to {}'.format(URL))
 255                             RapidLog.info(DATA)
 256                 else:
 257                     RapidLog.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0")
 258         self.gen_machine.stop_latency_cores()
 259         return (TestPassed)