nfvbench/summarizer.py

   1 #!/usr/bin/env python
   2 # Copyright 2016 Cisco Systems, Inc.  All rights reserved.
   3 #
   4 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   5 #    not use this file except in compliance with the License. You may obtain
   6 #    a copy of the License at
   7 #
   8 #         http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 #    Unless required by applicable law or agreed to in writing, software
  11 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13 #    License for the specific language governing permissions and limitations
  14 #    under the License.
  15 #
  16
  17 import bitmath
  18 from contextlib import contextmanager
  19 import math
  20 from specs import ChainType
  21 from tabulate import tabulate
  22
  23
  24 class Formatter(object):
  25     """Collection of string formatter methods"""
  26
  27     @staticmethod
  28     def fixed(data):
  29         return data
  30
  31     @staticmethod
  32     def int(data):
  33         return '{:,}'.format(data)
  34
  35     @staticmethod
  36     def float(decimal):
  37         return lambda data: '%.{}f'.format(decimal) % (data)
  38
  39     @staticmethod
  40     def standard(data):
  41         if type(data) == int:
  42             return Formatter.int(data)
  43         elif type(data) == float:
  44             return Formatter.float(4)(data)
  45         else:
  46             return Formatter.fixed(data)
  47
  48     @staticmethod
  49     def suffix(suffix_str):
  50         return lambda data: Formatter.standard(data) + suffix_str
  51
  52     @staticmethod
  53     def bits(data):
  54         # By default, `best_prefix` returns a value in byte format, this hack (multiply by 8.0)
  55         # will convert it into bit format.
  56         bit = 8.0 * bitmath.Bit(float(data))
  57         bit = bit.best_prefix(bitmath.SI)
  58         byte_to_bit_classes = {
  59             'kB': bitmath.kb,
  60             'MB': bitmath.Mb,
  61             'GB': bitmath.Gb,
  62             'TB': bitmath.Tb,
  63             'PB': bitmath.Pb,
  64             'EB': bitmath.Eb,
  65             'ZB': bitmath.Zb,
  66             'YB': bitmath.Yb,
  67         }
  68         bps = byte_to_bit_classes.get(bit.unit, bitmath.Bit).from_other(bit) / 8.0
  69         if bps.unit != 'Bit':
  70             return bps.format("{value:.4f} {unit}ps")
  71         else:
  72             return bps.format("{value:.4f} bps")
  73
  74     @staticmethod
  75     def percentage(data):
  76         if data is None:
  77             return ''
  78         elif math.isnan(data):
  79             return '-'
  80         else:
  81             return Formatter.suffix('%')(Formatter.float(4)(data))
  82
  83
  84 class Table(object):
  85     """ASCII readable table class"""
  86
  87     def __init__(self, header):
  88         header_row, self.formatters = zip(*header)
  89         self.data = [header_row]
  90         self.columns = len(header_row)
  91
  92     def add_row(self, row):
  93         assert(self.columns == len(row))
  94         formatted_row = []
  95         for entry, formatter in zip(row, self.formatters):
  96             formatted_row.append(formatter(entry))
  97         self.data.append(formatted_row)
  98
  99     def get_string(self, indent=0):
 100         spaces = ' ' * indent
 101         table = tabulate(self.data,
 102                          headers='firstrow',
 103                          tablefmt='grid',
 104                          stralign='center',
 105                          floatfmt='.2f')
 106         return table.replace('\n', '\n' + spaces)
 107
 108
 109 class Summarizer(object):
 110     """Generic summarizer class"""
 111
 112     indent_per_level = 2
 113
 114     def __init__(self):
 115         self.indent_size = 0
 116         self.marker_stack = [False]
 117         self.str = ''
 118
 119     def __indent(self, marker):
 120         self.indent_size += self.indent_per_level
 121         self.marker_stack.append(marker)
 122
 123     def __unindent(self):
 124         assert(self.indent_size >= self.indent_per_level)
 125         self.indent_size -= self.indent_per_level
 126         self.marker_stack.pop()
 127
 128     def __get_indent_string(self):
 129         current_str = ' ' * self.indent_size
 130         if self.marker_stack[-1]:
 131             current_str = current_str[:-2] + '> '
 132         return current_str
 133
 134     def _put(self, *args):
 135         self.str += self.__get_indent_string()
 136         if len(args) and type(args[-1]) == dict:
 137             self.str += ' '.join(map(str, args[:-1])) + '\n'
 138             self._put_dict(args[-1])
 139         else:
 140             self.str += ' '.join(map(str, args)) + '\n'
 141
 142     def _put_dict(self, data):
 143         with self._create_block(False):
 144             for key, value in data.iteritems():
 145                 if type(value) == dict:
 146                     self._put(key + ':')
 147                     self._put_dict(value)
 148                 else:
 149                     self._put(key + ':', value)
 150
 151     def _put_table(self, table):
 152         self.str += self.__get_indent_string()
 153         self.str += table.get_string(self.indent_size) + '\n'
 154
 155     def __str__(self):
 156         return self.str
 157
 158     @contextmanager
 159     def _create_block(self, marker=True):
 160         self.__indent(marker)
 161         yield
 162         self.__unindent()
 163
 164
 165 class NFVBenchSummarizer(Summarizer):
 166     """Summarize nfvbench json result"""
 167
 168     ndr_pdr_header = [
 169         ('-', Formatter.fixed),
 170         ('L2 Frame Size', Formatter.standard),
 171         ('Rate (fwd+rev)', Formatter.bits),
 172         ('Rate (fwd+rev)', Formatter.suffix(' pps')),
 173         ('Avg Drop Rate', Formatter.suffix('%')),
 174         ('Avg Latency (usec)', Formatter.standard),
 175         ('Min Latency (usec)', Formatter.standard),
 176         ('Max Latency (usec)', Formatter.standard)
 177     ]
 178
 179     single_run_header = [
 180         ('L2 Frame Size', Formatter.standard),
 181         ('Drop Rate', Formatter.suffix('%')),
 182         ('Avg Latency (usec)', Formatter.standard),
 183         ('Min Latency (usec)', Formatter.standard),
 184         ('Max Latency (usec)', Formatter.standard)
 185     ]
 186
 187     config_header = [
 188         ('Direction', Formatter.standard),
 189         ('Requested TX Rate (bps)', Formatter.bits),
 190         ('Actual TX Rate (bps)', Formatter.bits),
 191         ('RX Rate (bps)', Formatter.bits),
 192         ('Requested TX Rate (pps)', Formatter.suffix(' pps')),
 193         ('Actual TX Rate (pps)', Formatter.suffix(' pps')),
 194         ('RX Rate (pps)', Formatter.suffix(' pps'))
 195     ]
 196
 197     chain_analysis_header = [
 198         ('Interface', Formatter.standard),
 199         ('Device', Formatter.standard),
 200         ('Packets (fwd)', Formatter.standard),
 201         ('Drops (fwd)', Formatter.standard),
 202         ('Drop% (fwd)', Formatter.percentage),
 203         ('Packets (rev)', Formatter.standard),
 204         ('Drops (rev)', Formatter.standard),
 205         ('Drop% (rev)', Formatter.percentage)
 206     ]
 207
 208     direction_keys = ['direction-forward', 'direction-reverse', 'direction-total']
 209     direction_names = ['Forward', 'Reverse', 'Total']
 210
 211     def __init__(self, result):
 212         Summarizer.__init__(self)
 213         self.result = result
 214         self.config = self.result['config']
 215         self.__summarize()
 216
 217     def __summarize(self):
 218         self._put()
 219         self._put('========== NFVBench Summary ==========')
 220         self._put('Date:', self.result['date'])
 221         self._put('NFVBench version', self.result['nfvbench_version'])
 222         self._put('Openstack Neutron:', {
 223             'vSwitch': self.result['openstack_spec']['vswitch'],
 224             'Encapsulation': self.result['openstack_spec']['encaps']
 225         })
 226         self._put('Benchmarks:')
 227         with self._create_block():
 228             self._put('Networks:')
 229             with self._create_block():
 230                 network_benchmark = self.result['benchmarks']['network']
 231
 232                 self._put('Components:')
 233                 with self._create_block():
 234                     self._put('TOR:')
 235                     with self._create_block(False):
 236                         self._put('Type:', self.config['tor']['type'])
 237                     self._put('Traffic Generator:')
 238                     with self._create_block(False):
 239                         self._put('Profile:', self.config['generator_config']['name'])
 240                         self._put('Tool:', self.config['generator_config']['tool'])
 241                     if network_benchmark['versions']:
 242                         self._put('Versions:')
 243                         with self._create_block():
 244                             for component, version in network_benchmark['versions'].iteritems():
 245                                 self._put(component + ':', version)
 246
 247                 if self.config['ndr_run'] or self.config['pdr_run']:
 248                     self._put('Measurement Parameters:')
 249                     with self._create_block(False):
 250                         if self.config['ndr_run']:
 251                             self._put('NDR:', self.config['measurement']['NDR'])
 252                         if self.config['pdr_run']:
 253                             self._put('PDR:', self.config['measurement']['PDR'])
 254
 255                 self._put('Service chain:')
 256                 for result in network_benchmark['service_chain'].iteritems():
 257                     with self._create_block():
 258                         self.__chain_summarize(*result)
 259
 260     def __chain_summarize(self, chain_name, chain_benchmark):
 261         self._put(chain_name + ':')
 262         if chain_name == ChainType.PVVP:
 263             self._put('Mode:', chain_benchmark.get('mode'))
 264         with self._create_block():
 265             self._put('Traffic:')
 266             with self._create_block(False):
 267                 self.__traffic_summarize(chain_benchmark['result'])
 268
 269     def __traffic_summarize(self, traffic_benchmark):
 270         self._put('Profile:', traffic_benchmark['profile'])
 271         self._put('Bidirectional:', traffic_benchmark['bidirectional'])
 272         self._put('Flow count:', traffic_benchmark['flow_count'])
 273         self._put('Service chains count:', traffic_benchmark['service_chain_count'])
 274         self._put('Compute nodes:', traffic_benchmark['compute_nodes'].keys())
 275         with self._create_block(False):
 276             self._put()
 277             if not self.config['no_traffic']:
 278                 self._put('Run Summary:')
 279                 self._put()
 280                 with self._create_block(False):
 281                     self._put_table(self.__get_summary_table(traffic_benchmark['result']))
 282                     try:
 283                         self._put()
 284                         self._put(traffic_benchmark['result']['warning'])
 285                     except KeyError:
 286                         pass
 287
 288             for entry in traffic_benchmark['result'].iteritems():
 289                 if 'warning' in entry:
 290                     continue
 291                 self.__chain_analysis_summarize(*entry)
 292
 293     def __chain_analysis_summarize(self, frame_size, analysis):
 294         self._put()
 295         self._put('L2 frame size:', frame_size)
 296         if 'analysis_duration_sec' in analysis:
 297             self._put('Chain analysis duration:',
 298                       Formatter.float(3)(analysis['analysis_duration_sec']), 'seconds')
 299         if self.config['ndr_run']:
 300             self._put('NDR search duration:', Formatter.float(0)(analysis['ndr']['time_taken_sec']),
 301                       'seconds')
 302         if self.config['pdr_run']:
 303             self._put('PDR search duration:', Formatter.float(0)(analysis['pdr']['time_taken_sec']),
 304                       'seconds')
 305         self._put()
 306
 307         if not self.config['no_traffic'] and self.config['single_run']:
 308             self._put('Run Config:')
 309             self._put()
 310             with self._create_block(False):
 311                 self._put_table(self.__get_config_table(analysis['run_config']))
 312                 if 'warning' in analysis['run_config'] and analysis['run_config']['warning']:
 313                     self._put()
 314                     self._put(analysis['run_config']['warning'])
 315                 self._put()
 316
 317         if 'packet_analysis' in analysis:
 318             self._put('Chain Analysis:')
 319             self._put()
 320             with self._create_block(False):
 321                 self._put_table(self.__get_chain_analysis_table(analysis['packet_analysis']))
 322                 self._put()
 323
 324     def __get_summary_table(self, traffic_result):
 325         if self.config['single_run']:
 326             summary_table = Table(self.single_run_header)
 327         else:
 328             summary_table = Table(self.ndr_pdr_header)
 329
 330         if self.config['ndr_run']:
 331             for frame_size, analysis in traffic_result.iteritems():
 332                 if frame_size == 'warning':
 333                     continue
 334                 summary_table.add_row([
 335                     'NDR',
 336                     frame_size,
 337                     analysis['ndr']['rate_bps'],
 338                     int(analysis['ndr']['rate_pps']),
 339                     analysis['ndr']['stats']['overall']['drop_percentage'],
 340                     analysis['ndr']['stats']['overall']['avg_delay_usec'],
 341                     analysis['ndr']['stats']['overall']['min_delay_usec'],
 342                     analysis['ndr']['stats']['overall']['max_delay_usec']
 343                 ])
 344         if self.config['pdr_run']:
 345             for frame_size, analysis in traffic_result.iteritems():
 346                 if frame_size == 'warning':
 347                     continue
 348                 summary_table.add_row([
 349                     'PDR',
 350                     frame_size,
 351                     analysis['pdr']['rate_bps'],
 352                     int(analysis['pdr']['rate_pps']),
 353                     analysis['pdr']['stats']['overall']['drop_percentage'],
 354                     analysis['pdr']['stats']['overall']['avg_delay_usec'],
 355                     analysis['pdr']['stats']['overall']['min_delay_usec'],
 356                     analysis['pdr']['stats']['overall']['max_delay_usec']
 357                 ])
 358         if self.config['single_run']:
 359             for frame_size, analysis in traffic_result.iteritems():
 360                 summary_table.add_row([
 361                     frame_size,
 362                     analysis['stats']['overall']['drop_rate_percent'],
 363                     analysis['stats']['overall']['rx']['avg_delay_usec'],
 364                     analysis['stats']['overall']['rx']['min_delay_usec'],
 365                     analysis['stats']['overall']['rx']['max_delay_usec']
 366                 ])
 367         return summary_table
 368
 369     def __get_config_table(self, run_config):
 370         config_table = Table(self.config_header)
 371         for key, name in zip(self.direction_keys, self.direction_names):
 372             if key not in run_config:
 373                 continue
 374             config_table.add_row([
 375                 name,
 376                 run_config[key]['orig']['rate_bps'],
 377                 run_config[key]['tx']['rate_bps'],
 378                 run_config[key]['rx']['rate_bps'],
 379                 int(run_config[key]['orig']['rate_pps']),
 380                 int(run_config[key]['tx']['rate_pps']),
 381                 int(run_config[key]['rx']['rate_pps']),
 382             ])
 383         return config_table
 384
 385     def __get_chain_analysis_table(self, packet_analysis):
 386         chain_analysis_table = Table(self.chain_analysis_header)
 387         forward_analysis = packet_analysis['direction-forward']
 388         reverse_analysis = packet_analysis['direction-reverse']
 389         reverse_analysis.reverse()
 390
 391         for fwd, rev in zip(forward_analysis, reverse_analysis):
 392             chain_analysis_table.add_row([
 393                 fwd['interface'],
 394                 fwd['device'],
 395                 fwd['packet_count'],
 396                 fwd.get('packet_drop_count', None),
 397                 fwd.get('packet_drop_percentage', None),
 398                 rev['packet_count'],
 399                 rev.get('packet_drop_count', None),
 400                 rev.get('packet_drop_percentage', None),
 401             ])
 402         return chain_analysis_table