yardstick/benchmark/scenarios/compute/cpuload.py

   1 ##############################################################################
   2 # Copyright (c) 2015 Ericsson AB and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 """Processor statistics and system load."""
  11
  12 from __future__ import absolute_import
  13
  14 import logging
  15 import re
  16 import time
  17
  18 from six.moves import map, zip
  19
  20 import yardstick.ssh as ssh
  21 from yardstick.benchmark.scenarios import base
  22
  23 LOG = logging.getLogger(__name__)
  24
  25
  26 class CPULoad(base.Scenario):
  27
  28     """Collect processor statistics and system load.
  29
  30     This scenario reads system load averages and
  31     CPU usage statistics on a Linux host.
  32
  33     CPU usage statistics are read using the utility 'mpstat'.
  34
  35     If 'mpstat' is not installed on the host usage statistics
  36     are instead read directly from '/proc/stat'.
  37
  38     Load averages are read from the file '/proc/loadavg'
  39     on the Linux host.
  40
  41     Parameters
  42           interval - Time interval to measure CPU usage.
  43
  44           type:       [int]
  45           unit:       seconds
  46           default:    1
  47
  48           count (for mpstat only) - Number of CPU usage measurment.
  49
  50           type:       [int]
  51           unit:       N/A
  52           default:    1
  53
  54     """
  55
  56     __scenario_type__ = "CPUload"
  57
  58     MPSTAT_FIELD_SIZE = 10
  59
  60     def __init__(self, scenario_cfg, context_cfg):
  61         """Scenario construction."""
  62         self.scenario_cfg = scenario_cfg
  63         self.context_cfg = context_cfg
  64         self.setup_done = False
  65         self.has_mpstat = False
  66         self.has_count = False
  67
  68     def setup(self):
  69         """Scenario setup."""
  70         host = self.context_cfg['host']
  71
  72         self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"})
  73         self.client.wait(timeout=600)
  74
  75         # Check if mpstat prog is installed
  76         status, _, _ = self.client.execute("mpstat -V >/dev/null 2>&1")
  77         if status != 0:
  78             LOG.info("MPSTAT is NOT installed")
  79             self.has_mpstat = False
  80         else:
  81             LOG.info("MPSTAT is installed")
  82             self.has_mpstat = True
  83
  84         options = self.scenario_cfg['options']
  85         self.interval = options.get("interval", 1)
  86         if 'count' in options:
  87             self.count = options.get("count", 1)
  88             self.has_count = True
  89         else:
  90             self.has_count = False
  91
  92         self.setup_done = True
  93
  94     def _execute_command(self, cmd):
  95         """Execute a command on server."""
  96         LOG.info("Executing: %s", cmd)
  97         status, stdout, stderr = self.client.execute(cmd)
  98         if status != 0:
  99             raise RuntimeError("Failed executing command: ",
 100                                cmd, stderr)
 101         return stdout
 102
 103     def _get_loadavg(self):
 104         """Get system load."""
 105         return {'loadavg': self._execute_command("cat /proc/loadavg").split()}
 106
 107     def _get_cpu_usage_mpstat(self):
 108         """Get processor usage using mpstat."""
 109         if self.interval > 0 and self.has_count:
 110             cmd = "mpstat -P ON %s %s" % (self.interval, self.count)
 111         else:
 112             cmd = "mpstat -P ON %s 1" % self.interval
 113
 114         result = self._execute_command(cmd)
 115
 116         fields = []
 117         maximum = {}
 118         minimum = {}
 119         average = {}
 120
 121         time_marker = re.compile("^([0-9]+):([0-9]+):([0-9]+)$")
 122         ampm_marker = re.compile("(AM|PM)$")
 123
 124         # Parse CPU stats
 125         for row in result.split('\n'):
 126             line = row.split()
 127
 128             if line and re.match(time_marker, line[0]):
 129                 if re.match(ampm_marker, line[1]):
 130                     del line[:2]
 131                 else:
 132                     del line[:1]
 133
 134                 if line[0] == 'CPU':
 135                     # header fields
 136                     fields = line[1:]
 137                     if len(fields) != CPULoad.MPSTAT_FIELD_SIZE:
 138                         raise RuntimeError("mpstat: unexpected field size",
 139                                            fields)
 140                 else:
 141                     # value fields
 142                     cpu = 'cpu' if line[0] == 'all' else 'cpu' + line[0]
 143                     values = line[1:]
 144                     if values and len(values) == len(fields):
 145                         temp_dict = dict(list(zip(fields, values)))
 146                         if cpu not in maximum:
 147                             maximum[cpu] = temp_dict
 148                         else:
 149                             for item in temp_dict:
 150                                 if float(maximum[cpu][item]) <\
 151                                    float(temp_dict[item]):
 152                                     maximum[cpu][item] = temp_dict[item]
 153
 154                         if cpu not in minimum:
 155                             minimum[cpu] = temp_dict
 156                         else:
 157                             for item in temp_dict:
 158                                 if float(minimum[cpu][item]) >\
 159                                    float(temp_dict[item]):
 160                                     minimum[cpu][item] = temp_dict[item]
 161                     else:
 162                         raise RuntimeError("mpstat: parse error", fields, line)
 163
 164             elif line and line[0] == 'Average:':
 165                 del line[:1]
 166                 if line[0] == 'CPU':
 167                     # header fields
 168                     fields = line[1:]
 169                     if len(fields) != CPULoad.MPSTAT_FIELD_SIZE:
 170                         raise RuntimeError("mpstat average: unexpected field\
 171                                            size", fields)
 172                 else:
 173                     # value fields
 174                     cpu = 'cpu' if line[0] == 'all' else 'cpu' + line[0]
 175                     values = line[1:]
 176                     if values and len(values) == len(fields):
 177                         average[cpu] = dict(list(zip(fields, values)))
 178                     else:
 179                         raise RuntimeError("mpstat average: parse error",
 180                                            fields, line)
 181
 182         return {'mpstat_maximun': maximum, 'mpstat_minimum': minimum,
 183                 'mpstat_average': average}
 184
 185     def _get_cpu_usage(self):
 186         """Get processor usage from /proc/stat."""
 187         fields = ['%usr', '%nice', '%sys', '%idle', '%iowait',
 188                   '%irq', '%soft', '%steal', '%guest', '%gnice']
 189
 190         cmd = "grep '^cpu[0-9 ].' /proc/stat"
 191
 192         if self.interval > 0:
 193             previous = self._execute_command(cmd).splitlines()
 194             time.sleep(self.interval)
 195             current = self._execute_command(cmd).splitlines()
 196         else:
 197             current = self._execute_command(cmd).splitlines()
 198             previous = current
 199
 200         mpstat = {}
 201
 202         for (prev, cur) in zip(previous, current):
 203
 204             # Split string to list tokens
 205             cur_list = cur.split()
 206             prev_list = prev.split()
 207
 208             cpu = cur_list[0]
 209
 210             cur_stats = list(map(int, cur_list[1:]))
 211             if self.interval > 0:
 212                 prev_stats = list(map(int, prev_list[1:]))
 213             else:
 214                 prev_stats = [0] * len(cur_stats)
 215
 216             # NB: Don't add 'guest' and 'gnice' as they
 217             # are already included in 'usr' and 'nice'.
 218             uptime_prev = sum(prev_stats[0:8])
 219             uptime_cur = sum(cur_stats[0:8])
 220
 221             # Remove 'guest' and 'gnice' from 'usr' and 'nice'
 222             prev_stats[0] -= prev_stats[8]
 223             prev_stats[1] -= prev_stats[9]
 224             cur_stats[0] -= cur_stats[8]
 225             cur_stats[1] -= cur_stats[9]
 226
 227             # number of samples (jiffies) in the interval
 228             samples = (uptime_cur - uptime_prev) or 1
 229
 230             def _percent(x, y):
 231                 if x < y:
 232                     return 0.0
 233                 else:
 234                     return "%.2f" % (100.0 * (x - y) / samples)
 235
 236             load = list(map(_percent, cur_stats, prev_stats))
 237
 238             mpstat[cpu] = dict(list(zip(fields, load)))
 239
 240         return {'mpstat': mpstat}
 241
 242     def run(self, result):
 243         """Read processor statistics."""
 244         if not self.setup_done:
 245             self.setup()
 246
 247         result.update(self._get_loadavg())
 248
 249         if self.has_mpstat:
 250             result.update(self._get_cpu_usage_mpstat())
 251         else:
 252             result.update(self._get_cpu_usage())
 253
 254         # Note: No SLA as this scenario is only collecting statistics
 255
 256 # def _test():
 257 #     """internal test function."""
 258 #     import pkg_resources
 259 #     key_filename = pkg_resources.resource_filename('yardstick.resources',
 260 #                                                    'files/yardstick_key')
 261 #     ctx = {
 262 #         'host': {
 263 #             'ip': '172.16.0.175',
 264 #             'user': 'ubuntu',
 265 #             'key_filename': key_filename
 266 #         }
 267 #     }
 268
 269 #     logger = logging.getLogger('yardstick')
 270 #     logger.setLevel(logging.DEBUG)
 271
 272 #     args = {}
 273 #     result = {}
 274
 275 #     p = CPULoad(args, ctx)
 276 #     p.run(result)
 277 #     import json
 278 #     print(oslo_serialization.jsonutils.dump_as_bytes(result))
 279
 280 # if __name__ == '__main__':
 281 #     _test()