yardstick/benchmark/scenarios/compute/cpuload.py

   1 ##############################################################################
   2 # Copyright (c) 2015 Ericsson AB and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 """Processor statistics and system load."""
  11
  12 from __future__ import absolute_import
  13
  14 import logging
  15 import re
  16 import time
  17
  18 from six.moves import map, zip
  19
  20 import yardstick.ssh as ssh
  21 from yardstick.benchmark.scenarios import base
  22
  23 LOG = logging.getLogger(__name__)
  24
  25
  26 class CPULoad(base.Scenario):
  27
  28     """Collect processor statistics and system load.
  29
  30     This scenario reads system load averages and
  31     CPU usage statistics on a Linux host.
  32
  33     CPU usage statistics are read using the utility 'mpstat'.
  34
  35     If 'mpstat' is not installed on the host usage statistics
  36     are instead read directly from '/proc/stat'.
  37
  38     Load averages are read from the file '/proc/loadavg'
  39     on the Linux host.
  40
  41     Parameters
  42           interval - Time interval to measure CPU usage.
  43
  44           type:       [int]
  45           unit:       seconds
  46           default:    1
  47
  48           count (for mpstat only) - Number of CPU usage measurment.
  49
  50           type:       [int]
  51           unit:       N/A
  52           default:    1
  53
  54     """
  55
  56     __scenario_type__ = "CPUload"
  57
  58     MPSTAT_FIELD_SIZE = 10
  59
  60     def __init__(self, scenario_cfg, context_cfg):
  61         """Scenario construction."""
  62         self.scenario_cfg = scenario_cfg
  63         self.context_cfg = context_cfg
  64         self.setup_done = False
  65         self.has_mpstat = False
  66         self.has_count = False
  67
  68     def setup(self):
  69         """Scenario setup."""
  70         host = self.context_cfg['host']
  71         user = host.get('user', 'ubuntu')
  72         ip = host.get('ip', None)
  73         ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT)
  74         key_filename = host.get('key_filename', '~/.ssh/id_rsa')
  75
  76         LOG.info("user:%s, host:%s", user, ip)
  77         self.client = ssh.SSH(user, ip, key_filename=key_filename,
  78                               port=ssh_port)
  79         self.client.wait(timeout=600)
  80
  81         # Check if mpstat prog is installed
  82         status, _, _ = self.client.execute("mpstat -V >/dev/null 2>&1")
  83         if status != 0:
  84             LOG.info("MPSTAT is NOT installed")
  85             self.has_mpstat = False
  86         else:
  87             LOG.info("MPSTAT is installed")
  88             self.has_mpstat = True
  89
  90         options = self.scenario_cfg['options']
  91         self.interval = options.get("interval", 1)
  92         if 'count' in options:
  93             self.count = options.get("count", 1)
  94             self.has_count = True
  95         else:
  96             self.has_count = False
  97
  98         self.setup_done = True
  99
 100     def _execute_command(self, cmd):
 101         """Execute a command on server."""
 102         LOG.info("Executing: %s", cmd)
 103         status, stdout, stderr = self.client.execute(cmd)
 104         if status != 0:
 105             raise RuntimeError("Failed executing command: ",
 106                                cmd, stderr)
 107         return stdout
 108
 109     def _get_loadavg(self):
 110         """Get system load."""
 111         return {'loadavg': self._execute_command("cat /proc/loadavg").split()}
 112
 113     def _get_cpu_usage_mpstat(self):
 114         """Get processor usage using mpstat."""
 115         if self.interval > 0 and self.has_count:
 116             cmd = "mpstat -P ON %s %s" % (self.interval, self.count)
 117         else:
 118             cmd = "mpstat -P ON %s 1" % self.interval
 119
 120         result = self._execute_command(cmd)
 121
 122         fields = []
 123         maximum = {}
 124         minimum = {}
 125         average = {}
 126
 127         time_marker = re.compile("^([0-9]+):([0-9]+):([0-9]+)$")
 128         ampm_marker = re.compile("(AM|PM)$")
 129
 130         # Parse CPU stats
 131         for row in result.split('\n'):
 132             line = row.split()
 133
 134             if line and re.match(time_marker, line[0]):
 135                 if re.match(ampm_marker, line[1]):
 136                     del line[:2]
 137                 else:
 138                     del line[:1]
 139
 140                 if line[0] == 'CPU':
 141                     # header fields
 142                     fields = line[1:]
 143                     if len(fields) != CPULoad.MPSTAT_FIELD_SIZE:
 144                         raise RuntimeError("mpstat: unexpected field size",
 145                                            fields)
 146                 else:
 147                     # value fields
 148                     cpu = 'cpu' if line[0] == 'all' else 'cpu' + line[0]
 149                     values = line[1:]
 150                     if values and len(values) == len(fields):
 151                         temp_dict = dict(list(zip(fields, values)))
 152                         if cpu not in maximum:
 153                             maximum[cpu] = temp_dict
 154                         else:
 155                             for item in temp_dict:
 156                                 if float(maximum[cpu][item]) <\
 157                                    float(temp_dict[item]):
 158                                     maximum[cpu][item] = temp_dict[item]
 159
 160                         if cpu not in minimum:
 161                             minimum[cpu] = temp_dict
 162                         else:
 163                             for item in temp_dict:
 164                                 if float(minimum[cpu][item]) >\
 165                                    float(temp_dict[item]):
 166                                     minimum[cpu][item] = temp_dict[item]
 167                     else:
 168                         raise RuntimeError("mpstat: parse error", fields, line)
 169
 170             elif line and line[0] == 'Average:':
 171                 del line[:1]
 172                 if line[0] == 'CPU':
 173                     # header fields
 174                     fields = line[1:]
 175                     if len(fields) != CPULoad.MPSTAT_FIELD_SIZE:
 176                         raise RuntimeError("mpstat average: unexpected field\
 177                                            size", fields)
 178                 else:
 179                     # value fields
 180                     cpu = 'cpu' if line[0] == 'all' else 'cpu' + line[0]
 181                     values = line[1:]
 182                     if values and len(values) == len(fields):
 183                         average[cpu] = dict(list(zip(fields, values)))
 184                     else:
 185                         raise RuntimeError("mpstat average: parse error",
 186                                            fields, line)
 187
 188         return {'mpstat_maximun': maximum, 'mpstat_minimum': minimum,
 189                 'mpstat_average': average}
 190
 191     def _get_cpu_usage(self):
 192         """Get processor usage from /proc/stat."""
 193         fields = ['%usr', '%nice', '%sys', '%idle', '%iowait',
 194                   '%irq', '%soft', '%steal', '%guest', '%gnice']
 195
 196         cmd = "grep '^cpu[0-9 ].' /proc/stat"
 197
 198         if self.interval > 0:
 199             previous = self._execute_command(cmd).splitlines()
 200             time.sleep(self.interval)
 201             current = self._execute_command(cmd).splitlines()
 202         else:
 203             current = self._execute_command(cmd).splitlines()
 204             previous = current
 205
 206         mpstat = {}
 207
 208         for (prev, cur) in zip(previous, current):
 209
 210             # Split string to list tokens
 211             cur_list = cur.split()
 212             prev_list = prev.split()
 213
 214             cpu = cur_list[0]
 215
 216             cur_stats = list(map(int, cur_list[1:]))
 217             if self.interval > 0:
 218                 prev_stats = list(map(int, prev_list[1:]))
 219             else:
 220                 prev_stats = [0] * len(cur_stats)
 221
 222             # NB: Don't add 'guest' and 'gnice' as they
 223             # are already included in 'usr' and 'nice'.
 224             uptime_prev = sum(prev_stats[0:8])
 225             uptime_cur = sum(cur_stats[0:8])
 226
 227             # Remove 'guest' and 'gnice' from 'usr' and 'nice'
 228             prev_stats[0] -= prev_stats[8]
 229             prev_stats[1] -= prev_stats[9]
 230             cur_stats[0] -= cur_stats[8]
 231             cur_stats[1] -= cur_stats[9]
 232
 233             # number of samples (jiffies) in the interval
 234             samples = (uptime_cur - uptime_prev) or 1
 235
 236             def _percent(x, y):
 237                 if x < y:
 238                     return 0.0
 239                 else:
 240                     return "%.2f" % (100.0 * (x - y) / samples)
 241
 242             load = list(map(_percent, cur_stats, prev_stats))
 243
 244             mpstat[cpu] = dict(list(zip(fields, load)))
 245
 246         return {'mpstat': mpstat}
 247
 248     def run(self, result):
 249         """Read processor statistics."""
 250         if not self.setup_done:
 251             self.setup()
 252
 253         result.update(self._get_loadavg())
 254
 255         if self.has_mpstat:
 256             result.update(self._get_cpu_usage_mpstat())
 257         else:
 258             result.update(self._get_cpu_usage())
 259
 260         # Note: No SLA as this scenario is only collecting statistics
 261
 262 # def _test():
 263 #     """internal test function."""
 264 #     import pkg_resources
 265 #     key_filename = pkg_resources.resource_filename('yardstick.resources',
 266 #                                                    'files/yardstick_key')
 267 #     ctx = {
 268 #         'host': {
 269 #             'ip': '172.16.0.175',
 270 #             'user': 'ubuntu',
 271 #             'key_filename': key_filename
 272 #         }
 273 #     }
 274
 275 #     logger = logging.getLogger('yardstick')
 276 #     logger.setLevel(logging.DEBUG)
 277
 278 #     args = {}
 279 #     result = {}
 280
 281 #     p = CPULoad(args, ctx)
 282 #     p.run(result)
 283 #     import json
 284 #     print(oslo_serialization.jsonutils.dump_as_bytes(result))
 285
 286 # if __name__ == '__main__':
 287 #     _test()