yardstick/benchmark/scenarios/compute/cpuload.py

   1 ##############################################################################
   2 # Copyright (c) 2015 Ericsson AB and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 """Processor statistics and system load."""
  11
  12 import logging
  13 import time
  14 import re
  15 import yardstick.ssh as ssh
  16
  17 from yardstick.benchmark.scenarios import base
  18
  19
  20 LOG = logging.getLogger(__name__)
  21
  22
  23 class CPULoad(base.Scenario):
  24
  25     """Collect processor statistics and system load.
  26
  27     This scenario reads system load averages and
  28     CPU usage statistics on a Linux host.
  29
  30     CPU usage statistics are read using the utility 'mpstat'.
  31
  32     If 'mpstat' is not installed on the host usage statistics
  33     are instead read directly from '/proc/stat'.
  34
  35     Load averages are read from the file '/proc/loadavg'
  36     on the Linux host.
  37
  38     Parameters
  39           interval - Time interval to measure CPU usage.
  40
  41           type:       [int]
  42           unit:       seconds
  43           default:    1
  44
  45           count (for mpstat only) - Number of CPU usage measurment.
  46
  47           type:       [int]
  48           unit:       N/A
  49           default:    1
  50
  51     """
  52
  53     __scenario_type__ = "CPUload"
  54
  55     MPSTAT_FIELD_SIZE = 10
  56
  57     def __init__(self, scenario_cfg, context_cfg):
  58         """Scenario construction."""
  59         self.scenario_cfg = scenario_cfg
  60         self.context_cfg = context_cfg
  61         self.setup_done = False
  62         self.has_mpstat = False
  63         self.has_count = False
  64
  65     def setup(self):
  66         """Scenario setup."""
  67         host = self.context_cfg['host']
  68         user = host.get('user', 'ubuntu')
  69         ip = host.get('ip', None)
  70         ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT)
  71         key_filename = host.get('key_filename', '~/.ssh/id_rsa')
  72
  73         LOG.info("user:%s, host:%s", user, ip)
  74         self.client = ssh.SSH(user, ip, key_filename=key_filename,
  75                               port=ssh_port)
  76         self.client.wait(timeout=600)
  77
  78         # Check if mpstat prog is installed
  79         status, _, _ = self.client.execute("mpstat -V >/dev/null 2>&1")
  80         if status != 0:
  81             LOG.info("MPSTAT is NOT installed")
  82             self.has_mpstat = False
  83         else:
  84             LOG.info("MPSTAT is installed")
  85             self.has_mpstat = True
  86
  87         options = self.scenario_cfg['options']
  88         self.interval = options.get("interval", 1)
  89         if 'count' in options:
  90             self.count = options.get("count", 1)
  91             self.has_count = True
  92         else:
  93             self.has_count = False
  94
  95         self.setup_done = True
  96
  97     def _execute_command(self, cmd):
  98         """Execute a command on server."""
  99         LOG.info("Executing: %s" % cmd)
 100         status, stdout, stderr = self.client.execute(cmd)
 101         if status != 0:
 102             raise RuntimeError("Failed executing command: ",
 103                                cmd, stderr)
 104         return stdout
 105
 106     def _get_loadavg(self):
 107         """Get system load."""
 108         return {'loadavg': self._execute_command("cat /proc/loadavg").split()}
 109
 110     def _get_cpu_usage_mpstat(self):
 111         """Get processor usage using mpstat."""
 112         if self.interval > 0 and self.has_count:
 113             cmd = "mpstat -P ON %s %s" % (self.interval, self.count)
 114         else:
 115             cmd = "mpstat -P ON %s 1" % self.interval
 116
 117         result = self._execute_command(cmd)
 118
 119         fields = []
 120         maximum = {}
 121         minimum = {}
 122         average = {}
 123
 124         time_marker = re.compile("^([0-9]+):([0-9]+):([0-9]+)$")
 125         ampm_marker = re.compile("(AM|PM)$")
 126
 127         # Parse CPU stats
 128         for row in result.split('\n'):
 129             line = row.split()
 130
 131             if line and re.match(time_marker, line[0]):
 132                 if re.match(ampm_marker, line[1]):
 133                     del line[:2]
 134                 else:
 135                     del line[:1]
 136
 137                 if line[0] == 'CPU':
 138                     # header fields
 139                     fields = line[1:]
 140                     if len(fields) != CPULoad.MPSTAT_FIELD_SIZE:
 141                         raise RuntimeError("mpstat: unexpected field size",
 142                                            fields)
 143                 else:
 144                     # value fields
 145                     cpu = 'cpu' if line[0] == 'all' else 'cpu' + line[0]
 146                     values = line[1:]
 147                     if values and len(values) == len(fields):
 148                         temp_dict = dict(zip(fields, values))
 149                         if cpu not in maximum:
 150                             maximum[cpu] = temp_dict
 151                         else:
 152                             for item in temp_dict:
 153                                 if float(maximum[cpu][item]) <\
 154                                    float(temp_dict[item]):
 155                                     maximum[cpu][item] = temp_dict[item]
 156
 157                         if cpu not in minimum:
 158                             minimum[cpu] = temp_dict
 159                         else:
 160                             for item in temp_dict:
 161                                 if float(minimum[cpu][item]) >\
 162                                    float(temp_dict[item]):
 163                                     minimum[cpu][item] = temp_dict[item]
 164                     else:
 165                         raise RuntimeError("mpstat: parse error", fields, line)
 166
 167             elif line and line[0] == 'Average:':
 168                 del line[:1]
 169                 if line[0] == 'CPU':
 170                     # header fields
 171                     fields = line[1:]
 172                     if len(fields) != CPULoad.MPSTAT_FIELD_SIZE:
 173                         raise RuntimeError("mpstat average: unexpected field\
 174                                            size", fields)
 175                 else:
 176                     # value fields
 177                     cpu = 'cpu' if line[0] == 'all' else 'cpu' + line[0]
 178                     values = line[1:]
 179                     if values and len(values) == len(fields):
 180                         average[cpu] = dict(zip(fields, values))
 181                     else:
 182                         raise RuntimeError("mpstat average: parse error",
 183                                            fields, line)
 184
 185         return {'mpstat_maximun': maximum, 'mpstat_minimum': minimum,
 186                 'mpstat_average': average}
 187
 188     def _get_cpu_usage(self):
 189         """Get processor usage from /proc/stat."""
 190         fields = ['%usr', '%nice', '%sys', '%idle', '%iowait',
 191                   '%irq', '%soft', '%steal', '%guest', '%gnice']
 192
 193         cmd = "grep '^cpu[0-9 ].' /proc/stat"
 194
 195         if self.interval > 0:
 196             previous = self._execute_command(cmd).splitlines()
 197             time.sleep(self.interval)
 198             current = self._execute_command(cmd).splitlines()
 199         else:
 200             current = self._execute_command(cmd).splitlines()
 201             previous = current
 202
 203         mpstat = {}
 204
 205         for (prev, cur) in zip(previous, current):
 206
 207             # Split string to list tokens
 208             cur_list = cur.split()
 209             prev_list = prev.split()
 210
 211             cpu = cur_list[0]
 212
 213             cur_stats = map(int, cur_list[1:])
 214             if self.interval > 0:
 215                 prev_stats = map(int, prev_list[1:])
 216             else:
 217                 prev_stats = [0] * len(cur_stats)
 218
 219             # NB: Don't add 'guest' and 'gnice' as they
 220             # are already included in 'usr' and 'nice'.
 221             uptime_prev = sum(prev_stats[0:8])
 222             uptime_cur = sum(cur_stats[0:8])
 223
 224             # Remove 'guest' and 'gnice' from 'usr' and 'nice'
 225             prev_stats[0] -= prev_stats[8]
 226             prev_stats[1] -= prev_stats[9]
 227             cur_stats[0] -= cur_stats[8]
 228             cur_stats[1] -= cur_stats[9]
 229
 230             # number of samples (jiffies) in the interval
 231             samples = (uptime_cur - uptime_prev) or 1
 232
 233             def _percent(x, y):
 234                 if x < y:
 235                     return 0.0
 236                 else:
 237                     return "%.2f" % (100.0 * (x - y) / samples)
 238
 239             load = map(_percent, cur_stats, prev_stats)
 240
 241             mpstat[cpu] = dict(zip(fields, load))
 242
 243         return {'mpstat': mpstat}
 244
 245     def run(self, result):
 246         """Read processor statistics."""
 247         if not self.setup_done:
 248             self.setup()
 249
 250         result.update(self._get_loadavg())
 251
 252         if self.has_mpstat:
 253             result.update(self._get_cpu_usage_mpstat())
 254         else:
 255             result.update(self._get_cpu_usage())
 256
 257         # Note: No SLA as this scenario is only collecting statistics
 258
 259 # def _test():
 260 #     """internal test function."""
 261 #     import pkg_resources
 262 #     key_filename = pkg_resources.resource_filename('yardstick.resources',
 263 #                                                    'files/yardstick_key')
 264 #     ctx = {
 265 #         'host': {
 266 #             'ip': '172.16.0.175',
 267 #             'user': 'ubuntu',
 268 #             'key_filename': key_filename
 269 #         }
 270 #     }
 271
 272 #     logger = logging.getLogger('yardstick')
 273 #     logger.setLevel(logging.DEBUG)
 274
 275 #     args = {}
 276 #     result = {}
 277
 278 #     p = CPULoad(args, ctx)
 279 #     p.run(result)
 280 #     import json
 281 #     print json.dumps(result)
 282
 283 # if __name__ == '__main__':
 284 #     _test()