yardstick/benchmark/scenarios/compute/cpuload.py

   1 ##############################################################################
   2 # Copyright (c) 2015 Ericsson AB and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 """Processor statistics and system load."""
  11
  12 import logging
  13 import time
  14 import re
  15 import yardstick.ssh as ssh
  16
  17 from yardstick.benchmark.scenarios import base
  18
  19
  20 LOG = logging.getLogger(__name__)
  21
  22
  23 class CPULoad(base.Scenario):
  24
  25     """Collect processor statistics and system load.
  26
  27     This scenario reads system load averages and
  28     CPU usage statistics on a Linux host.
  29
  30     CPU usage statistics are read using the utility 'mpstat'.
  31
  32     If 'mpstat' is not installed on the host usage statistics
  33     are instead read directly from '/proc/stat'.
  34
  35     Load averages are read from the file '/proc/loadavg'
  36     on the Linux host.
  37
  38     Parameters
  39           interval - Time interval to measure CPU usage.
  40
  41           type:       [int]
  42           unit:       seconds
  43           default:    1
  44
  45           count (for mpstat only) - Number of CPU usage measurment.
  46
  47           type:       [int]
  48           unit:       N/A
  49           default:    1
  50
  51     """
  52
  53     __scenario_type__ = "CPUload"
  54
  55     MPSTAT_FIELD_SIZE = 10
  56
  57     def __init__(self, scenario_cfg, context_cfg):
  58         """Scenario construction."""
  59         self.scenario_cfg = scenario_cfg
  60         self.context_cfg = context_cfg
  61         self.setup_done = False
  62         self.has_mpstat = False
  63         self.has_count = False
  64
  65     def setup(self):
  66         """Scenario setup."""
  67         host = self.context_cfg['host']
  68         user = host.get('user', 'ubuntu')
  69         ip = host.get('ip', None)
  70         key_filename = host.get('key_filename', '~/.ssh/id_rsa')
  71
  72         LOG.info("user:%s, host:%s", user, ip)
  73         self.client = ssh.SSH(user, ip, key_filename=key_filename)
  74         self.client.wait(timeout=600)
  75
  76         # Check if mpstat prog is installed
  77         status, _, _ = self.client.execute("mpstat -V >/dev/null 2>&1")
  78         if status != 0:
  79             LOG.info("MPSTAT is NOT installed")
  80             self.has_mpstat = False
  81         else:
  82             LOG.info("MPSTAT is installed")
  83             self.has_mpstat = True
  84
  85         options = self.scenario_cfg['options']
  86         self.interval = options.get("interval", 1)
  87         if 'count' in options:
  88             self.count = options.get("count", 1)
  89             self.has_count = True
  90         else:
  91             self.has_count = False
  92
  93         self.setup_done = True
  94
  95     def _execute_command(self, cmd):
  96         """Execute a command on server."""
  97         LOG.info("Executing: %s" % cmd)
  98         status, stdout, stderr = self.client.execute(cmd)
  99         if status != 0:
 100             raise RuntimeError("Failed executing command: ",
 101                                cmd, stderr)
 102         return stdout
 103
 104     def _get_loadavg(self):
 105         """Get system load."""
 106         return {'loadavg': self._execute_command("cat /proc/loadavg").split()}
 107
 108     def _get_cpu_usage_mpstat(self):
 109         """Get processor usage using mpstat."""
 110         if self.interval > 0 and self.has_count:
 111             cmd = "mpstat -P ON %s %s" % (self.interval, self.count)
 112         else:
 113             cmd = "mpstat -P ON %s 1" % self.interval
 114
 115         result = self._execute_command(cmd)
 116
 117         fields = []
 118         maximum = {}
 119         minimum = {}
 120         average = {}
 121
 122         time_marker = re.compile("^([0-9]+):([0-9]+):([0-9]+)$")
 123         ampm_marker = re.compile("(AM|PM)$")
 124
 125         # Parse CPU stats
 126         for row in result.split('\n'):
 127             line = row.split()
 128
 129             if line and re.match(time_marker, line[0]):
 130                 if re.match(ampm_marker, line[1]):
 131                     del line[:2]
 132                 else:
 133                     del line[:1]
 134
 135                 if line[0] == 'CPU':
 136                     # header fields
 137                     fields = line[1:]
 138                     if len(fields) != CPULoad.MPSTAT_FIELD_SIZE:
 139                         raise RuntimeError("mpstat: unexpected field size",
 140                                            fields)
 141                 else:
 142                     # value fields
 143                     cpu = 'cpu' if line[0] == 'all' else 'cpu' + line[0]
 144                     values = line[1:]
 145                     if values and len(values) == len(fields):
 146                         temp_dict = dict(zip(fields, values))
 147                         if cpu not in maximum:
 148                             maximum[cpu] = temp_dict
 149                         else:
 150                             for item in temp_dict:
 151                                 if float(maximum[cpu][item]) <\
 152                                    float(temp_dict[item]):
 153                                     maximum[cpu][item] = temp_dict[item]
 154
 155                         if cpu not in minimum:
 156                             minimum[cpu] = temp_dict
 157                         else:
 158                             for item in temp_dict:
 159                                 if float(minimum[cpu][item]) >\
 160                                    float(temp_dict[item]):
 161                                     minimum[cpu][item] = temp_dict[item]
 162                     else:
 163                         raise RuntimeError("mpstat: parse error", fields, line)
 164
 165             elif line and line[0] == 'Average:':
 166                 del line[:1]
 167                 if line[0] == 'CPU':
 168                     # header fields
 169                     fields = line[1:]
 170                     if len(fields) != CPULoad.MPSTAT_FIELD_SIZE:
 171                         raise RuntimeError("mpstat average: unexpected field\
 172                                            size", fields)
 173                 else:
 174                     # value fields
 175                     cpu = 'cpu' if line[0] == 'all' else 'cpu' + line[0]
 176                     values = line[1:]
 177                     if values and len(values) == len(fields):
 178                         average[cpu] = dict(zip(fields, values))
 179                     else:
 180                         raise RuntimeError("mpstat average: parse error",
 181                                            fields, line)
 182
 183         return {'mpstat_maximun': maximum, 'mpstat_minimum': minimum,
 184                 'mpstat_average': average}
 185
 186     def _get_cpu_usage(self):
 187         """Get processor usage from /proc/stat."""
 188         fields = ['%usr', '%nice', '%sys', '%idle', '%iowait',
 189                   '%irq', '%soft', '%steal', '%guest', '%gnice']
 190
 191         cmd = "grep '^cpu[0-9 ].' /proc/stat"
 192
 193         if self.interval > 0:
 194             previous = self._execute_command(cmd).splitlines()
 195             time.sleep(self.interval)
 196             current = self._execute_command(cmd).splitlines()
 197         else:
 198             current = self._execute_command(cmd).splitlines()
 199             previous = current
 200
 201         mpstat = {}
 202
 203         for (prev, cur) in zip(previous, current):
 204
 205             # Split string to list tokens
 206             cur_list = cur.split()
 207             prev_list = prev.split()
 208
 209             cpu = cur_list[0]
 210
 211             cur_stats = map(int, cur_list[1:])
 212             if self.interval > 0:
 213                 prev_stats = map(int, prev_list[1:])
 214             else:
 215                 prev_stats = [0] * len(cur_stats)
 216
 217             # NB: Don't add 'guest' and 'gnice' as they
 218             # are already included in 'usr' and 'nice'.
 219             uptime_prev = sum(prev_stats[0:8])
 220             uptime_cur = sum(cur_stats[0:8])
 221
 222             # Remove 'guest' and 'gnice' from 'usr' and 'nice'
 223             prev_stats[0] -= prev_stats[8]
 224             prev_stats[1] -= prev_stats[9]
 225             cur_stats[0] -= cur_stats[8]
 226             cur_stats[1] -= cur_stats[9]
 227
 228             # number of samples (jiffies) in the interval
 229             samples = (uptime_cur - uptime_prev) or 1
 230
 231             def _percent(x, y):
 232                 if x < y:
 233                     return 0.0
 234                 else:
 235                     return "%.2f" % (100.0 * (x - y) / samples)
 236
 237             load = map(_percent, cur_stats, prev_stats)
 238
 239             mpstat[cpu] = dict(zip(fields, load))
 240
 241         return {'mpstat': mpstat}
 242
 243     def run(self, result):
 244         """Read processor statistics."""
 245         if not self.setup_done:
 246             self.setup()
 247
 248         result.update(self._get_loadavg())
 249
 250         if self.has_mpstat:
 251             result.update(self._get_cpu_usage_mpstat())
 252         else:
 253             result.update(self._get_cpu_usage())
 254
 255         # Note: No SLA as this scenario is only collecting statistics
 256
 257 # def _test():
 258 #     """internal test function."""
 259 #     import pkg_resources
 260 #     key_filename = pkg_resources.resource_filename('yardstick.resources',
 261 #                                                    'files/yardstick_key')
 262 #     ctx = {
 263 #         'host': {
 264 #             'ip': '172.16.0.175',
 265 #             'user': 'ubuntu',
 266 #             'key_filename': key_filename
 267 #         }
 268 #     }
 269
 270 #     logger = logging.getLogger('yardstick')
 271 #     logger.setLevel(logging.DEBUG)
 272
 273 #     args = {}
 274 #     result = {}
 275
 276 #     p = CPULoad(args, ctx)
 277 #     p.run(result)
 278 #     import json
 279 #     print json.dumps(result)
 280
 281 # if __name__ == '__main__':
 282 #     _test()