X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=yardstick%2Fbenchmark%2Fscenarios%2Fstorage%2Ffio.py;h=c57c6edf2eef4a99c545a3d73626eedbe7e139c3;hb=ce1cdc595a7c386cfa02fa03fb182eead3b92661;hp=b99e3427081492408924ba11d22940e017dda0f3;hpb=15d807c22f33fab8e1f9b61e1d0c041a8144ee89;p=yardstick.git diff --git a/yardstick/benchmark/scenarios/storage/fio.py b/yardstick/benchmark/scenarios/storage/fio.py index b99e34270..c57c6edf2 100644 --- a/yardstick/benchmark/scenarios/storage/fio.py +++ b/yardstick/benchmark/scenarios/storage/fio.py @@ -28,6 +28,18 @@ class Fio(base.Scenario): type: string unit: na default: /home/ubuntu/data.raw + job_file - fio job configuration file + type: string + unit: na + default: None + job_file_config - content of job configuration file + type: list + unit: na + default: None + directory - mount directoey for test volume + type: string + unit: na + default: None bs - block size used for the io units type: int unit: bytes @@ -71,20 +83,57 @@ class Fio(base.Scenario): def __init__(self, scenario_cfg, context_cfg): self.scenario_cfg = scenario_cfg self.context_cfg = context_cfg + self.options = self.scenario_cfg["options"] self.setup_done = False def setup(self): """scenario setup""" - self.target_script = pkg_resources.resource_filename( - "yardstick.benchmark.scenarios.storage", - Fio.TARGET_SCRIPT) host = self.context_cfg["host"] self.client = ssh.SSH.from_node(host, defaults={"user": "root"}) self.client.wait(timeout=600) - # copy script to host - self.client._put_file_shell(self.target_script, '~/fio.sh') + self.job_file = self.options.get("job_file", None) + config_lines = self.options.get("job_file_config", None) + + if self.job_file: + self.job_file_script = pkg_resources.resource_filename( + "yardstick.resources", 'files/' + self.job_file) + + # copy job file to host + self.client._put_file_shell(self.job_file_script, '~/job_file.ini') + elif config_lines: + LOG.debug("Job file configuration received, Fio job file will be created.") + self.job_file = 'tmp_job_file.ini' + self.job_file_script = pkg_resources.resource_filename( + "yardstick.resources", 'files/' + self.job_file) + with open(self.job_file_script, 'w') as f: + f.write('\n'.join(str(line) for line in config_lines)) + + # copy job file to host + self.client._put_file_shell(self.job_file_script, '~/job_file.ini') + else: + LOG.debug("No job file configuration received, Fio will use parameters.") + self.target_script = pkg_resources.resource_filename( + "yardstick.benchmark.scenarios.storage", Fio.TARGET_SCRIPT) + + # copy script to host + self.client._put_file_shell(self.target_script, '~/fio.sh') + + mount_dir = self.options.get("directory", None) + + if mount_dir: + LOG.debug("Formating volume...") + _, stdout, _ = self.client.execute( + "lsblk -dps | grep -m 1 disk | awk '{print $1}'") + block_device = stdout.strip() + if block_device: + self.client.execute("sudo mkfs.ext4 %s" % block_device) + cmd = "sudo mkdir %s" % mount_dir + self.client.execute(cmd) + LOG.debug("Mounting volume at: %s", mount_dir) + cmd = "sudo mount %s %s" % (block_device, mount_dir) + self.client.execute(cmd) self.setup_done = True @@ -92,57 +141,69 @@ class Fio(base.Scenario): """execute the benchmark""" default_args = "-ioengine=libaio -group_reporting -time_based -time_based " \ "--output-format=json" + timeout = 3600 if not self.setup_done: self.setup() - options = self.scenario_cfg["options"] - filename = options.get("filename", "/home/ubuntu/data.raw") - bs = options.get("bs", "4k") - iodepth = options.get("iodepth", "1") - rw = options.get("rw", "write") - ramp_time = options.get("ramp_time", 20) - size = options.get("size", "1g") - direct = options.get("direct", "1") - numjobs = options.get("numjobs", "1") - rwmixwrite = options.get("rwmixwrite", 50) - name = "yardstick-fio" - # if run by a duration runner - duration_time = self.scenario_cfg["runner"].get("duration", None) \ - if "runner" in self.scenario_cfg else None - # if run by an arithmetic runner - arithmetic_time = options.get("duration", None) - if duration_time: - runtime = duration_time - elif arithmetic_time: - runtime = arithmetic_time + if self.job_file: + cmd = "sudo fio job_file.ini --output-format=json" else: - runtime = 30 + filename = self.options.get("filename", "/home/ubuntu/data.raw") + bs = self.options.get("bs", "4k") + iodepth = self.options.get("iodepth", "1") + rw = self.options.get("rw", "write") + ramp_time = self.options.get("ramp_time", 20) + size = self.options.get("size", "1g") + direct = self.options.get("direct", "1") + numjobs = self.options.get("numjobs", "1") + rwmixwrite = self.options.get("rwmixwrite", 50) + name = "yardstick-fio" + # if run by a duration runner + duration_time = self.scenario_cfg["runner"].get("duration", None) \ + if "runner" in self.scenario_cfg else None + # if run by an arithmetic runner + arithmetic_time = self.options.get("duration", None) + if duration_time: + runtime = duration_time + elif arithmetic_time: + runtime = arithmetic_time + else: + runtime = 30 + # Set timeout, so that the cmd execution does not exit incorrectly + # when the test run time is last long + timeout = int(ramp_time) + int(runtime) + 600 + + cmd_args = "-filename=%s -direct=%s -bs=%s -iodepth=%s -rw=%s -rwmixwrite=%s " \ + "-size=%s -ramp_time=%s -numjobs=%s -runtime=%s -name=%s %s" \ + % (filename, direct, bs, iodepth, rw, rwmixwrite, size, ramp_time, numjobs, + runtime, name, default_args) + cmd = "sudo bash fio.sh %s %s" % (filename, cmd_args) - cmd_args = "-filename=%s -direct=%s -bs=%s -iodepth=%s -rw=%s -rwmixwrite=%s " \ - "-size=%s -ramp_time=%s -numjobs=%s -runtime=%s -name=%s %s" \ - % (filename, direct, bs, iodepth, rw, rwmixwrite, size, ramp_time, numjobs, - runtime, name, default_args) - cmd = "sudo bash fio.sh %s %s" % (filename, cmd_args) LOG.debug("Executing command: %s", cmd) - # Set timeout, so that the cmd execution does not exit incorrectly - # when the test run time is last long - timeout = int(ramp_time) + int(runtime) + 600 status, stdout, stderr = self.client.execute(cmd, timeout=timeout) if status: raise RuntimeError(stderr) raw_data = jsonutils.loads(stdout) - # The bandwidth unit is KB/s, and latency unit is us - if rw in ["read", "randread", "rw", "randrw"]: + if self.job_file: result["read_bw"] = raw_data["jobs"][0]["read"]["bw"] result["read_iops"] = raw_data["jobs"][0]["read"]["iops"] result["read_lat"] = raw_data["jobs"][0]["read"]["lat"]["mean"] - if rw in ["write", "randwrite", "rw", "randrw"]: result["write_bw"] = raw_data["jobs"][0]["write"]["bw"] result["write_iops"] = raw_data["jobs"][0]["write"]["iops"] result["write_lat"] = raw_data["jobs"][0]["write"]["lat"]["mean"] + else: + # The bandwidth unit is KB/s, and latency unit is us + if rw in ["read", "randread", "rw", "randrw"]: + result["read_bw"] = raw_data["jobs"][0]["read"]["bw"] + result["read_iops"] = raw_data["jobs"][0]["read"]["iops"] + result["read_lat"] = raw_data["jobs"][0]["read"]["lat"]["mean"] + if rw in ["write", "randwrite", "rw", "randrw"]: + result["write_bw"] = raw_data["jobs"][0]["write"]["bw"] + result["write_iops"] = raw_data["jobs"][0]["write"]["iops"] + result["write_lat"] = raw_data["jobs"][0]["write"]["lat"]["mean"] if "sla" in self.scenario_cfg: sla_error = "" @@ -162,7 +223,7 @@ class Fio(base.Scenario): sla_error += "%s %d < " \ "sla:%s(%d); " % (k, v, k, min_v) - assert sla_error == "", sla_error + self.verify_SLA(sla_error == "", sla_error) def _test():