#alarm_basename = doctor_alarm
+#profiler_type = poc
+
[installer]
#type = local
#ip = 127.0.0.1
import json
import os
import paramiko
+import re
def load_json_file(full_path):
file.write(json.dumps(data))
+def match_rep_in_file(regex, full_path):
+ if not os.path.isfile(full_path):
+ raise Exception('File(%s) does not exist' % full_path)
+
+ with open(full_path, 'r') as file:
+ for line in file:
+ result = re.search(regex, line)
+ if result:
+ return result
+
+ return None
+
+
class SSHClient(object):
def __init__(self, ip, username, password=None, pkey=None,
key_filename=None, log=None, look_for_keys=False,
import inspector\r
import monitor\r
import os_clients\r
+import profiler_poc\r
import user\r
\r
\r
user.OPTS,\r
network.OPTS,\r
instance.OPTS,\r
- alarm.OPTS))\r
+ alarm.OPTS,\r
+ profiler_poc.OPTS))\r
]\r
\r
\r
from alarm import Alarm
from common.constants import Host
+from common.utils import match_rep_in_file
import config
from consumer import get_consumer
from identity_auth import get_identity_auth
from network import Network
from monitor import get_monitor
from os_clients import nova_client
+from profiler_poc import main as profiler_main
from scenario.common import calculate_notification_time
from scenario.network_failure import NetworkFault
from user import User
else:
LOG.error('doctor test failed, notification_time=%s' % notification_time)
sys.exit(1)
+
+ if self.conf.profiler_type:
+ LOG.info('doctor test begin to run profile.......')
+ self.collect_logs()
+ self.run_profiler()
except Exception as e:
LOG.error('doctor test failed, Exception=%s' % e)
sys.exit(1)
time.sleep(2)
self.check_host_status(self.down_host.name, 'up')
+ def collect_logs(self):
+ self.fault.get_disable_network_log()
+
+ def run_profiler(self):
+
+ log_file = '{0}/{1}'.format(sys.path[0], 'disable_network.log')
+ reg = '(?<=doctor set link down at )\d+.\d+'
+ linkdown = float(match_rep_in_file(reg, log_file).group(0))
+
+ log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log')
+ reg = '(.* doctor mark vm.* error at )(\d+.\d+)'
+ vmdown = float(match_rep_in_file(reg, log_file).group(2))
+
+ reg = '(?<=doctor mark host.* down at )\d+.\d+'
+ hostdown = float(match_rep_in_file(reg, log_file).group(2))
+
+ reg = '(?<=doctor monitor detected at )\d+.\d+'
+ detected = float(match_rep_in_file(reg, log_file).group(0))
+
+ reg = '(?<=doctor consumer notified at )\d+.\d+'
+ notified = float(match_rep_in_file(reg, log_file).group(0))
+
+ # TODO(yujunz) check the actual delay to verify time sync status
+ # expected ~1s delay from $trigger to $linkdown
+ relative_start = linkdown
+ os.environ['DOCTOR_PROFILER_T00'] = str(int((linkdown - relative_start)*1000))
+ os.environ['DOCTOR_PROFILER_T01'] = str(int((detected - relative_start) * 1000))
+ os.environ['DOCTOR_PROFILER_T03'] = str(int((vmdown - relative_start) * 1000))
+ os.environ['DOCTOR_PROFILER_T04'] = str(int((hostdown - relative_start) * 1000))
+ os.environ['DOCTOR_PROFILER_T09'] = str(int((notified - relative_start) * 1000))
+
+ profiler_main(log=LOG)
+
def cleanup(self):
self.unset_forced_down_hosts()
self.inspector.stop()
import json
import os
+from oslo_config import cfg
+
+
+OPTS = [
+ cfg.StrOpt('profiler_type',
+ default=os.environ.get('PROFILER_TYPE', 'poc'),
+ help='the type of installer'),
+]
+
+
OUTPUT = 'doctor_profiling_output'
PREFIX = 'DOCTOR_PROFILER'
TOTAL_CHECK_POINTS = 10
"""
-def main():
+def main(log=None):
check_points = ["T{:02d}".format(i) for i in range(TOTAL_CHECK_POINTS)]
module_map = {"M{:02d}".format(i):
(MODULE_CHECK_POINTS[i], MODULE_CHECK_POINTS[i + 1])
def format_tag(tag):
return TAG_FORMAT.format(tag or '?')
- tags = {cp: format_tag(ms) for cp, ms in elapsed_ms.iteritems()}
+ tags = {cp: format_tag(ms) for cp, ms in elapsed_ms.items()}
def time_cost(cp):
if elapsed_ms[cp[0]] and elapsed_ms[cp[1]]:
# module time cost tags
modules_cost_ms = {module: time_cost(cp)
- for module, cp in module_map.iteritems()}
+ for module, cp in module_map.items()}
tags.update({module: format_tag(cost)
- for module, cost in modules_cost_ms.iteritems()})
+ for module, cost in modules_cost_ms.items()})
tags.update({'total': time_cost((check_points[0], check_points[-1]))})
logfile = open('{}.json'.format(OUTPUT), 'w')
logfile.write(json.dumps(tags))
- print profile
+ print(profile)
+ if log:
+ log.info('%s' % profile)
+
if __name__ == '__main__':
main()
export DOCTOR_PROFILER_T09=$(python -c \
"print(int(($notified-$relative_start)*1000))")
- python profiler-poc.py > doctor_profiler.log 2>&1
+ python profiler_poc.py > doctor_profiler.log 2>&1
fi
}
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-import os
-import re
import sys
-
-
-def match_rep_in_file(regex, full_path):
- if not os.path.isfile(full_path):
- raise Exception('File(%s) does not exist' % full_path)
-
- with open(full_path, 'r') as file:
- for line in file:
- result = re.search(regex, line)
- if result:
- return result.group(0)
-
- return None
+from common.utils import match_rep_in_file
def calculate_notification_time():
log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log')
reg = '(?<=doctor monitor detected at )\d+.\d+'
- detected = match_rep_in_file(reg, log_file)
- if not detected:
- raise Exception('Can not find detected time')
+ result = match_rep_in_file(reg, log_file)
+ if not result:
+ raise Exception('Can not match detected time')
+ detected = result.group(0)
reg = '(?<=doctor consumer notified at )\d+.\d+'
- notified = match_rep_in_file(reg, log_file)
- if not notified:
- raise Exception('Can not find notified time')
+ result = match_rep_in_file(reg, log_file)
+ if not result:
+ raise Exception('Can not match notified time')
+ notified = result.group(0)
return float(notified) - float(detected)
\ No newline at end of file
def cleanup(self):
self.log.info('fault inject cleanup......')
- self.get_diable_network_log()
+ self.get_disable_network_log()
- def get_diable_network_log(self):
- if not self.GetLog:
+ def get_disable_network_log(self):
+ if self.GetLog:
self.log.info('Already get the disable_netork.log from down_host......')
return
if self.host is not None:
CI_DEBUG
INSTALLER_TYPE
INSTALLER_IP
+ PROFILER_TYPE
changedir = {toxinidir}/tests
commands = python main.py