def __del__(self):
self.client.close()
- def ssh(self, command):
+ def ssh(self, command, raise_enabled=True):
if self.log:
self.log.info("Executing: %s" % command)
stdin, stdout, stderr = self.client.exec_command(command)
output = list()
for line in stdout.read().splitlines():
output.append(line.decode('utf-8'))
- if ret:
+ if ret and raise_enabled:
if self.log:
self.log.info("*** FAILED to run command %s (%s)"
% (command, ret))
class ApexInstaller(BaseInstaller):
node_user_name = 'heat-admin'
+ installer_username = 'stack'
cm_set_script = 'set_config.py'
nc_set_compute_script = 'set_compute_config.py'
cg_set_script = 'set_congress.py'
cm_restore_script = 'restore_config.py'
nc_restore_compute_script = 'restore_compute_config.py'
cg_restore_script = 'restore_congress.py'
+ ac_restart_script = 'restart_aodh.py'
+ ac_restore_script = 'restore_aodh.py'
+ python = 'python'
def __init__(self, conf, log):
super(ApexInstaller, self).__init__(conf, log)
self.client = SSHClient(self.conf.installer.ip,
- self.conf.installer.username,
+ self.installer_username,
key_filename=self.conf.installer.key_file,
look_for_keys=True)
self.key_file = None
def setup(self):
self.log.info('Setup Apex installer start......')
self.key_file = self.get_ssh_key_from_installer()
- self._get_and_set_ips()
+ self._get_overcloud_conf()
self.create_flavor()
self.set_apply_patches()
self.setup_stunnel()
key_path = '/home/stack/.ssh/id_rsa'
return self._get_ssh_key(self.client, key_path)
- def _get_and_set_ips(self):
- self.log.info('Get controller and compute ips from Apex installer'
+ def _get_overcloud_conf(self):
+ self.log.info('Get overcloud config details from Apex installer'
'......')
command = "source stackrc; nova list | grep ' overcloud-'"
self.controllers.append(ip)
elif 'overcloud-novacompute-' in line:
self.computes.append(ip)
+ command = "grep docker /home/stack/deploy_command"
+ self.use_containers = self._check_cmd_remote(self.client, command)
self.log.info('controller_ips:%s' % self.controllers)
self.log.info('compute_ips:%s' % self.computes)
+ self.log.info('use_containers:%s' % self.use_containers)
def get_host_ip_from_hostname(self, hostname):
self.log.info('Get host ip by hostname=%s from Apex installer......'
def get_transport_url(self):
client = SSHClient(self.controllers[0], self.node_user_name,
key_filename=self.key_file)
+ if self.use_containers:
+ ncbase = "/var/lib/config-data/puppet-generated/nova"
+ else:
+ ncbase = ""
+ command = 'sudo grep "^transport_url" %s/etc/nova/nova.conf' % ncbase
- command = 'sudo grep "^transport_url" /etc/nova/nova.conf'
ret, url = client.ssh(command)
if ret:
raise Exception('Exec command to get host ip from controller(%s)'
self.log.debug('get_transport_url %s' % ret)
return ret
+ def _set_docker_restart_cmd(self, service):
+ # There can be multiple instances running so need to restart all
+ cmd = "for container in `sudo docker ps | grep "
+ cmd += service
+ cmd += " | awk '{print $1}'`; do sudo docker restart $container; \
+ done;"
+ return cmd
+
def set_apply_patches(self):
self.log.info('Set apply patches start......')
- restart_cmd = 'sudo systemctl restart' \
- ' openstack-ceilometer-notification.service'
-
set_scripts = [self.cm_set_script]
+ if self.use_containers:
+ restart_cmd = (self._set_docker_restart_cmd(
+ "ceilometer-notification"))
+ set_scripts.append(self.ac_restart_script)
+ else:
+ restart_cmd = 'sudo systemctl restart' \
+ ' openstack-ceilometer-notification.service'
+
if self.conf.test_case != 'fault_management':
- restart_cmd += ' openstack-nova-scheduler.service'
+ if self.use_containers:
+ restart_cmd += self._set_docker_restart_cmd("nova-scheduler")
+ else:
+ restart_cmd += ' openstack-nova-scheduler.service'
+ set_scripts.append(self.nc_set_compute_script)
if self.conf.inspector.type == Inspector.CONGRESS:
- restart_cmd += ' openstack-congress-server.service'
+ if self.use_containers:
+ restart_cmd += self._set_docker_restart_cmd("congress-server")
+ else:
+ restart_cmd += ' openstack-congress-server.service'
set_scripts.append(self.cg_set_script)
for node_ip in self.controllers:
self.controller_clients.append(client)
self._run_apply_patches(client,
restart_cmd,
- set_scripts)
+ set_scripts,
+ python=self.python)
if self.conf.test_case != 'fault_management':
- restart_cmd = 'sudo systemctl restart' \
- ' openstack-nova-compute.service'
+ if self.use_containers:
+ restart_cmd = self._set_docker_restart_cmd("nova-compute")
+ else:
+ restart_cmd = 'sudo systemctl restart' \
+ ' openstack-nova-compute.service'
for node_ip in self.computes:
client = SSHClient(node_ip, self.node_user_name,
key_filename=self.key_file)
self.compute_clients.append(client)
self._run_apply_patches(client,
restart_cmd,
- [self.nc_set_compute_script])
+ [self.nc_set_compute_script],
+ python=self.python)
if self.conf.test_case != 'fault_management':
time.sleep(10)
def restore_apply_patches(self):
self.log.info('restore apply patches start......')
- restart_cmd = 'sudo systemctl restart' \
- ' openstack-ceilometer-notification.service'
-
restore_scripts = [self.cm_restore_script]
+ if self.use_containers:
+ restart_cmd = (self._set_docker_restart_cmd(
+ "ceilometer-notification"))
+ restore_scripts.append(self.ac_restore_script)
+ else:
+ restart_cmd = 'sudo systemctl restart' \
+ ' openstack-ceilometer-notification.service'
+
if self.conf.test_case != 'fault_management':
- restart_cmd += ' openstack-nova-scheduler.service'
+ if self.use_containers:
+ restart_cmd += self._set_docker_restart_cmd("nova-scheduler")
+ else:
+ restart_cmd += ' openstack-nova-scheduler.service'
+ restore_scripts.append(self.nc_restore_compute_script)
if self.conf.inspector.type == Inspector.CONGRESS:
- restart_cmd += ' openstack-congress-server.service'
+ if self.use_containers:
+ restart_cmd += self._set_docker_restart_cmd("congress-server")
+ else:
+ restart_cmd += ' openstack-congress-server.service'
restore_scripts.append(self.cg_restore_script)
for client in self.controller_clients:
self._run_apply_patches(client,
restart_cmd,
- restore_scripts)
+ restore_scripts,
+ python=self.python)
if self.conf.test_case != 'fault_management':
- restart_cmd = 'sudo systemctl restart' \
- ' openstack-nova-compute.service'
+ if self.use_containers:
+ restart_cmd = self._set_docker_restart_cmd("nova-compute")
+ else:
+ restart_cmd = 'sudo systemctl restart' \
+ ' openstack-nova-compute.service'
for client in self.compute_clients:
self._run_apply_patches(client,
restart_cmd,
- [self.nc_restore_compute_script])
+ [self.nc_restore_compute_script],
+ python=self.python)
self.conf = conf
self.log = log
self.servers = list()
+ self.use_containers = False
@abc.abstractproperty
def node_user_name(self):
% (output, command, self.conf.installer.type))
return output
- def _run_apply_patches(self, client, restart_cmd, script_names):
+ def _check_cmd_remote(self, client, command):
+ self.log.info('Check command=%s return in %s installer......'
+ % (command, self.conf.installer.type))
+
+ ret, output = client.ssh(command, raise_enabled=False)
+ self.log.info('return %s' % ret)
+ if ret == 0:
+ ret = True
+ else:
+ ret = False
+ return ret
+
+ def _run_apply_patches(self, client, restart_cmd, script_names,
+ python='python3'):
installer_dir = os.path.dirname(os.path.realpath(__file__))
if isinstance(script_names, list):
script_abs_path = '{0}/{1}/{2}'.format(installer_dir,
'common', script_name)
client.scp(script_abs_path, script_name)
- cmd = 'sudo python3 %s' % script_name
+ cmd = 'sudo %s %s' % (python, script_name)
ret, output = client.ssh(cmd)
if ret:
- raise Exception('Do the command in controller'
+ raise Exception('Do the command in remote'
' node failed, ret=%s, cmd=%s, output=%s'
% (ret, cmd, output))
client.ssh(restart_cmd)
--- /dev/null
+##############################################################################
+# Copyright (c) 2018 Nokia Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import socket
+import subprocess
+
+
+def restart_aodh_event_alarm():
+ # Restart aodh-evaluator docker with localhost as controller host ip
+ # This makes our alarm sending look the same as without container
+
+ orig_docker_id = subprocess.check_output("docker ps | grep aodh-evaluator "
+ "| awk '{print $1}'", shell=True)
+ get_docker_startup = (
+ 'docker inspect --format=\'{{range .Config.Env}} -e "{{.}}" {{end}} '
+ '{{range .Mounts}} -v {{.Source}}:{{.Destination}}{{if .Mode}}:'
+ '{{.Mode}}{{end}}{{end}} -ti {{.Config.Image}}\''
+ )
+ docker_start = subprocess.check_output("%s %s" % (get_docker_startup,
+ orig_docker_id), shell=True)
+ with open("orig_docker_id", 'w') as oid:
+ oid.write(orig_docker_id)
+ oid.close()
+ subprocess.check_output("docker stop %s" % orig_docker_id, shell=True)
+ ip = socket.gethostbyname(socket.gethostname())
+
+ ae_start = '-d --add-host="localhost:%s" %s' % (ip, docker_start)
+ subprocess.check_output("docker run %s" % ae_start, shell=True)
+ new_docker_id = subprocess.check_output("docker ps | grep aodh-evaluator "
+ " | awk '{print $1}'", shell=True)
+ if orig_docker_id == new_docker_id:
+ raise Exception("Docker ids matching!")
+ with open("new_docker_id", 'w') as nid:
+ nid.write(new_docker_id)
+ nid.close()
+
+restart_aodh_event_alarm()
--- /dev/null
+##############################################################################
+# Copyright (c) 2018 Nokia Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import subprocess
+
+
+def restore_aodh_event_alarm():
+ # Remove modified docker and restore original
+ orig = "orig_docker_id"
+ new = "new_docker_id"
+ if os.path.isfile(orig):
+ with open("orig_docker_id", 'r') as oid:
+ orig_docker_id = oid.read()
+ oid.close()
+ if os.path.isfile(new):
+ with open("new_docker_id", 'r') as nid:
+ new_docker_id = nid.read()
+ nid.close()
+ subprocess.check_output("docker stop %s" % new_docker_id,
+ shell=True)
+ subprocess.check_output("docker rm %s" % new_docker_id, shell=True)
+ os.remove(new)
+ subprocess.check_output("docker start %s" % orig_docker_id, shell=True)
+ os.remove(orig)
+
+restore_aodh_event_alarm()
def restore_cpu_allocation_ratio():
- nova_file = '/etc/nova/nova.conf'
- nova_file_bak = '/etc/nova/nova.bak'
+ nova_base = "/var/lib/config-data/puppet-generated/nova"
+ if not os.path.isdir(nova_base):
+ nova_base = ""
+ nova_file = nova_base + '/etc/nova/nova.conf'
+ nova_file_bak = nova_base + '/etc/nova/nova.bak'
if not os.path.isfile(nova_file_bak):
print('Bak_file:%s does not exist.' % nova_file_bak)
import os
import shutil
-ep_file = '/etc/ceilometer/event_pipeline.yaml'
-ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
+
+cbase = "/var/lib/config-data/puppet-generated/ceilometer"
+if not os.path.isdir(cbase):
+ cbase = ""
def restore_ep_config():
+ ep_file = cbase + '/etc/ceilometer/event_pipeline.yaml'
+ ep_file_bak = cbase + '/etc/ceilometer/event_pipeline.yaml.bak'
if not os.path.isfile(ep_file_bak):
print('Bak_file:%s does not exist.' % ep_file_bak)
def restore_ed_config():
-
- ed_file = '/etc/ceilometer/event_definitions.yaml'
- ed_file_bak = '/etc/ceilometer/event_definitions.bak'
+ ed_file = cbase + '/etc/ceilometer/event_definitions.yaml'
+ ed_file_bak = cbase + '/etc/ceilometer/event_definitions.bak'
if not os.path.isfile(ed_file_bak):
print("Bak_file doesn't exist: %s." % ed_file_bak)
else:
print('restore: %s' % ed_file)
- shutil.copyfile(ed_file_bak, ed_file)
+ if os.stat(ed_file_bak).st_size == 0:
+ print('Bak_file empty, so removing also: %s' % ed_file)
+ os.remove(ed_file)
+ else:
+ shutil.copyfile(ed_file_bak, ed_file)
os.remove(ed_file_bak)
return
-
-def restore_cpu_allocation_ratio():
- nova_file = '/etc/nova/nova.conf'
- nova_file_bak = '/etc/nova/nova.bak'
-
- if not os.path.isfile(nova_file_bak):
- print('Bak_file:%s does not exist.' % nova_file_bak)
- else:
- print('restore: %s' % nova_file)
- shutil.copyfile(nova_file_bak, nova_file)
- os.remove(nova_file_bak)
- return
-
restore_ep_config()
restore_ed_config()
-restore_cpu_allocation_ratio()
def restore_drivers_config():
- co_conf = "/etc/congress/congress.conf"
- co_conf_bak = "/etc/congress/congress.conf.bak"
+ co_base = "/var/lib/config-data/puppet-generated/congress"
+ if not os.path.isdir(co_base):
+ co_base = ""
+ co_conf = co_base + "/etc/congress/congress.conf"
+ co_conf_bak = co_base + "/etc/congress/congress.conf.bak"
if not os.path.isfile(co_conf_bak):
print('Bak_file:%s does not exist.' % co_conf_bak)
import shutil
+def make_initial_config(service, dest):
+ for mk in ["", "/etc", "/%s" % service]:
+ dest += mk
+ os.mkdir(dest)
+ src = "/etc/%s/%s.conf" % (service, service)
+ dest += "/%s.conf" % service
+ shutil.copyfile(src, dest)
+
+
def set_cpu_allocation_ratio():
- nova_file = '/etc/nova/nova.conf'
- nova_file_bak = '/etc/nova/nova.bak'
+ docker_conf_base_dir = "/var/lib/config-data/puppet-generated"
+ if not os.path.isdir(docker_conf_base_dir):
+ nova_base = ""
+ else:
+ nova_base = "%s/nova" % docker_conf_base_dir
+ if not os.path.isdir(nova_base):
+ # nova.conf to be used might not exist
+ make_initial_config("nova", nova_base)
+ nova_file = nova_base + '/etc/nova/nova.conf'
+ nova_file_bak = nova_base + '/etc/nova/nova.bak'
if not os.path.isfile(nova_file):
raise Exception("File doesn't exist: %s." % nova_file)
import shutil
import yaml
-ep_file = '/etc/ceilometer/event_pipeline.yaml'
-ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
-event_notifier_topic = 'notifier://?topic=alarm.all'
+
+cbase = "/var/lib/config-data/puppet-generated/ceilometer"
+if not os.path.isdir(cbase):
+ cbase = ""
def set_notifier_topic():
+ ep_file = cbase + '/etc/ceilometer/event_pipeline.yaml'
+ ep_file_bak = cbase + '/etc/ceilometer/event_pipeline.yaml.bak'
+ event_notifier_topic = 'notifier://?topic=alarm.all'
config_modified = False
if not os.path.isfile(ep_file):
file.write(yaml.safe_dump(config))
-def set_maintenance_event_definitions():
- ed_file = '/etc/ceilometer/event_definitions.yaml'
- ed_file_bak = '/etc/ceilometer/event_definitions.bak'
+def set_event_definitions():
+ ed_file = cbase + '/etc/ceilometer/event_definitions.yaml'
+ ed_file_bak = cbase + '/etc/ceilometer/event_definitions.bak'
+ orig_ed_file_exist = True
+ modify_config = False
if not os.path.isfile(ed_file):
- raise Exception("File doesn't exist: %s." % ed_file)
-
- with open(ed_file, 'r') as file:
- config = yaml.safe_load(file)
+ # Deployment did not modify file, so it did not exist
+ src_file = '/etc/ceilometer/event_definitions.yaml'
+ if not os.path.isfile(src_file):
+ config = []
+ orig_ed_file_exist = False
+ else:
+ shutil.copyfile('/etc/ceilometer/event_definitions.yaml', ed_file)
+ if orig_ed_file_exist:
+ with open(ed_file, 'r') as file:
+ config = yaml.safe_load(file)
et_list = [et['event_type'] for et in config]
+ if 'compute.instance.update' in et_list:
+ print('NOTE: compute.instance.update allready configured')
+ else:
+ print('NOTE: add compute.instance.update to event_definitions.yaml')
+ modify_config = True
+ instance_update = {
+ 'event_type': 'compute.instance.update',
+ 'traits': {
+ 'deleted_at': {'fields': 'payload.deleted_at',
+ 'type': 'datetime'},
+ 'disk_gb': {'fields': 'payload.disk_gb',
+ 'type': 'int'},
+ 'display_name': {'fields': 'payload.display_name'},
+ 'ephemeral_gb': {'fields': 'payload.ephemeral_gb',
+ 'type': 'int'},
+ 'host': {'fields': 'publisher_id.`split(., 1, 1)`'},
+ 'instance_id': {'fields': 'payload.instance_id'},
+ 'instance_type': {'fields': 'payload.instance_type'},
+ 'instance_type_id': {'fields': 'payload.instance_type_id',
+ 'type': 'int'},
+ 'launched_at': {'fields': 'payload.launched_at',
+ 'type': 'datetime'},
+ 'memory_mb': {'fields': 'payload.memory_mb',
+ 'type': 'int'},
+ 'old_state': {'fields': 'payload.old_state'},
+ 'os_architecture': {
+ 'fields':
+ "payload.image_meta.'org.openstack__1__architecture'"},
+ 'os_distro': {
+ 'fields':
+ "payload.image_meta.'org.openstack__1__os_distro'"},
+ 'os_version': {
+ 'fields':
+ "payload.image_meta.'org.openstack__1__os_version'"},
+ 'resource_id': {'fields': 'payload.instance_id'},
+ 'root_gb': {'fields': 'payload.root_gb',
+ 'type': 'int'},
+ 'service': {'fields': 'publisher_id.`split(., 0, -1)`'},
+ 'state': {'fields': 'payload.state'},
+ 'tenant_id': {'fields': 'payload.tenant_id'},
+ 'user_id': {'fields': 'payload.user_id'},
+ 'vcpus': {'fields': 'payload.vcpus', 'type': 'int'}
+ }
+ }
+ config.append(instance_update)
+
if 'maintenance.scheduled' in et_list:
- add_mscheduled = False
print('NOTE: maintenance.scheduled allready configured')
else:
print('NOTE: add maintenance.scheduled to event_definitions.yaml')
- add_mscheduled = True
+ modify_config = True
mscheduled = {
'event_type': 'maintenance.scheduled',
'traits': {
'session_id': {'fields': 'payload.session_id'},
'project_id': {'fields': 'payload.project_id'},
'metadata': {'fields': 'payload.metadata'}
+ }
}
- }
config.append(mscheduled)
if 'maintenance.host' in et_list:
- add_mhost = False
print('NOTE: maintenance.host allready configured')
else:
print('NOTE: add maintenance.host to event_definitions.yaml')
- add_mhost = True
+ modify_config = True
mhost = {
'event_type': 'maintenance.host',
'traits': {
'project_id': {'fields': 'payload.project_id'},
'state': {'fields': 'payload.state'},
'session_id': {'fields': 'payload.session_id'}
+ }
}
- }
config.append(mhost)
- if add_mscheduled or add_mhost:
- shutil.copyfile(ed_file, ed_file_bak)
+ if modify_config:
+ if orig_ed_file_exist:
+ shutil.copyfile(ed_file, ed_file_bak)
+ else:
+ with open(ed_file_bak, 'w+') as file:
+ file.close()
with open(ed_file, 'w+') as file:
file.write(yaml.safe_dump(config))
-
-def set_cpu_allocation_ratio():
- nova_file = '/etc/nova/nova.conf'
- nova_file_bak = '/etc/nova/nova.bak'
-
- if not os.path.isfile(nova_file):
- raise Exception("File doesn't exist: %s." % nova_file)
- # TODO (tojuvone): Unfortunately ConfigParser did not produce working conf
- fcheck = open(nova_file)
- found_list = ([ca for ca in fcheck.readlines() if "cpu_allocation_ratio"
- in ca])
- fcheck.close()
- if found_list and len(found_list):
- change = False
- found = False
- for car in found_list:
- if car.startswith('#'):
- continue
- if car.startswith('cpu_allocation_ratio'):
- found = True
- if "1.0" not in car.split('=')[1]:
- change = True
- if not found or change:
- # need to add or change
- shutil.copyfile(nova_file, nova_file_bak)
- fin = open(nova_file_bak)
- fout = open(nova_file, "wt")
- for line in fin:
- if change and line.startswith("cpu_allocation_ratio"):
- line = "cpu_allocation_ratio=1.0"
- if not found and line.startswith("[DEFAULT]"):
- line += "cpu_allocation_ratio=1.0\n"
- fout.write(line)
- fin.close()
- fout.close()
-
set_notifier_topic()
-set_maintenance_event_definitions()
-set_cpu_allocation_ratio()
+set_event_definitions()
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
import configparser
+import os
import shutil
def set_drivers_config():
- co_conf = "/etc/congress/congress.conf"
- co_conf_bak = "/etc/congress/congress.conf.bak"
+ co_base = "/var/lib/config-data/puppet-generated/congress"
+ if not os.path.isdir(co_base):
+ co_base = ""
+ co_conf = co_base + "/etc/congress/congress.conf"
+ co_conf_bak = co_base + "/etc/congress/congress.conf.bak"
doctor_driver = "congress.datasources.doctor_driver.DoctorDriver"
config_modified = False
import os
+vi_base = "/var/lib/config-data/puppet-generated/vitrage"
+if not os.path.isdir(vi_base):
+ vi_base = ""
vitrage_template_file = \
- '/etc/vitrage/templates/vitrage_host_down_scenarios.yaml'
+ vi_base + '/etc/vitrage/templates/vitrage_host_down_scenarios.yaml'
template = """
metadata:
return ret.json()['state']
def wait_maintenance_complete(self, session_id):
- retries = 60
+ retries = 66
state = None
- time.sleep(600)
+ time.sleep(540)
while state != 'MAINTENANCE_COMPLETE' and retries > 0:
time.sleep(10)
state = self.get_maintenance_state(session_id)
action_failed = '%s_FAILED' % action
status = action_in_progress
- stack_retries = 150
+ stack_retries = 160
while status == action_in_progress and stack_retries > 0:
time.sleep(2)
try:
template=template,
parameters=parameters)
self.stack_id = stack['stack']['id']
- self.wait_stack_create()
+ try:
+ self.wait_stack_create()
+ except Exception:
+ # It might not always work at first
+ self.log.info('retry creating maintenance stack.......')
+ self.delete()
+ time.sleep(3)
+ stack = self.heat.stacks.create(stack_name=self.stack_name,
+ files=files,
+ template=template,
+ parameters=parameters)
+ self.stack_id = stack['stack']['id']
+ self.wait_stack_create()
def update(self, stack_name, stack_id, template, parameters={}, files={}):
self.heat.stacks.update(stack_name=stack_name,
Code Review / doctor.git / commitdiff