1 ##############################################################################
2 # Copyright (c) 2018 Nokia Corporation and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
10 from flask import Flask
11 from flask import request
13 from novaclient.exceptions import BadRequest
14 import oslo_messaging as messaging
17 from threading import Thread
18 from traceback import format_exc
19 from uuid import uuid1 as generate_uuid
21 from doctor_tests.admin_tool.base import BaseAdminTool
22 from doctor_tests.identity_auth import get_identity_auth
23 from doctor_tests.identity_auth import get_session
24 from doctor_tests.os_clients import aodh_client
25 from doctor_tests.os_clients import nova_client
28 class SampleAdminTool(BaseAdminTool):
30 def __init__(self, trasport_url, conf, log):
31 super(SampleAdminTool, self).__init__(conf, log)
32 self.trasport_url = trasport_url
36 self.log.info('sample admin tool start......')
37 self.app = AdminTool(self.trasport_url, self.conf, self, self.log)
41 self.log.info('sample admin tool stop......')
45 'Content-Type': 'application/json',
46 'Accept': 'application/json',
48 url = 'http://%s:%d/shutdown'\
49 % (self.conf.admin_tool.ip,
50 self.conf.admin_tool.port)
51 requests.post(url, data='', headers=headers)
54 class AdminMain(Thread):
56 def __init__(self, trasport_url, session_id, data, parent, conf, log):
58 self.session_id = session_id
62 self.url = 'http://0.0.0.0:%s' % conf.admin_tool.port
63 self.projects_state = dict() # current state for each project
64 self.proj_server_actions = dict() # actions for each project server
65 self.projects_servers = dict() # servers processed in current state
66 self.maint_proj_servers = dict() # servers under whole maintenance
67 self.hosts = data['hosts']
68 self.maintenance_at = data['maintenance_at']
69 self.computes_disabled = list()
70 self.metadata = data['metadata']
71 self.auth = get_identity_auth(project=self.conf.doctor_project)
72 self.state = data['state']
73 self.aodh = aodh_client(self.conf.aodh_version,
74 get_session(auth=self.auth))
75 self.nova = nova_client(self.conf.nova_version,
76 get_session(auth=self.auth))
77 self.log.info('transport_url %s' % trasport_url)
78 transport = messaging.get_transport(self.conf, trasport_url)
79 self.notif_proj = messaging.Notifier(transport,
80 'maintenance.planned',
82 topics=['notifications'])
83 self.notif_proj = self.notif_proj.prepare(publisher_id='admin_tool')
84 self.notif_admin = messaging.Notifier(transport,
87 topics=['notifications'])
88 self.notif_admin = self.notif_admin.prepare(publisher_id='admin_tool')
89 self.log.info('Admin tool session %s initialized' % self.session_id)
92 for host in self.computes_disabled:
93 self.log.info('enable nova-compute on %s' % host)
94 self.nova.services.enable(host, 'nova-compute')
96 def _projects_not_in_wanted_states(self, wanted_states):
97 if len([v for v in self.projects_state.values()
98 if v not in wanted_states]):
103 def projects_not_in_state(self, state):
104 if len([v for v in self.projects_state.values()
110 def wait_projects_state(self, wanted_states, wait_seconds):
111 retries = wait_seconds
112 while (retries > 0 and
113 self._projects_not_in_wanted_states(wanted_states)):
115 retries = retries - 1
116 if self._projects_not_in_wanted_states(wanted_states):
117 self.log.error('Admin tool session %s: projects in invalid states '
118 '%s' % (self.session_id, self.projects_state))
119 raise Exception('Admin tool session %s: not all projects in states'
120 ' %s' % (self.session_id, wanted_states))
122 self.log.info('all projects replied')
124 def _project_notify(self, project_id, instance_ids, allowed_actions,
125 actions_at, state, metadata):
126 reply_url = '%s/%s/maintenance' % (self.url, project_id)
128 payload = dict(project_id=project_id,
129 instance_ids=instance_ids,
130 allowed_actions=allowed_actions,
132 actions_at=actions_at,
133 session_id=self.session_id,
137 self.log.debug('Sending "maintenance.planned" to project: %s' %
140 self.notif_proj.info({'some': 'context'}, 'maintenance.scheduled',
143 def _admin_notify(self, project, host, state, session_id):
144 payload = dict(project_id=project, host=host, state=state,
145 session_id=session_id)
147 self.log.debug('Sending "maintenance.host": %s' % payload)
149 self.notif_admin.info({'some': 'context'}, 'maintenance.host', payload)
151 def down_scale(self):
152 for project in self.projects_servers:
153 self.log.info('DOWN_SCALE to project %s' % project)
154 self.log.debug('instance_ids %s' % self.projects_servers[project])
155 instance_ids = '%s/%s/maintenance' % (self.url, project)
158 actions_at = (datetime.datetime.utcnow() +
159 datetime.timedelta(seconds=wait_seconds)
160 ).strftime('%Y-%m-%d %H:%M:%S')
162 metadata = self.metadata
163 self._project_notify(project, instance_ids,
164 allowed_actions, actions_at, state,
166 allowed_states = ['ACK_DOWN_SCALE', 'NACK_DOWN_SCALE']
167 self.wait_projects_state(allowed_states, wait_seconds)
168 if self.projects_not_in_state('ACK_DOWN_SCALE'):
169 raise Exception('Admin tool session %s: all states not '
170 'ACK_DOWN_SCALE %s' %
171 (self.session_id, self.projects_state))
173 def maintenance(self):
174 for project in self.projects_servers:
175 self.log.info('\nMAINTENANCE to project %s\n' % project)
176 self.log.debug('instance_ids %s' % self.projects_servers[project])
177 instance_ids = '%s/%s/maintenance' % (self.url, project)
179 actions_at = self.maintenance_at
181 metadata = self.metadata
182 maint_at = self.str_to_datetime(self.maintenance_at)
183 td = maint_at - datetime.datetime.utcnow()
184 wait_seconds = int(td.total_seconds())
185 if wait_seconds < 10:
186 raise Exception('Admin tool session %s: No time for project to'
188 (self.session_id, wait_seconds))
189 self._project_notify(project, instance_ids,
190 allowed_actions, actions_at, state,
192 allowed_states = ['ACK_MAINTENANCE', 'NACK_MAINTENANCE']
193 self.wait_projects_state(allowed_states, wait_seconds)
194 if self.projects_not_in_state('ACK_MAINTENANCE'):
195 raise Exception('Admin tool session %s: all states not '
196 'ACK_MAINTENANCE %s' %
197 (self.session_id, self.projects_state))
199 def maintenance_complete(self):
200 for project in self.projects_servers:
201 self.log.info('MAINTENANCE_COMPLETE to project %s' % project)
202 instance_ids = '%s/%s/maintenance' % (self.url, project)
205 actions_at = (datetime.datetime.utcnow() +
206 datetime.timedelta(seconds=wait_seconds)
207 ).strftime('%Y-%m-%d %H:%M:%S')
208 state = 'MAINTENANCE_COMPLETE'
209 metadata = self.metadata
210 self._project_notify(project, instance_ids,
211 allowed_actions, actions_at, state,
213 allowed_states = ['ACK_MAINTENANCE_COMPLETE',
214 'NACK_MAINTENANCE_COMPLETE']
215 self.wait_projects_state(allowed_states, wait_seconds)
216 if self.projects_not_in_state('ACK_MAINTENANCE_COMPLETE'):
217 raise Exception('Admin tool session %s: all states not '
218 'ACK_MAINTENANCE_COMPLETE %s' %
219 (self.session_id, self.projects_state))
221 def need_down_scale(self, host_servers):
222 room_for_instances = 0
223 for host in host_servers:
225 for project in host_servers[host]:
226 for instance in host_servers[host][project]:
228 room_for_instances += (2 - instances)
229 self.log.info('there is room for %d instances' % room_for_instances)
230 if room_for_instances > 1:
235 def find_host_to_be_empty(self, host_servers):
236 host_to_be_empty = None
237 host_nonha_instances = 0
238 for host in host_servers:
241 for project in host_servers[host]:
242 for instance in host_servers[host][project]:
243 if ('doctor_ha_app_' in
244 host_servers[host][project][instance]):
248 self.log.info('host %s has %d ha and %d non ha instances' %
249 (host, ha_instances, nonha_instances))
250 if ha_instances == 0:
252 if nonha_instances < host_nonha_instances:
253 host_to_be_empty = host
254 host_nonha_instances = nonha_instances
256 host_to_be_empty = host
257 host_nonha_instances = nonha_instances
258 self.log.info('host %s selected to be empty' % host_to_be_empty)
259 return host_to_be_empty
261 def make_compute_host_empty(self, host, projects_servers, statebase):
263 state_ack = 'ACK_%s' % statebase
264 state_nack = 'NACK_%s' % statebase
265 for project in projects_servers:
266 # self.projects_servers must have servers under action
267 self.projects_servers[project] = projects_servers[project].copy()
268 self.log.info('%s to project %s' % (state, project))
269 self.project_servers_log_info(project, projects_servers)
270 instance_ids = '%s/%s/maintenance' % (self.url, project)
271 allowed_actions = ['MIGRATE', 'LIVE_MIGRATE', 'OWN_ACTION']
273 actions_at = (datetime.datetime.utcnow() +
274 datetime.timedelta(seconds=wait_seconds)
275 ).strftime('%Y-%m-%d %H:%M:%S')
276 metadata = self.metadata
277 self._project_notify(project, instance_ids,
278 allowed_actions, actions_at, state,
280 allowed_states = [state_ack, state_nack]
281 self.wait_projects_state(allowed_states, wait_seconds)
282 if self.projects_not_in_state(state_ack):
283 raise Exception('Admin tool session %s: all states not %s %s' %
284 (self.session_id, state_ack, self.projects_state))
285 self.actions_to_have_empty_host(host)
287 def notify_action_done(self, project, instance_id):
288 instance_ids = instance_id
291 state = "INSTANCE_ACTION_DONE"
293 self._project_notify(project, instance_ids, allowed_actions,
294 actions_at, state, metadata)
296 def actions_to_have_empty_host(self, host):
298 while len(self.proj_server_actions) == 0:
301 raise Exception('Admin tool session %s: project server actions'
302 ' not set' % self.session_id)
304 for project in self.proj_server_actions:
305 for server, action in self.proj_server_actions[project].items():
306 self.log.info('Action %s server %s: %s' % (action, server,
307 self.projects_servers[project][server]))
308 if action == 'MIGRATE':
309 self.migrate_server(server)
310 self.notify_action_done(project, server)
311 elif action == 'OWN_ACTION':
314 raise Exception('Admin tool session %s: server %s action '
316 (self.session_id, server, action))
317 self.proj_server_actions = dict()
318 self._wait_host_empty(host)
320 def migrate_server(self, server_id):
321 server = self.nova.servers.get(server_id)
322 vm_state = server.__dict__.get('OS-EXT-STS:vm_state')
323 self.log.info('server %s state %s' % (server_id, vm_state))
324 last_vm_state = vm_state
331 while vm_state != 'resized' and retries > 0:
332 # try to confirm within 3min
333 server = self.nova.servers.get(server_id)
334 vm_state = server.__dict__.get('OS-EXT-STS:vm_state')
335 if vm_state == 'resized':
336 server.confirm_resize()
337 self.log.info('server %s migration confirmed' %
340 if last_vm_state != vm_state:
341 self.log.info('server %s state: %s' % (server_id,
343 if vm_state == 'error':
344 raise Exception('server %s migration failed, state: %s'
345 % (server_id, vm_state))
347 retries = retries - 1
348 last_vm_state = vm_state
349 # Timout waiting state to change
353 if retry_migrate == 0:
354 raise Exception('server %s migrate failed' % server_id)
355 # Might take time for scheduler to sync inconsistent instance
357 retry_time = 180 - (retry_migrate * 30)
358 self.log.info('server %s migrate failed, retry in %s sec'
359 % (server_id, retry_time))
360 time.sleep(retry_time)
361 except Exception as e:
362 self.log.error('server %s migration failed, Exception=%s' %
364 self.log.error(format_exc())
365 raise Exception('server %s migration failed, state: %s' %
366 (server_id, vm_state))
368 retry_migrate = retry_migrate - 1
369 raise Exception('server %s migration timeout, state: %s' %
370 (server_id, vm_state))
372 def _wait_host_empty(self, host):
373 hid = self.nova.hypervisors.search(host)[0].id
375 # wait 4min to get host empty
377 hvisor = self.nova.hypervisors.get(hid)
378 vcpus_used = hvisor.__getattr__('vcpus_used')
380 if vcpus_used_last == 0:
381 self.log.info('%s still has %d vcpus reserved. wait...'
382 % (host, vcpus_used))
383 elif vcpus_used != vcpus_used_last:
384 self.log.info('%s still has %d vcpus reserved. wait...'
385 % (host, vcpus_used))
386 vcpus_used_last = vcpus_used
389 self.log.info('%s empty' % host)
391 raise Exception('%s host not empty' % host)
393 def projects_listen_alarm(self, match_event):
394 match_projects = ([str(alarm['project_id']) for alarm in
395 self.aodh.alarm.list() if
396 str(alarm['event_rule']['event_type']) ==
398 all_projects_match = True
399 for project in list(self.projects_state):
400 if project not in match_projects:
401 self.log.error('Admin tool session %s: project %s not '
403 (self.session_id, project, match_event))
404 all_projects_match = False
405 return all_projects_match
407 def project_servers_log_info(self, project, host_servers):
408 info = 'Project servers:\n'
409 for server in host_servers[project]:
410 info += (' %s: %s\n' %
411 (server, host_servers[project][server]))
412 self.log.info('%s' % info)
414 def servers_log_info(self, host_servers):
416 for host in self.hosts:
417 info += '%s:\n' % host
418 if host in host_servers:
419 for project in host_servers[host]:
420 info += ' %s:\n' % project
421 for server in host_servers[host][project]:
422 info += (' %s: %s\n' %
423 (server, host_servers[host][project][server]))
424 self.log.info('%s' % info)
426 def update_server_info(self):
427 opts = {'all_tenants': True}
428 servers = self.nova.servers.list(search_opts=opts)
429 self.projects_servers = dict()
430 host_servers = dict()
431 for server in servers:
433 host = str(server.__dict__.get('OS-EXT-SRV-ATTR:host'))
434 project = str(server.tenant_id)
435 server_name = str(server.name)
436 server_id = str(server.id)
438 raise Exception('can not get params from server=%s' %
440 if host not in self.hosts:
442 if host not in host_servers:
443 host_servers[host] = dict()
444 if project not in host_servers[host]:
445 host_servers[host][project] = dict()
446 if project not in self.projects_servers:
447 self.projects_servers[project] = dict()
448 if project not in self.projects_state:
449 self.projects_state[project] = None
450 host_servers[host][project][server_id] = server_name
451 self.projects_servers[project][server_id] = server_name
454 def str_to_datetime(self, dt_str):
455 mdate, mtime = dt_str.split()
456 year, month, day = map(int, mdate.split('-'))
457 hours, minutes, seconds = map(int, mtime.split(':'))
458 return datetime.datetime(year, month, day, hours, minutes, seconds)
460 def host_maintenance(self, host):
461 self.log.info('maintaining host %s' % host)
462 # no implementation to make real maintenance
466 while self.state != 'MAINTENANCE_COMPLETE':
467 self.log.info('--==session %s: processing state %s==--' %
468 (self.session_id, self.state))
469 if self.state == 'MAINTENANCE':
470 host_servers = self.update_server_info()
471 self.servers_log_info(host_servers)
473 if not self.projects_listen_alarm('maintenance.scheduled'):
474 raise Exception('all projects do not listen maintenance '
478 maint_at = self.str_to_datetime(self.maintenance_at)
479 if maint_at > datetime.datetime.utcnow():
480 time_now = (datetime.datetime.utcnow().strftime(
481 '%Y-%m-%d %H:%M:%S'))
482 self.log.info('Time now: %s maintenance starts: %s....' %
483 (time_now, self.maintenance_at))
484 td = maint_at - datetime.datetime.utcnow()
485 time.sleep(td.total_seconds())
486 time_now = (datetime.datetime.utcnow().strftime(
487 '%Y-%m-%d %H:%M:%S'))
488 self.log.info('Time to start maintenance starts: %s' %
491 # check if we have empty compute host
492 # True -> PLANNED_MAINTENANCE
493 # False -> check if we can migrate VMs to get empty host
494 # True -> PREPARE_MAINTENANCE
495 # False -> DOWN_SCALE
496 maintenance_empty_hosts = ([h for h in self.hosts if h not in
499 if len(maintenance_empty_hosts) == 0:
500 if self.need_down_scale(host_servers):
501 self.log.info('Need to down scale')
502 self.state = 'DOWN_SCALE'
504 self.log.info('Free capacity, but need empty host')
505 self.state = 'PREPARE_MAINTENANCE'
507 self.log.info('Free capacity, but need empty host')
508 self.state = 'PLANNED_MAINTENANCE'
509 self.log.info('--==State change from MAINTENANCE to %s==--'
511 elif self.state == 'DOWN_SCALE':
512 # Test case is hard coded to have all compute capacity used
513 # We need to down scale to have one empty compute host
515 self.state = 'PREPARE_MAINTENANCE'
516 host_servers = self.update_server_info()
517 self.servers_log_info(host_servers)
518 self.log.info('--==State change from DOWN_SCALE to'
519 ' %s==--' % self.state)
521 elif self.state == 'PREPARE_MAINTENANCE':
522 # It might be down scale did not free capacity on a single
523 # compute host, so we need to arrange free capacity to a single
525 self.maint_proj_servers = self.projects_servers.copy()
526 maintenance_empty_hosts = ([h for h in self.hosts if h not in
528 if len(maintenance_empty_hosts) == 0:
529 self.log.info('no empty hosts for maintenance')
530 if self.need_down_scale(host_servers):
531 raise Exception('Admin tool session %s: Not enough '
532 'free capacity for maintenance' %
534 host = self.find_host_to_be_empty(host_servers)
536 self.make_compute_host_empty(host, host_servers[host],
537 'PREPARE_MAINTENANCE')
539 # We do not currently support another down scale if
540 # first was not enough
541 raise Exception('Admin tool session %s: No host '
542 'candidate to be emptied' %
545 for host in maintenance_empty_hosts:
546 self.log.info('%s already empty '
547 'for maintenance' % host)
548 self.state = 'PLANNED_MAINTENANCE'
549 host_servers = self.update_server_info()
550 self.servers_log_info(host_servers)
551 self.log.info('--==State change from PREPARE_MAINTENANCE to %s'
553 elif self.state == 'PLANNED_MAINTENANCE':
554 maintenance_hosts = list()
555 maintenance_empty_hosts = list()
556 # TODO This should be admin. hack for now to have it work
557 admin_project = list(self.projects_state)[0]
558 for host in self.hosts:
559 self.log.info('disable nova-compute on host %s' % host)
560 self.nova.services.disable_log_reason(host, 'nova-compute',
562 self.computes_disabled.append(host)
563 if host in host_servers and len(host_servers[host]):
564 maintenance_hosts.append(host)
566 maintenance_empty_hosts.append(host)
567 self.log.info('--==Start to maintain empty hosts==--\n%s' %
568 maintenance_empty_hosts)
569 for host in maintenance_empty_hosts:
570 # scheduler has problems, let's see if just down scaled
571 # host is really empty
572 self._wait_host_empty(host)
573 self.log.info('IN_MAINTENANCE host %s' % host)
574 self._admin_notify(admin_project, host, 'IN_MAINTENANCE',
576 self.host_maintenance(host)
577 self._admin_notify(admin_project, host,
578 'MAINTENANCE_COMPLETE',
580 self.nova.services.enable(host, 'nova-compute')
581 self.computes_disabled.remove(host)
582 self.log.info('MAINTENANCE_COMPLETE host %s' % host)
583 self.log.info('--==Start to maintain occupied hosts==--\n%s' %
585 for host in maintenance_hosts:
586 self.log.info('PLANNED_MAINTENANCE host %s' % host)
587 self.make_compute_host_empty(host, host_servers[host],
588 'PLANNED_MAINTENANCE')
589 self.log.info('IN_MAINTENANCE host %s' % host)
590 self._admin_notify(admin_project, host, 'IN_MAINTENANCE',
592 self.host_maintenance(host)
593 self._admin_notify(admin_project, host,
594 'MAINTENANCE_COMPLETE',
596 self.nova.services.enable(host, 'nova-compute')
597 self.computes_disabled.remove(host)
598 self.log.info('MAINTENANCE_COMPLETE host %s' % host)
599 self.state = 'PLANNED_MAINTENANCE_COMPLETE'
600 host_servers = self.update_server_info()
601 self.servers_log_info(host_servers)
602 elif self.state == 'PLANNED_MAINTENANCE_COMPLETE':
603 self.log.info('Projects still need to up scale back to full '
605 self.maintenance_complete()
606 host_servers = self.update_server_info()
607 self.servers_log_info(host_servers)
608 self.state = 'MAINTENANCE_COMPLETE'
610 raise Exception('Admin tool session %s: session in invalid '
611 'state %s' % (self.session_id, self.state))
612 self.log.info('--==Maintenance session %s: '
613 'MAINTENANCE SESSION COMPLETE==--' % self.session_id)
615 def project_input(self, project_id, data):
616 self.log.debug('Admin tool session %s: project %s input' %
617 (self.session_id, project_id))
618 if 'instance_actions' in data:
619 self.proj_server_actions[project_id] = (
620 data['instance_actions'].copy())
621 self.projects_state[project_id] = data['state']
623 def project_get_instances(self, project_id):
624 ret = list(self.projects_servers[project_id])
625 self.log.debug('Admin tool session %s: project %s GET return: %s' %
626 (self.session_id, project_id, ret))
633 class AdminTool(Thread):
635 def __init__(self, trasport_url, conf, admin_tool, log):
636 Thread.__init__(self)
637 self.admin_tool = admin_tool
640 self.port = self.conf.admin_tool.port
641 self.maint_sessions = {}
643 self.maintenance_hosts = []
644 self.trasport_url = trasport_url
647 app = Flask('admin_tool')
649 @app.route('/maintenance', methods=['POST'])
650 def admin_maintenance_api_post():
651 data = json.loads(request.data.decode('utf8'))
652 self.log.info('maintenance message: %s' % data)
653 if 'session_id' in data:
654 if data['state'] == 'REMOVE_MAINTENANCE_SESSION':
655 session_id = data['session_id']
656 self.log.info('remove session %s'
658 self.maint_sessions[session_id].cleanup()
659 self.maint_sessions[session_id].stop()
660 del self.maint_sessions[session_id]
662 session_id = str(generate_uuid())
663 self.log.info('creating session: %s' % session_id)
664 self.maint_sessions[session_id] = (
665 AdminMain(self.trasport_url,
671 self.maint_sessions[session_id].start()
672 reply = json.dumps({'session_id': session_id,
673 'state': 'ACK_%s' % data['state']})
674 self.log.debug('reply: %s' % reply)
675 return reply, 200, None
677 @app.route('/maintenance', methods=['GET'])
678 def admin_maintenance_api_get():
679 data = json.loads(request.data.decode('utf8'))
680 self.log.debug('Admin get maintenance: %s' % data)
681 session_id = data['session_id']
682 reply = json.dumps({'state':
683 self.maint_sessions[session_id].state})
684 self.log.debug('reply: %s' % reply)
685 return reply, 200, None
687 @app.route('/<projet_id>/maintenance', methods=['PUT'])
688 def project_maintenance_api_put(projet_id=None):
689 data = json.loads(request.data.decode('utf8'))
690 self.log.debug('%s project put: %s' % (projet_id, data))
691 self.project_input(projet_id, data)
694 @app.route('/<projet_id>/maintenance', methods=['GET'])
695 def project_maintenance_api_get(projet_id=None):
696 data = json.loads(request.data.decode('utf8'))
697 self.log.debug('%s project get %s' % (projet_id, data))
698 instances = self.project_get_instances(projet_id, data)
699 reply = json.dumps({'instance_ids': instances})
700 self.log.debug('%s reply: %s' % (projet_id, reply))
701 return reply, 200, None
703 @app.route('/shutdown', methods=['POST'])
705 for session in self.maint_sessions:
706 self.log.info('shutdown admin tool session %s thread' %
708 self.maint_sessions[session].cleanup()
709 self.maint_sessions[session].stop()
710 self.log.info('shutdown admin_tool server at %s' % time.time())
711 func = request.environ.get('werkzeug.server.shutdown')
713 raise RuntimeError('Not running with the Werkzeug Server')
715 return 'admin_tool app shutting down...'
717 app.run(host='0.0.0.0', port=self.port)
719 def project_input(self, project_id, data):
720 session_id = data['session_id']
721 self.maint_sessions[session_id].project_input(project_id, data)
723 def project_get_instances(self, project_id, data):
724 session_id = data['session_id']
725 return self.maint_sessions[session_id].project_get_instances(