src/ceph/qa/tasks/restart.py

   1 """
   2 Daemon restart
   3 """
   4 import logging
   5 import pipes
   6
   7 from teuthology import misc as teuthology
   8 from teuthology.orchestra import run as tor
   9
  10 from teuthology.orchestra import run
  11 log = logging.getLogger(__name__)
  12
  13 def restart_daemon(ctx, config, role, id_, *args):
  14     """
  15     Handle restart (including the execution of the command parameters passed)
  16     """
  17     log.info('Restarting {r}.{i} daemon...'.format(r=role, i=id_))
  18     daemon = ctx.daemons.get_daemon(role, id_)
  19     log.debug('Waiting for exit of {r}.{i} daemon...'.format(r=role, i=id_))
  20     try:
  21         daemon.wait_for_exit()
  22     except tor.CommandFailedError as e:
  23         log.debug('Command Failed: {e}'.format(e=e))
  24     if len(args) > 0:
  25         confargs = ['--{k}={v}'.format(k=k, v=v) for k,v in zip(args[0::2], args[1::2])]
  26         log.debug('Doing restart of {r}.{i} daemon with args: {a}...'.format(r=role, i=id_, a=confargs))
  27         daemon.restart_with_args(confargs)
  28     else:
  29         log.debug('Doing restart of {r}.{i} daemon...'.format(r=role, i=id_))
  30         daemon.restart()
  31
  32 def get_tests(ctx, config, role, remote, testdir):
  33     """Download restart tests"""
  34     srcdir = '{tdir}/restart.{role}'.format(tdir=testdir, role=role)
  35
  36     refspec = config.get('branch')
  37     if refspec is None:
  38         refspec = config.get('sha1')
  39     if refspec is None:
  40         refspec = config.get('tag')
  41     if refspec is None:
  42         refspec = 'HEAD'
  43     log.info('Pulling restart qa/workunits from ref %s', refspec)
  44
  45     remote.run(
  46         logger=log.getChild(role),
  47         args=[
  48             'mkdir', '--', srcdir,
  49             run.Raw('&&'),
  50             'git',
  51             'archive',
  52             '--remote=git://git.ceph.com/ceph.git',
  53             '%s:qa/workunits' % refspec,
  54             run.Raw('|'),
  55             'tar',
  56             '-C', srcdir,
  57             '-x',
  58             '-f-',
  59             run.Raw('&&'),
  60             'cd', '--', srcdir,
  61             run.Raw('&&'),
  62             'if', 'test', '-e', 'Makefile', run.Raw(';'), 'then', 'make', run.Raw(';'), 'fi',
  63             run.Raw('&&'),
  64             'find', '-executable', '-type', 'f', '-printf', r'%P\0'.format(srcdir=srcdir),
  65             run.Raw('>{tdir}/restarts.list'.format(tdir=testdir)),
  66             ],
  67         )
  68     restarts = sorted(teuthology.get_file(
  69                         remote,
  70                         '{tdir}/restarts.list'.format(tdir=testdir)).split('\0'))
  71     return (srcdir, restarts)
  72
  73 def task(ctx, config):
  74     """
  75     Execute commands and allow daemon restart with config options.
  76     Each process executed can output to stdout restart commands of the form:
  77         restart <role> <id> <conf_key1> <conf_value1> <conf_key2> <conf_value2>
  78     This will restart the daemon <role>.<id> with the specified config values once
  79     by modifying the conf file with those values, and then replacing the old conf file
  80     once the daemon is restarted.
  81     This task does not kill a running daemon, it assumes the daemon will abort on an
  82     assert specified in the config.
  83
  84         tasks:
  85         - install:
  86         - ceph:
  87         - restart:
  88             exec:
  89               client.0:
  90                 - test_backtraces.py
  91
  92     """
  93     assert isinstance(config, dict), "task kill got invalid config"
  94
  95     testdir = teuthology.get_testdir(ctx)
  96
  97     try:
  98         assert 'exec' in config, "config requires exec key with <role>: <command> entries"
  99         for role, task in config['exec'].iteritems():
 100             log.info('restart for role {r}'.format(r=role))
 101             (remote,) = ctx.cluster.only(role).remotes.iterkeys()
 102             srcdir, restarts = get_tests(ctx, config, role, remote, testdir)
 103             log.info('Running command on role %s host %s', role, remote.name)
 104             spec = '{spec}'.format(spec=task[0])
 105             log.info('Restarts list: %s', restarts)
 106             log.info('Spec is %s', spec)
 107             to_run = [w for w in restarts if w == task or w.find(spec) != -1]
 108             log.info('To run: %s', to_run)
 109             for c in to_run:
 110                 log.info('Running restart script %s...', c)
 111                 args = [
 112                     run.Raw('TESTDIR="{tdir}"'.format(tdir=testdir)),
 113                     ]
 114                 env = config.get('env')
 115                 if env is not None:
 116                     for var, val in env.iteritems():
 117                         quoted_val = pipes.quote(val)
 118                         env_arg = '{var}={val}'.format(var=var, val=quoted_val)
 119                         args.append(run.Raw(env_arg))
 120                 args.extend([
 121                             'adjust-ulimits',
 122                             'ceph-coverage',
 123                             '{tdir}/archive/coverage'.format(tdir=testdir),
 124                             '{srcdir}/{c}'.format(
 125                                 srcdir=srcdir,
 126                                 c=c,
 127                                 ),
 128                             ])
 129                 proc = remote.run(
 130                     args=args,
 131                     stdout=tor.PIPE,
 132                     stdin=tor.PIPE,
 133                     stderr=log,
 134                     wait=False,
 135                     )
 136                 log.info('waiting for a command from script')
 137                 while True:
 138                     l = proc.stdout.readline()
 139                     if not l or l == '':
 140                         break
 141                     log.debug('script command: {c}'.format(c=l))
 142                     ll = l.strip()
 143                     cmd = ll.split(' ')
 144                     if cmd[0] == "done":
 145                         break
 146                     assert cmd[0] == 'restart', "script sent invalid command request to kill task"
 147                     # cmd should be: restart <role> <id> <conf_key1> <conf_value1> <conf_key2> <conf_value2>
 148                     # or to clear, just: restart <role> <id>
 149                     restart_daemon(ctx, config, cmd[1], cmd[2], *cmd[3:])
 150                     proc.stdin.writelines(['restarted\n'])
 151                     proc.stdin.flush()
 152                 try:
 153                     proc.wait()
 154                 except tor.CommandFailedError:
 155                     raise Exception('restart task got non-zero exit status from script: {s}'.format(s=c))
 156     finally:
 157         log.info('Finishing %s on %s...', task, role)
 158         remote.run(
 159             logger=log.getChild(role),
 160             args=[
 161                 'rm', '-rf', '--', '{tdir}/restarts.list'.format(tdir=testdir), srcdir,
 162                 ],
 163             )