src/ceph/qa/tasks/cephfs/kernel_mount.py

   1 from StringIO import StringIO
   2 import json
   3 import logging
   4 from textwrap import dedent
   5 from teuthology.orchestra.run import CommandFailedError
   6 from teuthology import misc
   7
   8 from teuthology.orchestra import remote as orchestra_remote
   9 from teuthology.orchestra import run
  10 from teuthology.contextutil import MaxWhileTries
  11 from .mount import CephFSMount
  12
  13 log = logging.getLogger(__name__)
  14
  15
  16 UMOUNT_TIMEOUT = 300
  17
  18
  19 class KernelMount(CephFSMount):
  20     def __init__(self, mons, test_dir, client_id, client_remote,
  21                  ipmi_user, ipmi_password, ipmi_domain):
  22         super(KernelMount, self).__init__(test_dir, client_id, client_remote)
  23         self.mons = mons
  24
  25         self.mounted = False
  26         self.ipmi_user = ipmi_user
  27         self.ipmi_password = ipmi_password
  28         self.ipmi_domain = ipmi_domain
  29
  30     def write_secret_file(self, remote, role, keyring, filename):
  31         """
  32         Stash the keyring in the filename specified.
  33         """
  34         remote.run(
  35             args=[
  36                 'adjust-ulimits',
  37                 'ceph-coverage',
  38                 '{tdir}/archive/coverage'.format(tdir=self.test_dir),
  39                 'ceph-authtool',
  40                 '--name={role}'.format(role=role),
  41                 '--print-key',
  42                 keyring,
  43                 run.Raw('>'),
  44                 filename,
  45             ],
  46         )
  47
  48     def mount(self, mount_path=None, mount_fs_name=None):
  49         log.info('Mounting kclient client.{id} at {remote} {mnt}...'.format(
  50             id=self.client_id, remote=self.client_remote, mnt=self.mountpoint))
  51
  52         keyring = self.get_keyring_path()
  53         secret = '{tdir}/ceph.data/client.{id}.secret'.format(tdir=self.test_dir, id=self.client_id)
  54         self.write_secret_file(self.client_remote, 'client.{id}'.format(id=self.client_id),
  55                                keyring, secret)
  56
  57         self.client_remote.run(
  58             args=[
  59                 'mkdir',
  60                 '--',
  61                 self.mountpoint,
  62             ],
  63         )
  64
  65         if mount_path is None:
  66             mount_path = "/"
  67
  68         opts = 'name={id},secretfile={secret},norequire_active_mds'.format(id=self.client_id,
  69                                                       secret=secret)
  70
  71         if mount_fs_name is not None:
  72             opts += ",mds_namespace={0}".format(mount_fs_name)
  73
  74         self.client_remote.run(
  75             args=[
  76                 'sudo',
  77                 'adjust-ulimits',
  78                 'ceph-coverage',
  79                 '{tdir}/archive/coverage'.format(tdir=self.test_dir),
  80                 '/sbin/mount.ceph',
  81                 '{mons}:{mount_path}'.format(mons=','.join(self.mons), mount_path=mount_path),
  82                 self.mountpoint,
  83                 '-v',
  84                 '-o',
  85                 opts
  86             ],
  87         )
  88
  89         self.client_remote.run(
  90             args=['sudo', 'chmod', '1777', self.mountpoint])
  91
  92         self.mounted = True
  93
  94     def umount(self, force=False):
  95         log.debug('Unmounting client client.{id}...'.format(id=self.client_id))
  96
  97         cmd=['sudo', 'umount', self.mountpoint]
  98         if force:
  99             cmd.append('-f')
 100
 101         try:
 102             self.client_remote.run(args=cmd)
 103         except Exception as e:
 104             self.client_remote.run(args=[
 105                 'sudo',
 106                 run.Raw('PATH=/usr/sbin:$PATH'),
 107                 'lsof',
 108                 run.Raw(';'),
 109                 'ps', 'auxf',
 110             ])
 111             raise e
 112
 113         rproc = self.client_remote.run(
 114             args=[
 115                 'rmdir',
 116                 '--',
 117                 self.mountpoint,
 118             ],
 119             wait=False
 120         )
 121         run.wait([rproc], UMOUNT_TIMEOUT)
 122         self.mounted = False
 123
 124     def cleanup(self):
 125         pass
 126
 127     def umount_wait(self, force=False, require_clean=False):
 128         """
 129         Unlike the fuse client, the kernel client's umount is immediate
 130         """
 131         if not self.is_mounted():
 132             return
 133
 134         try:
 135             self.umount(force)
 136         except (CommandFailedError, MaxWhileTries):
 137             if not force:
 138                 raise
 139
 140             self.kill()
 141             self.kill_cleanup()
 142
 143         self.mounted = False
 144
 145     def is_mounted(self):
 146         return self.mounted
 147
 148     def wait_until_mounted(self):
 149         """
 150         Unlike the fuse client, the kernel client is up and running as soon
 151         as the initial mount() function returns.
 152         """
 153         assert self.mounted
 154
 155     def teardown(self):
 156         super(KernelMount, self).teardown()
 157         if self.mounted:
 158             self.umount()
 159
 160     def kill(self):
 161         """
 162         The Ceph kernel client doesn't have a mechanism to kill itself (doing
 163         that in side the kernel would be weird anyway), so we reboot the whole node
 164         to get the same effect.
 165
 166         We use IPMI to reboot, because we don't want the client to send any
 167         releases of capabilities.
 168         """
 169
 170         con = orchestra_remote.getRemoteConsole(self.client_remote.hostname,
 171                                                 self.ipmi_user,
 172                                                 self.ipmi_password,
 173                                                 self.ipmi_domain)
 174         con.power_off()
 175
 176         self.mounted = False
 177
 178     def kill_cleanup(self):
 179         assert not self.mounted
 180
 181         con = orchestra_remote.getRemoteConsole(self.client_remote.hostname,
 182                                                 self.ipmi_user,
 183                                                 self.ipmi_password,
 184                                                 self.ipmi_domain)
 185         con.power_on()
 186
 187         # Wait for node to come back up after reboot
 188         misc.reconnect(None, 300, [self.client_remote])
 189
 190         # Remove mount directory
 191         self.client_remote.run(
 192             args=[
 193                 'rmdir',
 194                 '--',
 195                 self.mountpoint,
 196             ],
 197         )
 198
 199     def _find_debug_dir(self):
 200         """
 201         Find the debugfs folder for this mount
 202         """
 203         pyscript = dedent("""
 204             import glob
 205             import os
 206             import json
 207
 208             def get_id_to_dir():
 209                 result = {}
 210                 for dir in glob.glob("/sys/kernel/debug/ceph/*"):
 211                     mds_sessions_lines = open(os.path.join(dir, "mds_sessions")).readlines()
 212                     client_id = mds_sessions_lines[1].split()[1].strip('"')
 213
 214                     result[client_id] = dir
 215                 return result
 216
 217             print json.dumps(get_id_to_dir())
 218             """)
 219
 220         p = self.client_remote.run(args=[
 221             'sudo', 'python', '-c', pyscript
 222         ], stdout=StringIO())
 223         client_id_to_dir = json.loads(p.stdout.getvalue())
 224
 225         try:
 226             return client_id_to_dir[self.client_id]
 227         except KeyError:
 228             log.error("Client id '{0}' debug dir not found (clients seen were: {1})".format(
 229                 self.client_id, ",".join(client_id_to_dir.keys())
 230             ))
 231             raise
 232
 233     def _read_debug_file(self, filename):
 234         debug_dir = self._find_debug_dir()
 235
 236         pyscript = dedent("""
 237             import os
 238
 239             print open(os.path.join("{debug_dir}", "{filename}")).read()
 240             """).format(debug_dir=debug_dir, filename=filename)
 241
 242         p = self.client_remote.run(args=[
 243             'sudo', 'python', '-c', pyscript
 244         ], stdout=StringIO())
 245         return p.stdout.getvalue()
 246
 247     def get_global_id(self):
 248         """
 249         Look up the CephFS client ID for this mount, using debugfs.
 250         """
 251
 252         assert self.mounted
 253
 254         mds_sessions = self._read_debug_file("mds_sessions")
 255         lines = mds_sessions.split("\n")
 256         return int(lines[0].split()[1])
 257
 258     def get_osd_epoch(self):
 259         """
 260         Return 2-tuple of osd_epoch, osd_epoch_barrier
 261         """
 262         osd_map = self._read_debug_file("osdmap")
 263         lines = osd_map.split("\n")
 264         first_line_tokens = lines[0].split()
 265         epoch, barrier = int(first_line_tokens[1]), int(first_line_tokens[3])
 266
 267         return epoch, barrier