--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright (C) 2015, 2016, 2017 Red Hat <contact@redhat.com>
+# Copyright (C) 2014 Inktank <info@inktank.com>
+# Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
+# Copyright (C) 2014 Catalyst.net Ltd
+#
+# Author: Loic Dachary <loic@dachary.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library Public License for more details.
+#
+
+from __future__ import print_function
+
+import argparse
+import base64
+import errno
+import fcntl
+import functools
+import json
+import logging
+import os
+import platform
+import re
+import subprocess
+import stat
+import sys
+import tempfile
+import uuid
+import time
+import shlex
+import shutil
+import pwd
+import grp
+import textwrap
+import glob
+
+CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026'
+CEPH_LOCKBOX_ONDISK_MAGIC = 'ceph lockbox volume v001'
+
+KEY_MANAGEMENT_MODE_V1 = 'ceph-mon v1'
+
+PTYPE = {
+ 'regular': {
+ 'journal': {
+ # identical because creating a journal is atomic
+ 'ready': '45b0969e-9b03-4f30-b4c6-b4b80ceff106',
+ 'tobe': '45b0969e-9b03-4f30-b4c6-b4b80ceff106',
+ },
+ 'block': {
+ # identical because creating a block is atomic
+ 'ready': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
+ 'tobe': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
+ },
+ 'block.db': {
+ # identical because creating a block is atomic
+ 'ready': '30cd0809-c2b2-499c-8879-2d6b78529876',
+ 'tobe': '30cd0809-c2b2-499c-8879-2d6b785292be',
+ },
+ 'block.wal': {
+ # identical because creating a block is atomic
+ 'ready': '5ce17fce-4087-4169-b7ff-056cc58473f9',
+ 'tobe': '5ce17fce-4087-4169-b7ff-056cc58472be',
+ },
+ 'osd': {
+ 'ready': '4fbd7e29-9d25-41b8-afd0-062c0ceff05d',
+ 'tobe': '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be',
+ },
+ 'lockbox': {
+ 'ready': 'fb3aabf9-d25f-47cc-bf5e-721d1816496b',
+ 'tobe': 'fb3aabf9-d25f-47cc-bf5e-721d181642be',
+ },
+ },
+ 'luks': {
+ 'journal': {
+ 'ready': '45b0969e-9b03-4f30-b4c6-35865ceff106',
+ 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
+ },
+ 'block': {
+ 'ready': 'cafecafe-9b03-4f30-b4c6-35865ceff106',
+ 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
+ },
+ 'block.db': {
+ 'ready': '166418da-c469-4022-adf4-b30afd37f176',
+ 'tobe': '7521c784-4626-4260-bc8d-ba77a0f5f2be',
+ },
+ 'block.wal': {
+ 'ready': '86a32090-3647-40b9-bbbd-38d8c573aa86',
+ 'tobe': '92dad30f-175b-4d40-a5b0-5c0a258b42be',
+ },
+ 'osd': {
+ 'ready': '4fbd7e29-9d25-41b8-afd0-35865ceff05d',
+ 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
+ },
+ },
+ 'plain': {
+ 'journal': {
+ 'ready': '45b0969e-9b03-4f30-b4c6-5ec00ceff106',
+ 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
+ },
+ 'block': {
+ 'ready': 'cafecafe-9b03-4f30-b4c6-5ec00ceff106',
+ 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
+ },
+ 'block.db': {
+ 'ready': '93b0052d-02d9-4d8a-a43b-33a3ee4dfbc3',
+ 'tobe': '69d17c68-3e58-4399-aff0-b68265f2e2be',
+ },
+ 'block.wal': {
+ 'ready': '306e8683-4fe2-4330-b7c0-00a917c16966',
+ 'tobe': 'f2d89683-a621-4063-964a-eb1f7863a2be',
+ },
+ 'osd': {
+ 'ready': '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d',
+ 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
+ },
+ },
+ 'mpath': {
+ 'journal': {
+ 'ready': '45b0969e-8ae0-4982-bf9d-5a8d867af560',
+ 'tobe': '45b0969e-8ae0-4982-bf9d-5a8d867af560',
+ },
+ 'block': {
+ 'ready': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
+ 'tobe': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
+ },
+ 'block.db': {
+ 'ready': 'ec6d6385-e346-45dc-be91-da2a7c8b3261',
+ 'tobe': 'ec6d6385-e346-45dc-be91-da2a7c8b32be',
+ },
+ 'block.wal': {
+ 'ready': '01b41e1b-002a-453c-9f17-88793989ff8f',
+ 'tobe': '01b41e1b-002a-453c-9f17-88793989f2be',
+ },
+ 'osd': {
+ 'ready': '4fbd7e29-8ae0-4982-bf9d-5a8d867af560',
+ 'tobe': '89c57f98-8ae0-4982-bf9d-5a8d867af560',
+ },
+ 'lockbox': {
+ 'ready': '7f4a666a-16f3-47a2-8445-152ef4d03f6c',
+ 'tobe': '7f4a666a-16f3-47a2-8445-152ef4d032be',
+ },
+ },
+}
+
+try:
+ # see https://bugs.python.org/issue23098
+ os.major(0x80002b00)
+except OverflowError:
+ os.major = lambda devid: ((devid >> 8) & 0xfff) | ((devid >> 32) & ~0xfff)
+ os.minor = lambda devid: (devid & 0xff) | ((devid >> 12) & ~0xff)
+
+
+class Ptype(object):
+
+ @staticmethod
+ def get_ready_by_type(what):
+ return [x['ready'] for x in PTYPE[what].values()]
+
+ @staticmethod
+ def get_ready_by_name(name):
+ return [x[name]['ready'] for x in PTYPE.values() if name in x]
+
+ @staticmethod
+ def is_regular_space(ptype):
+ return Ptype.is_what_space('regular', ptype)
+
+ @staticmethod
+ def is_mpath_space(ptype):
+ return Ptype.is_what_space('mpath', ptype)
+
+ @staticmethod
+ def is_plain_space(ptype):
+ return Ptype.is_what_space('plain', ptype)
+
+ @staticmethod
+ def is_luks_space(ptype):
+ return Ptype.is_what_space('luks', ptype)
+
+ @staticmethod
+ def is_what_space(what, ptype):
+ for name in Space.NAMES:
+ if ptype == PTYPE[what][name]['ready']:
+ return True
+ return False
+
+ @staticmethod
+ def space_ptype_to_name(ptype):
+ for what in PTYPE.values():
+ for name in Space.NAMES:
+ if ptype == what[name]['ready']:
+ return name
+ raise ValueError('ptype ' + ptype + ' not found')
+
+ @staticmethod
+ def is_dmcrypt_space(ptype):
+ for name in Space.NAMES:
+ if Ptype.is_dmcrypt(ptype, name):
+ return True
+ return False
+
+ @staticmethod
+ def is_dmcrypt(ptype, name):
+ for what in ('plain', 'luks'):
+ if ptype == PTYPE[what][name]['ready']:
+ return True
+ return False
+
+
+SYSFS = '/sys'
+
+if platform.system() == 'FreeBSD':
+ FREEBSD = True
+ DEFAULT_FS_TYPE = 'zfs'
+ PROCDIR = '/compat/linux/proc'
+ # FreeBSD does not have blockdevices any more
+ BLOCKDIR = '/dev'
+ ROOTGROUP = 'wheel'
+else:
+ FREEBSD = False
+ DEFAULT_FS_TYPE = 'xfs'
+ PROCDIR = '/proc'
+ BLOCKDIR = '/sys/block'
+ ROOTGROUP = 'root'
+
+"""
+OSD STATUS Definition
+"""
+OSD_STATUS_OUT_DOWN = 0
+OSD_STATUS_OUT_UP = 1
+OSD_STATUS_IN_DOWN = 2
+OSD_STATUS_IN_UP = 3
+
+MOUNT_OPTIONS = dict(
+ btrfs='noatime,user_subvol_rm_allowed',
+ # user_xattr is default ever since linux 2.6.39 / 3.0, but we'll
+ # delay a moment before removing it fully because we did have some
+ # issues with ext4 before the xatts-in-leveldb work, and it seemed
+ # that user_xattr helped
+ ext4='noatime,user_xattr',
+ xfs='noatime,inode64',
+)
+
+MKFS_ARGS = dict(
+ btrfs=[
+ # btrfs requires -f, for the same reason as xfs (see comment below)
+ '-f',
+ '-m', 'single',
+ '-l', '32768',
+ '-n', '32768',
+ ],
+ xfs=[
+ # xfs insists on not overwriting previous fs; even if we wipe
+ # partition table, we often recreate it exactly the same way,
+ # so we'll see ghosts of filesystems past
+ '-f',
+ '-i', 'size=2048',
+ ],
+ zfs=[
+ '-o', 'atime=off'
+ ],
+)
+
+INIT_SYSTEMS = [
+ 'upstart',
+ 'sysvinit',
+ 'systemd',
+ 'openrc',
+ 'bsdrc',
+ 'auto',
+ 'none',
+]
+
+STATEDIR = '/var/lib/ceph'
+
+SYSCONFDIR = '/etc/ceph'
+
+prepare_lock = None
+activate_lock = None
+SUPPRESS_PREFIX = None
+
+# only warn once about some things
+warned_about = {}
+
+# Nuke the TERM variable to avoid confusing any subprocesses we call.
+# For example, libreadline will print weird control sequences for some
+# TERM values.
+if 'TERM' in os.environ:
+ del os.environ['TERM']
+
+LOG_NAME = __name__
+if LOG_NAME == '__main__':
+ LOG_NAME = os.path.basename(sys.argv[0])
+LOG = logging.getLogger(LOG_NAME)
+
+# Allow user-preferred values for subprocess user and group
+CEPH_PREF_USER = None
+CEPH_PREF_GROUP = None
+
+
+class FileLock(object):
+ def __init__(self, fn):
+ self.fn = fn
+ self.fd = None
+
+ def __enter__(self):
+ assert not self.fd
+ self.fd = os.open(self.fn, os.O_WRONLY | os.O_CREAT)
+ fcntl.lockf(self.fd, fcntl.LOCK_EX)
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ assert self.fd
+ fcntl.lockf(self.fd, fcntl.LOCK_UN)
+ os.close(self.fd)
+ self.fd = None
+
+
+class Error(Exception):
+ """
+ Error
+ """
+
+ def __str__(self):
+ doc = _bytes2str(self.__doc__.strip())
+ try:
+ str_type = basestring
+ except NameError:
+ str_type = str
+ args = [a if isinstance(a, str_type) else str(a) for a in self.args]
+ return ': '.join([doc] + [_bytes2str(a) for a in args])
+
+
+class MountError(Error):
+ """
+ Mounting filesystem failed
+ """
+
+
+class UnmountError(Error):
+ """
+ Unmounting filesystem failed
+ """
+
+
+class BadMagicError(Error):
+ """
+ Does not look like a Ceph OSD, or incompatible version
+ """
+
+
+class TruncatedLineError(Error):
+ """
+ Line is truncated
+ """
+
+
+class TooManyLinesError(Error):
+ """
+ Too many lines
+ """
+
+
+class FilesystemTypeError(Error):
+ """
+ Cannot discover filesystem type
+ """
+
+
+class CephDiskException(Exception):
+ """
+ A base exception for ceph-disk to provide custom (ad-hoc) messages that
+ will be caught and dealt with when main() is executed
+ """
+ pass
+
+
+class ExecutableNotFound(CephDiskException):
+ """
+ Exception to report on executables not available in PATH
+ """
+ pass
+
+
+def is_systemd():
+ """
+ Detect whether systemd is running
+ """
+ with open(PROCDIR + '/1/comm', 'r') as f:
+ return 'systemd' in f.read()
+
+
+def is_upstart():
+ """
+ Detect whether upstart is running
+ """
+ (out, err, _) = command(['init', '--version'])
+ return 'upstart' in out
+
+
+def maybe_mkdir(*a, **kw):
+ """
+ Creates a new directory if it doesn't exist, removes
+ existing symlink before creating the directory.
+ """
+ # remove any symlink, if it is there..
+ if os.path.exists(*a) and stat.S_ISLNK(os.lstat(*a).st_mode):
+ LOG.debug('Removing old symlink at %s', *a)
+ os.unlink(*a)
+ try:
+ os.mkdir(*a, **kw)
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ pass
+ else:
+ raise
+
+
+def which(executable):
+ """find the location of an executable"""
+ envpath = os.environ.get('PATH') or os.defpath
+ PATH = envpath.split(os.pathsep)
+
+ locations = PATH + [
+ '/usr/local/bin',
+ '/bin',
+ '/usr/bin',
+ '/usr/local/sbin',
+ '/usr/sbin',
+ '/sbin',
+ ]
+
+ for location in locations:
+ executable_path = os.path.join(location, executable)
+ if (os.path.isfile(executable_path) and
+ os.access(executable_path, os.X_OK)):
+ return executable_path
+
+
+def _get_command_executable(arguments):
+ """
+ Return the full path for an executable, raise if the executable is not
+ found. If the executable has already a full path do not perform any checks.
+ """
+ if os.path.isabs(arguments[0]): # an absolute path
+ return arguments
+ executable = which(arguments[0])
+ if not executable:
+ command_msg = 'Could not run command: %s' % ' '.join(arguments)
+ executable_msg = '%s not in path.' % arguments[0]
+ raise ExecutableNotFound('%s %s' % (executable_msg, command_msg))
+
+ # swap the old executable for the new one
+ arguments[0] = executable
+ return arguments
+
+
+def command(arguments, **kwargs):
+ """
+ Safely execute a ``subprocess.Popen`` call making sure that the
+ executable exists and raising a helpful error message
+ if it does not.
+
+ .. note:: This should be the preferred way of calling ``subprocess.Popen``
+ since it provides the caller with the safety net of making sure that
+ executables *will* be found and will error nicely otherwise.
+
+ This returns the output of the command and the return code of the
+ process in a tuple: (stdout, stderr, returncode).
+ """
+
+ arguments = list(map(_bytes2str, _get_command_executable(arguments)))
+
+ LOG.info('Running command: %s' % ' '.join(arguments))
+ process = subprocess.Popen(
+ arguments,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ **kwargs)
+ out, err = process.communicate()
+
+ return _bytes2str(out), _bytes2str(err), process.returncode
+
+
+def command_with_stdin(arguments, stdin):
+ LOG.info("Running command with stdin: " + " ".join(arguments))
+ process = subprocess.Popen(
+ arguments,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, err = process.communicate(stdin)
+ LOG.debug(out)
+ if process.returncode != 0:
+ LOG.error(err)
+ raise SystemExit(
+ "'{cmd}' failed with status code {returncode}".format(
+ cmd=arguments,
+ returncode=process.returncode,
+ )
+ )
+ return out
+
+
+def _bytes2str(string):
+ return string.decode('utf-8') if isinstance(string, bytes) else string
+
+
+def command_init(arguments, **kwargs):
+ """
+ Safely execute a non-blocking ``subprocess.Popen`` call
+ making sure that the executable exists and raising a helpful
+ error message if it does not.
+
+ .. note:: This should be the preferred way of calling ``subprocess.Popen``
+ since it provides the caller with the safety net of making sure that
+ executables *will* be found and will error nicely otherwise.
+
+ This returns the process.
+ """
+
+ arguments = list(map(_bytes2str, _get_command_executable(arguments)))
+
+ LOG.info('Running command: %s' % ' '.join(arguments))
+ process = subprocess.Popen(
+ arguments,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ **kwargs)
+ return process
+
+
+def command_wait(process):
+ """
+ Wait for the process finish and parse its output.
+ """
+
+ out, err = process.communicate()
+
+ return _bytes2str(out), _bytes2str(err), process.returncode
+
+
+def command_check_call(arguments, exit=False):
+ """
+ Safely execute a ``subprocess.check_call`` call making sure that the
+ executable exists and raising a helpful error message if it does not.
+
+ When ``exit`` is set to ``True`` this helper will do a clean (sans
+ traceback) system exit.
+ .. note:: This should be the preferred way of calling
+ ``subprocess.check_call`` since it provides the caller with the safety net
+ of making sure that executables *will* be found and will error nicely
+ otherwise.
+ """
+ arguments = _get_command_executable(arguments)
+ command = ' '.join(arguments)
+ LOG.info('Running command: %s', command)
+ try:
+ return subprocess.check_call(arguments)
+ except subprocess.CalledProcessError as error:
+ if exit:
+ if error.output:
+ LOG.error(error.output)
+ raise SystemExit(
+ "'{cmd}' failed with status code {returncode}".format(
+ cmd=command,
+ returncode=error.returncode,
+ )
+ )
+ raise
+
+
+#
+# An alternative block_path implementation would be
+#
+# name = basename(dev)
+# return /sys/devices/virtual/block/$name
+#
+# It is however more fragile because it relies on the fact
+# that the basename of the device the user will use always
+# matches the one the driver will use. On Ubuntu 14.04, for
+# instance, when multipath creates a partition table on
+#
+# /dev/mapper/353333330000007d0 -> ../dm-0
+#
+# it will create partition devices named
+#
+# /dev/mapper/353333330000007d0-part1
+#
+# which is the same device as /dev/dm-1 but not a symbolic
+# link to it:
+#
+# ubuntu@other:~$ ls -l /dev/mapper /dev/dm-1
+# brw-rw---- 1 root disk 252, 1 Aug 15 17:52 /dev/dm-1
+# lrwxrwxrwx 1 root root 7 Aug 15 17:52 353333330000007d0 -> ../dm-0
+# brw-rw---- 1 root disk 252, 1 Aug 15 17:52 353333330000007d0-part1
+#
+# Using the basename in this case fails.
+#
+
+
+def block_path(dev):
+ if FREEBSD:
+ return dev
+ path = os.path.realpath(dev)
+ rdev = os.stat(path).st_rdev
+ (M, m) = (os.major(rdev), os.minor(rdev))
+ return "{sysfs}/dev/block/{M}:{m}".format(sysfs=SYSFS, M=M, m=m)
+
+
+def get_dm_uuid(dev):
+ uuid_path = os.path.join(block_path(dev), 'dm', 'uuid')
+ LOG.debug("get_dm_uuid " + dev + " uuid path is " + uuid_path)
+ if not os.path.exists(uuid_path):
+ return False
+ uuid = open(uuid_path, 'r').read()
+ LOG.debug("get_dm_uuid " + dev + " uuid is " + uuid)
+ return uuid
+
+
+def is_mpath(dev):
+ """
+ True if the path is managed by multipath
+ """
+ if FREEBSD:
+ return False
+ uuid = get_dm_uuid(dev)
+ return (uuid and
+ (re.match('part\d+-mpath-', uuid) or
+ re.match('mpath-', uuid)))
+
+
+def get_dev_name(path):
+ """
+ get device name from path. e.g.::
+
+ /dev/sda -> sda, /dev/cciss/c0d1 -> cciss!c0d1
+
+ a device "name" is something like::
+
+ sdb
+ cciss!c0d1
+
+ """
+ assert path.startswith('/dev/')
+ base = path[5:]
+ return base.replace('/', '!')
+
+
+def get_dev_path(name):
+ """
+ get a path (/dev/...) from a name (cciss!c0d1)
+ a device "path" is something like::
+
+ /dev/sdb
+ /dev/cciss/c0d1
+
+ """
+ return '/dev/' + name.replace('!', '/')
+
+
+def get_dev_relpath(name):
+ """
+ get a relative path to /dev from a name (cciss!c0d1)
+ """
+ return name.replace('!', '/')
+
+
+def get_dev_size(dev, size='megabytes'):
+ """
+ Attempt to get the size of a device so that we can prevent errors
+ from actions to devices that are smaller, and improve error reporting.
+
+ Because we want to avoid breakage in case this approach is not robust, we
+ will issue a warning if we failed to get the size.
+
+ :param size: bytes or megabytes
+ :param dev: the device to calculate the size
+ """
+ fd = os.open(dev, os.O_RDONLY)
+ dividers = {'bytes': 1, 'megabytes': 1024 * 1024}
+ try:
+ device_size = os.lseek(fd, 0, os.SEEK_END)
+ divider = dividers.get(size, 1024 * 1024) # default to megabytes
+ return device_size // divider
+ except Exception as error:
+ LOG.warning('failed to get size of %s: %s' % (dev, str(error)))
+ finally:
+ os.close(fd)
+
+
+def stmode_is_diskdevice(dmode):
+ if stat.S_ISBLK(dmode):
+ return True
+ else:
+ # FreeBSD does not have block devices
+ # All disks are character devices
+ return FREEBSD and stat.S_ISCHR(dmode)
+
+
+def dev_is_diskdevice(dev):
+ dmode = os.stat(dev).st_mode
+ return stmode_is_diskdevice(dmode)
+
+
+def ldev_is_diskdevice(dev):
+ dmode = os.lstat(dev).st_mode
+ return stmode_is_diskdevice(dmode)
+
+
+def path_is_diskdevice(path):
+ dev = os.path.realpath(path)
+ return dev_is_diskdevice(dev)
+
+
+def get_partition_mpath(dev, pnum):
+ part_re = "part{pnum}-mpath-".format(pnum=pnum)
+ partitions = list_partitions_mpath(dev, part_re)
+ if partitions:
+ return partitions[0]
+ else:
+ return None
+
+
+def retry(on_error=Exception, max_tries=10, wait=0.2, backoff=0):
+ def wrapper(func):
+ @functools.wraps(func)
+ def repeat(*args, **kwargs):
+ for tries in range(max_tries - 1):
+ try:
+ return func(*args, **kwargs)
+ except on_error:
+ time.sleep(wait + backoff * tries)
+ return func(*args, **kwargs)
+ return repeat
+ return wrapper
+
+
+@retry(Error)
+def get_partition_dev(dev, pnum):
+ """
+ get the device name for a partition
+
+ assume that partitions are named like the base dev,
+ with a number, and optionally
+ some intervening characters (like 'p'). e.g.,
+
+ sda 1 -> sda1
+ cciss/c0d1 1 -> cciss!c0d1p1
+ """
+ partname = None
+ error_msg = ""
+ if is_mpath(dev):
+ partname = get_partition_mpath(dev, pnum)
+ else:
+ name = get_dev_name(os.path.realpath(dev))
+ sys_entry = os.path.join(BLOCKDIR, name)
+ error_msg = " in %s" % sys_entry
+ for f in os.listdir(sys_entry):
+ if f.startswith(name) and f.endswith(str(pnum)):
+ # we want the shortest name that starts with the base name
+ # and ends with the partition number
+ if not partname or len(f) < len(partname):
+ partname = f
+ if partname:
+ return get_dev_path(partname)
+ else:
+ raise Error('partition %d for %s does not appear to exist%s' %
+ (pnum, dev, error_msg))
+
+
+def list_all_partitions():
+ """
+ Return a list of devices and partitions
+ """
+ if not FREEBSD:
+ names = os.listdir(BLOCKDIR)
+ dev_part_list = {}
+ for name in names:
+ # /dev/fd0 may hang http://tracker.ceph.com/issues/6827
+ if re.match(r'^fd\d$', name):
+ continue
+ dev_part_list[name] = list_partitions(get_dev_path(name))
+ else:
+ with open(os.path.join(PROCDIR, "partitions")) as partitions:
+ for line in partitions:
+ columns = line.split()
+ if len(columns) >= 4:
+ name = columns[3]
+ dev_part_list[name] = list_partitions(get_dev_path(name))
+ return dev_part_list
+
+
+def list_partitions(dev):
+ dev = os.path.realpath(dev)
+ if is_mpath(dev):
+ return list_partitions_mpath(dev)
+ else:
+ return list_partitions_device(dev)
+
+
+def list_partitions_mpath(dev, part_re="part\d+-mpath-"):
+ p = block_path(dev)
+ partitions = []
+ holders = os.path.join(p, 'holders')
+ for holder in os.listdir(holders):
+ uuid_path = os.path.join(holders, holder, 'dm', 'uuid')
+ uuid = open(uuid_path, 'r').read()
+ LOG.debug("list_partitions_mpath: " + uuid_path + " uuid = " + uuid)
+ if re.match(part_re, uuid):
+ partitions.append(holder)
+ return partitions
+
+
+def list_partitions_device(dev):
+ """
+ Return a list of partitions on the given device name
+ """
+ partitions = []
+ basename = get_dev_name(dev)
+ for name in os.listdir(block_path(dev)):
+ if name.startswith(basename):
+ partitions.append(name)
+ return partitions
+
+
+def get_partition_base(dev):
+ """
+ Get the base device for a partition
+ """
+ dev = os.path.realpath(dev)
+ if not ldev_is_diskdevice(dev):
+ raise Error('not a block device', dev)
+
+ name = get_dev_name(dev)
+ if os.path.exists(os.path.join('/sys/block', name)):
+ raise Error('not a partition', dev)
+
+ # find the base
+ for basename in os.listdir('/sys/block'):
+ if os.path.exists(os.path.join('/sys/block', basename, name)):
+ return get_dev_path(basename)
+ raise Error('no parent device for partition', dev)
+
+
+def is_partition_mpath(dev):
+ uuid = get_dm_uuid(dev)
+ return bool(re.match('part\d+-mpath-', uuid))
+
+
+def partnum_mpath(dev):
+ uuid = get_dm_uuid(dev)
+ return re.findall('part(\d+)-mpath-', uuid)[0]
+
+
+def get_partition_base_mpath(dev):
+ slave_path = os.path.join(block_path(dev), 'slaves')
+ slaves = os.listdir(slave_path)
+ assert slaves
+ name_path = os.path.join(slave_path, slaves[0], 'dm', 'name')
+ name = open(name_path, 'r').read().strip()
+ return os.path.join('/dev/mapper', name)
+
+
+def is_partition(dev):
+ """
+ Check whether a given device path is a partition or a full disk.
+ """
+ if is_mpath(dev):
+ return is_partition_mpath(dev)
+
+ dev = os.path.realpath(dev)
+ st = os.lstat(dev)
+ if not stmode_is_diskdevice(st.st_mode):
+ raise Error('not a block device', dev)
+
+ name = get_dev_name(dev)
+ if os.path.exists(os.path.join(BLOCKDIR, name)):
+ return False
+
+ # make sure it is a partition of something else
+ major = os.major(st.st_rdev)
+ minor = os.minor(st.st_rdev)
+ if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)):
+ return True
+
+ raise Error('not a disk or partition', dev)
+
+
+def is_mounted(dev):
+ """
+ Check if the given device is mounted.
+ """
+ dev = os.path.realpath(dev)
+ with open(PROCDIR + '/mounts', 'rb') as proc_mounts:
+ for line in proc_mounts:
+ fields = line.split()
+ if len(fields) < 3:
+ continue
+ mounts_dev = fields[0]
+ path = fields[1]
+ if os.path.isabs(mounts_dev) and os.path.exists(mounts_dev):
+ mounts_dev = os.path.realpath(mounts_dev)
+ if mounts_dev == dev:
+ return _bytes2str(path)
+ return None
+
+
+def is_held(dev):
+ """
+ Check if a device is held by another device (e.g., a dm-crypt mapping)
+ """
+ assert os.path.exists(dev)
+ if is_mpath(dev):
+ return []
+
+ dev = os.path.realpath(dev)
+ base = get_dev_name(dev)
+
+ # full disk?
+ directory = '/sys/block/{base}/holders'.format(base=base)
+ if os.path.exists(directory):
+ return os.listdir(directory)
+
+ # partition?
+ part = base
+ while len(base):
+ directory = '/sys/block/{base}/{part}/holders'.format(
+ part=part, base=base)
+ if os.path.exists(directory):
+ return os.listdir(directory)
+ base = base[:-1]
+ return []
+
+
+def verify_not_in_use(dev, check_partitions=False):
+ """
+ Verify if a given device (path) is in use (e.g. mounted or
+ in use by device-mapper).
+
+ :raises: Error if device is in use.
+ """
+ assert os.path.exists(dev)
+ if is_mounted(dev):
+ raise Error('Device is mounted', dev)
+ holders = is_held(dev)
+ if holders:
+ raise Error('Device %s is in use by a device-mapper '
+ 'mapping (dm-crypt?)' % dev, ','.join(holders))
+
+ if check_partitions and not is_partition(dev):
+ for partname in list_partitions(dev):
+ partition = get_dev_path(partname)
+ if is_mounted(partition):
+ raise Error('Device is mounted', partition)
+ holders = is_held(partition)
+ if holders:
+ raise Error('Device %s is in use by a device-mapper '
+ 'mapping (dm-crypt?)'
+ % partition, ','.join(holders))
+
+
+def must_be_one_line(line):
+ """
+ Checks if given line is really one single line.
+
+ :raises: TruncatedLineError or TooManyLinesError
+ :return: Content of the line, or None if line isn't valid.
+ """
+ line = _bytes2str(line)
+
+ if line[-1:] != '\n':
+ raise TruncatedLineError(line)
+ line = line[:-1]
+ if '\n' in line:
+ raise TooManyLinesError(line)
+ return line
+
+
+def read_one_line(parent, name):
+ """
+ Read a file whose sole contents are a single line.
+
+ Strips the newline.
+
+ :return: Contents of the line, or None if file did not exist.
+ """
+ path = os.path.join(parent, name)
+ try:
+ line = open(path, 'rb').read()
+ except IOError as e:
+ if e.errno == errno.ENOENT:
+ return None
+ else:
+ raise
+
+ try:
+ line = must_be_one_line(line)
+ except (TruncatedLineError, TooManyLinesError) as e:
+ raise Error(
+ 'File is corrupt: {path}: {msg}'.format(
+ path=path,
+ msg=e,
+ )
+ )
+ return line
+
+
+def write_one_line(parent, name, text):
+ """
+ Write a file whose sole contents are a single line.
+
+ Adds a newline.
+ """
+ path = os.path.join(parent, name)
+ tmp = '{path}.{pid}.tmp'.format(path=path, pid=os.getpid())
+ with open(tmp, 'wb') as tmp_file:
+ tmp_file.write(text.encode('utf-8') + b'\n')
+ os.fsync(tmp_file.fileno())
+ path_set_context(tmp)
+ os.rename(tmp, path)
+
+
+def init_get():
+ """
+ Get a init system using 'ceph-detect-init'
+ """
+ init = _check_output(
+ args=[
+ 'ceph-detect-init',
+ '--default', 'sysvinit',
+ ],
+ )
+ init = must_be_one_line(init)
+ return init
+
+
+def check_osd_magic(path):
+ """
+ Check that this path has the Ceph OSD magic.
+
+ :raises: BadMagicError if this does not look like a Ceph OSD data
+ dir.
+ """
+ magic = read_one_line(path, 'magic')
+ if magic is None:
+ # probably not mkfs'ed yet
+ raise BadMagicError(path)
+ if magic != CEPH_OSD_ONDISK_MAGIC:
+ raise BadMagicError(path)
+
+
+def check_osd_id(osd_id):
+ """
+ Ensures osd id is numeric.
+ """
+ if not re.match(r'^[0-9]+$', osd_id):
+ raise Error('osd id is not numeric', osd_id)
+
+
+def allocate_osd_id(
+ cluster,
+ fsid,
+ keyring,
+ path,
+):
+ """
+ Allocates an OSD id on the given cluster.
+
+ :raises: Error if the call to allocate the OSD id fails.
+ :return: The allocated OSD id.
+ """
+ lockbox_path = os.path.join(STATEDIR, 'osd-lockbox', fsid)
+ lockbox_osd_id = read_one_line(lockbox_path, 'whoami')
+ osd_keyring = os.path.join(path, 'keyring')
+ if lockbox_osd_id:
+ LOG.debug('Getting OSD id from Lockbox...')
+ osd_id = lockbox_osd_id
+ shutil.move(os.path.join(lockbox_path, 'osd_keyring'),
+ osd_keyring)
+ path_set_context(osd_keyring)
+ os.unlink(os.path.join(lockbox_path, 'whoami'))
+ return osd_id
+
+ LOG.debug('Allocating OSD id...')
+ secrets = Secrets()
+ try:
+ wanttobe = read_one_line(path, 'wanttobe')
+ if os.path.exists(os.path.join(path, 'wanttobe')):
+ os.unlink(os.path.join(path, 'wanttobe'))
+ id_arg = wanttobe and [wanttobe] or []
+ osd_id = command_with_stdin(
+ [
+ 'ceph',
+ '--cluster', cluster,
+ '--name', 'client.bootstrap-osd',
+ '--keyring', keyring,
+ '-i', '-',
+ 'osd', 'new',
+ fsid,
+ ] + id_arg,
+ secrets.get_json()
+ )
+ except subprocess.CalledProcessError as e:
+ raise Error('ceph osd create failed', e, e.output)
+ osd_id = must_be_one_line(osd_id)
+ check_osd_id(osd_id)
+ secrets.write_osd_keyring(osd_keyring, osd_id)
+ return osd_id
+
+
+def get_osd_id(path):
+ """
+ Gets the OSD id of the OSD at the given path.
+ """
+ osd_id = read_one_line(path, 'whoami')
+ if osd_id is not None:
+ check_osd_id(osd_id)
+ return osd_id
+
+
+def get_ceph_user():
+ global CEPH_PREF_USER
+
+ if CEPH_PREF_USER is not None:
+ try:
+ pwd.getpwnam(CEPH_PREF_USER)
+ return CEPH_PREF_USER
+ except KeyError:
+ print("No such user:", CEPH_PREF_USER)
+ sys.exit(2)
+ else:
+ try:
+ pwd.getpwnam('ceph')
+ return 'ceph'
+ except KeyError:
+ return 'root'
+
+
+def get_ceph_group():
+ global CEPH_PREF_GROUP
+
+ if CEPH_PREF_GROUP is not None:
+ try:
+ grp.getgrnam(CEPH_PREF_GROUP)
+ return CEPH_PREF_GROUP
+ except KeyError:
+ print("No such group:", CEPH_PREF_GROUP)
+ sys.exit(2)
+ else:
+ try:
+ grp.getgrnam('ceph')
+ return 'ceph'
+ except KeyError:
+ return 'root'
+
+
+def path_set_context(path):
+ # restore selinux context to default policy values
+ if which('restorecon'):
+ command(['restorecon', '-R', path])
+
+ # if ceph user exists, set owner to ceph
+ if get_ceph_user() == 'ceph':
+ command(['chown', '-R', 'ceph:ceph', path])
+
+
+def _check_output(args=None, **kwargs):
+ out, err, ret = command(args, **kwargs)
+ if ret:
+ cmd = args[0]
+ error = subprocess.CalledProcessError(ret, cmd)
+ error.output = out + err
+ raise error
+ return _bytes2str(out)
+
+
+def get_conf(cluster, variable):
+ """
+ Get the value of the given configuration variable from the
+ cluster.
+
+ :raises: Error if call to ceph-conf fails.
+ :return: The variable value or None.
+ """
+ try:
+ out, err, ret = command(
+ [
+ 'ceph-conf',
+ '--cluster={cluster}'.format(
+ cluster=cluster,
+ ),
+ '--name=osd.',
+ '--lookup',
+ variable,
+ ],
+ close_fds=True,
+ )
+ except OSError as e:
+ raise Error('error executing ceph-conf', e, err)
+ if ret == 1:
+ # config entry not found
+ return None
+ elif ret != 0:
+ raise Error('getting variable from configuration failed')
+ value = out.split('\n', 1)[0]
+ # don't differentiate between "var=" and no var set
+ if not value:
+ return None
+ return value
+
+
+def get_conf_with_default(cluster, variable):
+ """
+ Get a config value that is known to the C++ code.
+
+ This will fail if called on variables that are not defined in
+ common config options.
+ """
+ try:
+ out = _check_output(
+ args=[
+ 'ceph-osd',
+ '--cluster={cluster}'.format(
+ cluster=cluster,
+ ),
+ '--show-config-value={variable}'.format(
+ variable=variable,
+ ),
+ ],
+ close_fds=True,
+ )
+ except subprocess.CalledProcessError as e:
+ raise Error(
+ 'getting variable from configuration failed',
+ e,
+ )
+
+ value = str(out).split('\n', 1)[0]
+ return value
+
+
+def get_fsid(cluster):
+ """
+ Get the fsid of the cluster.
+
+ :return: The fsid or raises Error.
+ """
+ fsid = get_conf_with_default(cluster=cluster, variable='fsid')
+ if fsid is None:
+ raise Error('getting cluster uuid from configuration failed')
+ return fsid.lower()
+
+
+def get_dmcrypt_key_path(
+ _uuid,
+ key_dir,
+ luks
+):
+ """
+ Get path to dmcrypt key file.
+
+ :return: Path to the dmcrypt key file, callers should check for existence.
+ """
+ if luks:
+ path = os.path.join(key_dir, _uuid + ".luks.key")
+ else:
+ path = os.path.join(key_dir, _uuid)
+
+ return path
+
+
+def get_dmcrypt_key(
+ _uuid,
+ key_dir,
+ luks
+):
+ legacy_path = get_dmcrypt_key_path(_uuid, key_dir, luks)
+ if os.path.exists(legacy_path):
+ return (legacy_path,)
+ path = os.path.join(STATEDIR, 'osd-lockbox', _uuid)
+ if os.path.exists(path):
+ mode = get_oneliner(path, 'key-management-mode')
+ osd_uuid = get_oneliner(path, 'osd-uuid')
+ ceph_fsid = read_one_line(path, 'ceph_fsid')
+ if ceph_fsid is None:
+ LOG.warning("no `ceph_fsid` found falling back to 'ceph' "
+ "for cluster name")
+ cluster = 'ceph'
+ else:
+ cluster = find_cluster_by_uuid(ceph_fsid)
+ if cluster is None:
+ raise Error('No cluster conf found in ' + SYSCONFDIR +
+ ' with fsid %s' % ceph_fsid)
+
+ if mode == KEY_MANAGEMENT_MODE_V1:
+ key, stderr, ret = command(
+ [
+ 'ceph',
+ '--cluster', cluster,
+ '--name',
+ 'client.osd-lockbox.' + osd_uuid,
+ '--keyring',
+ os.path.join(path, 'keyring'),
+ 'config-key',
+ 'get',
+ 'dm-crypt/osd/' + osd_uuid + '/luks',
+ ],
+ )
+ LOG.debug("stderr " + stderr)
+ assert ret == 0
+ return base64.b64decode(key)
+ else:
+ raise Error('unknown key-management-mode ' + str(mode))
+ raise Error('unable to read dm-crypt key', path, legacy_path)
+
+
+def _dmcrypt_map(
+ rawdev,
+ key,
+ _uuid,
+ cryptsetup_parameters,
+ luks,
+ format_dev=False,
+):
+ dev = dmcrypt_is_mapped(_uuid)
+ if dev:
+ return dev
+
+ if isinstance(key, tuple):
+ # legacy, before lockbox
+ assert os.path.exists(key[0])
+ keypath = key[0]
+ key = None
+ else:
+ keypath = '-'
+ dev = '/dev/mapper/' + _uuid
+ luksFormat_args = [
+ 'cryptsetup',
+ '--batch-mode',
+ '--key-file',
+ keypath,
+ 'luksFormat',
+ rawdev,
+ ] + cryptsetup_parameters
+
+ luksOpen_args = [
+ 'cryptsetup',
+ '--key-file',
+ keypath,
+ 'luksOpen',
+ rawdev,
+ _uuid,
+ ]
+
+ create_args = [
+ 'cryptsetup',
+ '--key-file',
+ keypath,
+ 'create',
+ _uuid,
+ rawdev,
+ ] + cryptsetup_parameters
+
+ try:
+ if luks:
+ if format_dev:
+ command_with_stdin(luksFormat_args, key)
+ command_with_stdin(luksOpen_args, key)
+ else:
+ # Plain mode has no format function, nor any validation
+ # that the key is correct.
+ command_with_stdin(create_args, key)
+ # set proper ownership of mapped device
+ command_check_call(['chown', 'ceph:ceph', dev])
+ return dev
+
+ except subprocess.CalledProcessError as e:
+ raise Error('unable to map device', rawdev, e)
+
+
+@retry(Error, max_tries=10, wait=0.5, backoff=1.0)
+def dmcrypt_unmap(_uuid):
+ if not os.path.exists('/dev/mapper/' + _uuid):
+ return
+ try:
+ command_check_call(['cryptsetup', 'remove', _uuid])
+ except subprocess.CalledProcessError as e:
+ raise Error('unable to unmap device', _uuid, e)
+
+
+def mount(
+ dev,
+ fstype,
+ options,
+):
+ """
+ Mounts a device with given filessystem type and
+ mount options to a tempfile path under /var/lib/ceph/tmp.
+ """
+ # sanity check: none of the arguments are None
+ if dev is None:
+ raise ValueError('dev may not be None')
+ if fstype is None:
+ raise ValueError('fstype may not be None')
+
+ # pick best-of-breed mount options based on fs type
+ if options is None:
+ options = MOUNT_OPTIONS.get(fstype, '')
+
+ myTemp = STATEDIR + '/tmp'
+ # mkdtemp expect 'dir' to be existing on the system
+ # Let's be sure it's always the case
+ if not os.path.exists(myTemp):
+ os.makedirs(myTemp)
+
+ # mount
+ path = tempfile.mkdtemp(
+ prefix='mnt.',
+ dir=myTemp,
+ )
+ try:
+ LOG.debug('Mounting %s on %s with options %s', dev, path, options)
+ command_check_call(
+ [
+ 'mount',
+ '-t', fstype,
+ '-o', options,
+ '--',
+ dev,
+ path,
+ ],
+ )
+ if which('restorecon'):
+ command(
+ [
+ 'restorecon',
+ path,
+ ],
+ )
+ except subprocess.CalledProcessError as e:
+ try:
+ os.rmdir(path)
+ except (OSError, IOError):
+ pass
+ raise MountError(e)
+
+ return path
+
+
+@retry(UnmountError, max_tries=3, wait=0.5, backoff=1.0)
+def unmount(
+ path,
+ do_rm=True,
+):
+ """
+ Unmount and removes the given mount point.
+ """
+ try:
+ LOG.debug('Unmounting %s', path)
+ command_check_call(
+ [
+ '/bin/umount',
+ '--',
+ path,
+ ],
+ )
+ except subprocess.CalledProcessError as e:
+ raise UnmountError(e)
+ if not do_rm:
+ return
+ os.rmdir(path)
+
+
+###########################################
+
+def extract_parted_partition_numbers(partitions):
+ numbers_as_strings = re.findall('^\d+', partitions, re.MULTILINE)
+ return map(int, numbers_as_strings)
+
+
+def get_free_partition_index(dev):
+ """
+ Get the next free partition index on a given device.
+
+ :return: Index number (> 1 if there is already a partition on the device)
+ or 1 if there is no partition table.
+ """
+ try:
+ lines = _check_output(
+ args=[
+ 'parted',
+ '--machine',
+ '--',
+ dev,
+ 'print',
+ ],
+ )
+ except subprocess.CalledProcessError as e:
+ LOG.info('cannot read partition index; assume it '
+ 'isn\'t present\n (Error: %s)' % e)
+ return 1
+
+ if not lines:
+ raise Error('parted failed to output anything')
+ LOG.debug('get_free_partition_index: analyzing ' + lines)
+ if ('CHS;' not in lines and
+ 'CYL;' not in lines and
+ 'BYT;' not in lines):
+ raise Error('parted output expected to contain one of ' +
+ 'CHH; CYL; or BYT; : ' + lines)
+ if os.path.realpath(dev) not in lines:
+ raise Error('parted output expected to contain ' + dev + ': ' + lines)
+ _, partitions = lines.split(os.path.realpath(dev))
+ partition_numbers = extract_parted_partition_numbers(partitions)
+ if partition_numbers:
+ return max(partition_numbers) + 1
+ else:
+ return 1
+
+
+def check_journal_reqs(args):
+ _, _, allows_journal = command([
+ 'ceph-osd', '--check-allows-journal',
+ '-i', '0',
+ '--log-file', '$run_dir/$cluster-osd-check.log',
+ '--cluster', args.cluster,
+ '--setuser', get_ceph_user(),
+ '--setgroup', get_ceph_group(),
+ ])
+ _, _, wants_journal = command([
+ 'ceph-osd', '--check-wants-journal',
+ '-i', '0',
+ '--log-file', '$run_dir/$cluster-osd-check.log',
+ '--cluster', args.cluster,
+ '--setuser', get_ceph_user(),
+ '--setgroup', get_ceph_group(),
+ ])
+ _, _, needs_journal = command([
+ 'ceph-osd', '--check-needs-journal',
+ '-i', '0',
+ '--log-file', '$run_dir/$cluster-osd-check.log',
+ '--cluster', args.cluster,
+ '--setuser', get_ceph_user(),
+ '--setgroup', get_ceph_group(),
+ ])
+ return (not allows_journal, not wants_journal, not needs_journal)
+
+
+def update_partition(dev, description):
+ """
+ Must be called after modifying a partition table so the kernel
+ know about the change and fire udev events accordingly. A side
+ effect of partprobe is to remove partitions and add them again.
+ The first udevadm settle waits for ongoing udev events to
+ complete, just in case one of them rely on an existing partition
+ on dev. The second udevadm settle guarantees to the caller that
+ all udev events related to the partition table change have been
+ processed, i.e. the 95-ceph-osd.rules actions and mode changes,
+ group changes etc. are complete.
+ """
+ LOG.debug('Calling partprobe on %s device %s', description, dev)
+ partprobe_ok = False
+ error = 'unknown error'
+ partprobe = _get_command_executable(['partprobe'])[0]
+ for i in range(5):
+ command_check_call(['udevadm', 'settle', '--timeout=600'])
+ try:
+ _check_output(['flock', '-s', dev, partprobe, dev])
+ partprobe_ok = True
+ break
+ except subprocess.CalledProcessError as e:
+ error = e.output
+ if ('unable to inform the kernel' not in error and
+ 'Device or resource busy' not in error):
+ raise
+ LOG.debug('partprobe %s failed : %s (ignored, waiting 60s)'
+ % (dev, error))
+ time.sleep(60)
+ if not partprobe_ok:
+ raise Error('partprobe %s failed : %s' % (dev, error))
+ command_check_call(['udevadm', 'settle', '--timeout=600'])
+
+
+def zap_linux(dev):
+ try:
+ # Thoroughly wipe all partitions of any traces of
+ # Filesystems or OSD Journals
+ #
+ # In addition we need to write 10M of data to each partition
+ # to make sure that after re-creating the same partition
+ # there is no trace left of any previous Filesystem or OSD
+ # Journal
+
+ LOG.debug('Writing zeros to existing partitions on %s', dev)
+
+ for partname in list_partitions(dev):
+ partition = get_dev_path(partname)
+ command_check_call(
+ [
+ 'wipefs',
+ '--all',
+ partition,
+ ],
+ )
+
+ command_check_call(
+ [
+ 'dd',
+ 'if=/dev/zero',
+ 'of={path}'.format(path=partition),
+ 'bs=1M',
+ 'count=10',
+ ],
+ )
+
+ LOG.debug('Zapping partition table on %s', dev)
+
+ # try to wipe out any GPT partition table backups. sgdisk
+ # isn't too thorough.
+ lba_size = 4096
+ size = 33 * lba_size
+ with open(dev, 'wb') as dev_file:
+ dev_file.seek(-size, os.SEEK_END)
+ dev_file.write(size * b'\0')
+
+ command_check_call(
+ [
+ 'sgdisk',
+ '--zap-all',
+ '--',
+ dev,
+ ],
+ )
+ command_check_call(
+ [
+ 'sgdisk',
+ '--clear',
+ '--mbrtogpt',
+ '--',
+ dev,
+ ],
+ )
+ update_partition(dev, 'zapped')
+
+ except subprocess.CalledProcessError as e:
+ raise Error(e)
+
+
+def zap_freebsd(dev):
+ try:
+ # For FreeBSD we just need to zap the partition.
+ command_check_call(
+ [
+ 'gpart',
+ 'destroy',
+ '-F',
+ dev,
+ ],
+ )
+
+ except subprocess.CalledProcessError as e:
+ raise Error(e)
+
+
+def zap(dev):
+ """
+ Destroy the partition table and content of a given disk.
+ """
+ dev = os.path.realpath(dev)
+ dmode = os.stat(dev).st_mode
+ if not stat.S_ISBLK(dmode) or is_partition(dev):
+ raise Error('not full block device; cannot zap', dev)
+ if FREEBSD:
+ zap_freebsd(dev)
+ else:
+ zap_linux(dev)
+
+
+def adjust_symlink(target, path):
+ create = True
+ if os.path.lexists(path):
+ try:
+ mode = os.lstat(path).st_mode
+ if stat.S_ISREG(mode):
+ LOG.debug('Removing old file %s', path)
+ os.unlink(path)
+ elif stat.S_ISLNK(mode):
+ old = os.readlink(path)
+ if old != target:
+ LOG.debug('Removing old symlink %s -> %s', path, old)
+ os.unlink(path)
+ else:
+ create = False
+ except:
+ raise Error('unable to remove (or adjust) old file (symlink)',
+ path)
+ if create:
+ LOG.debug('Creating symlink %s -> %s', path, target)
+ try:
+ os.symlink(target, path)
+ except:
+ raise Error('unable to create symlink %s -> %s' % (path, target))
+
+
+def get_mount_options(cluster, fs_type):
+ mount_options = get_conf(
+ cluster,
+ variable='osd_mount_options_{fstype}'.format(
+ fstype=fs_type,
+ ),
+ )
+ if mount_options is None:
+ mount_options = get_conf(
+ cluster,
+ variable='osd_fs_mount_options_{fstype}'.format(
+ fstype=fs_type,
+ ),
+ )
+ else:
+ # remove whitespaces
+ mount_options = "".join(mount_options.split())
+ return mount_options
+
+
+class Device(object):
+
+ def __init__(self, path, args):
+ self.args = args
+ self.path = path
+ self.dev_size = None
+ self.partitions = {}
+ self.ptype_map = None
+ assert not is_partition(self.path)
+
+ def create_partition(self, uuid, name, size=0, num=0):
+ ptype = self.ptype_tobe_for_name(name)
+ if num == 0:
+ num = get_free_partition_index(dev=self.path)
+ if size > 0:
+ new = '--new={num}:0:+{size}M'.format(num=num, size=size)
+ if size > self.get_dev_size():
+ LOG.error('refusing to create %s on %s' % (name, self.path))
+ LOG.error('%s size (%sM) is bigger than device (%sM)'
+ % (name, size, self.get_dev_size()))
+ raise Error('%s device size (%sM) is not big enough for %s'
+ % (self.path, self.get_dev_size(), name))
+ else:
+ new = '--largest-new={num}'.format(num=num)
+
+ LOG.debug('Creating %s partition num %d size %d on %s',
+ name, num, size, self.path)
+ command_check_call(
+ [
+ 'sgdisk',
+ new,
+ '--change-name={num}:ceph {name}'.format(num=num, name=name),
+ '--partition-guid={num}:{uuid}'.format(num=num, uuid=uuid),
+ '--typecode={num}:{uuid}'.format(num=num, uuid=ptype),
+ '--mbrtogpt',
+ '--',
+ self.path,
+ ],
+ exit=True
+ )
+ update_partition(self.path, 'created')
+ return num
+
+ def ptype_tobe_for_name(self, name):
+ LOG.debug("name = " + name)
+ if name == 'data':
+ name = 'osd'
+ if name == 'lockbox':
+ if is_mpath(self.path):
+ return PTYPE['mpath']['lockbox']['tobe']
+ else:
+ return PTYPE['regular']['lockbox']['tobe']
+ if self.ptype_map is None:
+ partition = DevicePartition.factory(
+ path=self.path, dev=None, args=self.args)
+ self.ptype_map = partition.ptype_map
+ return self.ptype_map[name]['tobe']
+
+ def get_partition(self, num):
+ if num not in self.partitions:
+ dev = get_partition_dev(self.path, num)
+ partition = DevicePartition.factory(
+ path=self.path, dev=dev, args=self.args)
+ partition.set_partition_number(num)
+ self.partitions[num] = partition
+ return self.partitions[num]
+
+ def get_dev_size(self):
+ if self.dev_size is None:
+ self.dev_size = get_dev_size(self.path)
+ return self.dev_size
+
+ @staticmethod
+ def factory(path, args):
+ return Device(path, args)
+
+
+class DevicePartition(object):
+
+ def __init__(self, args):
+ self.args = args
+ self.num = None
+ self.rawdev = None
+ self.dev = None
+ self.uuid = None
+ self.ptype_map = None
+ self.ptype = None
+ self.set_variables_ptype()
+
+ def get_uuid(self):
+ if self.uuid is None:
+ self.uuid = get_partition_uuid(self.rawdev)
+ return self.uuid
+
+ def get_ptype(self):
+ if self.ptype is None:
+ self.ptype = get_partition_type(self.rawdev)
+ return self.ptype
+
+ def set_partition_number(self, num):
+ self.num = num
+
+ def get_partition_number(self):
+ return self.num
+
+ def set_dev(self, dev):
+ self.dev = dev
+ self.rawdev = dev
+
+ def get_dev(self):
+ return self.dev
+
+ def get_rawdev(self):
+ return self.rawdev
+
+ def set_variables_ptype(self):
+ self.ptype_map = PTYPE['regular']
+
+ def ptype_for_name(self, name):
+ return self.ptype_map[name]['ready']
+
+ @staticmethod
+ @retry(OSError)
+ def factory(path, dev, args):
+ dmcrypt_type = CryptHelpers.get_dmcrypt_type(args)
+ if ((path is not None and is_mpath(path)) or
+ (dev is not None and is_mpath(dev))):
+ partition = DevicePartitionMultipath(args)
+ elif dmcrypt_type == 'luks':
+ partition = DevicePartitionCryptLuks(args)
+ elif dmcrypt_type == 'plain':
+ partition = DevicePartitionCryptPlain(args)
+ else:
+ partition = DevicePartition(args)
+ partition.set_dev(dev)
+ return partition
+
+
+class DevicePartitionMultipath(DevicePartition):
+
+ def set_variables_ptype(self):
+ self.ptype_map = PTYPE['mpath']
+
+
+class DevicePartitionCrypt(DevicePartition):
+
+ def __init__(self, args):
+ super(DevicePartitionCrypt, self).__init__(args)
+ self.osd_dm_key = None
+ self.cryptsetup_parameters = CryptHelpers.get_cryptsetup_parameters(
+ self.args)
+ self.dmcrypt_type = CryptHelpers.get_dmcrypt_type(self.args)
+ self.dmcrypt_keysize = CryptHelpers.get_dmcrypt_keysize(self.args)
+
+ def setup_crypt(self):
+ pass
+
+ def map(self):
+ self.setup_crypt()
+ self.dev = _dmcrypt_map(
+ rawdev=self.rawdev,
+ key=self.osd_dm_key,
+ _uuid=self.get_uuid(),
+ cryptsetup_parameters=self.cryptsetup_parameters,
+ luks=self.luks(),
+ format_dev=True,
+ )
+
+ def unmap(self):
+ self.setup_crypt()
+ dmcrypt_unmap(self.get_uuid())
+ self.dev = self.rawdev
+
+ def format(self):
+ self.setup_crypt()
+ self.map()
+
+
+class DevicePartitionCryptPlain(DevicePartitionCrypt):
+
+ def luks(self):
+ return False
+
+ def setup_crypt(self):
+ if self.osd_dm_key is not None:
+ return
+
+ self.cryptsetup_parameters += ['--key-size', str(self.dmcrypt_keysize)]
+
+ self.osd_dm_key = get_dmcrypt_key(
+ self.get_uuid(), self.args.dmcrypt_key_dir,
+ False)
+
+ def set_variables_ptype(self):
+ self.ptype_map = PTYPE['plain']
+
+
+class DevicePartitionCryptLuks(DevicePartitionCrypt):
+
+ def luks(self):
+ return True
+
+ def setup_crypt(self):
+ if self.osd_dm_key is not None:
+ return
+
+ if self.dmcrypt_keysize == 1024:
+ # We don't force this into the cryptsetup_parameters,
+ # as we want the cryptsetup defaults
+ # to prevail for the actual LUKS key lengths.
+ pass
+ else:
+ self.cryptsetup_parameters += ['--key-size',
+ str(self.dmcrypt_keysize)]
+
+ self.osd_dm_key = get_dmcrypt_key(
+ self.get_uuid(), self.args.dmcrypt_key_dir,
+ True)
+
+ def set_variables_ptype(self):
+ self.ptype_map = PTYPE['luks']
+
+
+class Prepare(object):
+
+ def __init__(self, args):
+ self.args = args
+
+ @staticmethod
+ def parser():
+ parser = argparse.ArgumentParser(add_help=False)
+ parser.add_argument(
+ '--cluster',
+ metavar='NAME',
+ default='ceph',
+ help='cluster name to assign this disk to',
+ )
+ parser.add_argument(
+ '--cluster-uuid',
+ metavar='UUID',
+ help='cluster uuid to assign this disk to',
+ )
+ parser.add_argument(
+ '--osd-uuid',
+ metavar='UUID',
+ help='unique OSD uuid to assign this disk to',
+ )
+ parser.add_argument(
+ '--osd-id',
+ metavar='ID',
+ help='unique OSD id to assign this disk to',
+ )
+ parser.add_argument(
+ '--crush-device-class',
+ help='crush device class to assign this disk to',
+ )
+ parser.add_argument(
+ '--dmcrypt',
+ action='store_true', default=None,
+ help='encrypt DATA and/or JOURNAL devices with dm-crypt',
+ )
+ parser.add_argument(
+ '--dmcrypt-key-dir',
+ metavar='KEYDIR',
+ default='/etc/ceph/dmcrypt-keys',
+ help='directory where dm-crypt keys are stored',
+ )
+ parser.add_argument(
+ '--prepare-key',
+ metavar='PATH',
+ help='bootstrap-osd keyring path template (%(default)s)',
+ default='{statedir}/bootstrap-osd/{cluster}.keyring',
+ dest='prepare_key_template',
+ )
+ parser.add_argument(
+ '--no-locking',
+ action='store_true', default=None,
+ help='let many prepare\'s run in parallel',
+ )
+ return parser
+
+ @staticmethod
+ def set_subparser(subparsers):
+ parents = [
+ Prepare.parser(),
+ PrepareData.parser(),
+ Lockbox.parser(),
+ ]
+ parents.extend(PrepareFilestore.parent_parsers())
+ parents.extend(PrepareBluestore.parent_parsers())
+ parser = subparsers.add_parser(
+ 'prepare',
+ parents=parents,
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\
+ If the --bluestore argument is given, a bluestore objectstore
+ will be created. If --filestore is provided, a legacy FileStore
+ objectstore will be created. If neither is specified, we default
+ to BlueStore.
+
+ When an entire device is prepared for bluestore, two
+ partitions are created. The first partition is for metadata,
+ the second partition is for blocks that contain data.
+
+ Unless explicitly specified with --block.db or
+ --block.wal, the bluestore DB and WAL data is stored on
+ the main block device. For instance:
+
+ ceph-disk prepare --bluestore /dev/sdc
+
+ Will create
+
+ /dev/sdc1 for osd metadata
+ /dev/sdc2 for block, db, and wal data (the rest of the disk)
+
+
+ If either --block.db or --block.wal are specified to be
+ the same whole device, they will be created as partition
+ three and four respectively. For instance:
+
+ ceph-disk prepare --bluestore \\
+ --block.db /dev/sdc \\
+ --block.wal /dev/sdc \\
+ /dev/sdc
+
+ Will create
+
+ /dev/sdc1 for osd metadata
+ /dev/sdc2 for block (the rest of the disk)
+ /dev/sdc3 for db
+ /dev/sdc4 for wal
+
+ """)),
+ help='Prepare a directory or disk for a Ceph OSD',
+ )
+ parser.set_defaults(
+ func=Prepare.main,
+ )
+ return parser
+
+ def prepare(self):
+ if self.args.no_locking:
+ self._prepare()
+ else:
+ with prepare_lock:
+ self._prepare()
+
+ @staticmethod
+ def factory(args):
+ if args.bluestore:
+ return PrepareBluestore(args)
+ else:
+ return PrepareFilestore(args)
+
+ @staticmethod
+ def main(args):
+ Prepare.factory(args).prepare()
+
+
+class PrepareFilestore(Prepare):
+
+ def __init__(self, args):
+ super(PrepareFilestore, self).__init__(args)
+ if args.dmcrypt:
+ self.lockbox = Lockbox(args)
+ self.data = PrepareFilestoreData(args)
+ self.journal = PrepareJournal(args)
+
+ @staticmethod
+ def parent_parsers():
+ return [
+ PrepareJournal.parser(),
+ ]
+
+ def _prepare(self):
+ if self.data.args.dmcrypt:
+ self.lockbox.prepare()
+ self.data.prepare(self.journal)
+
+
+class PrepareBluestore(Prepare):
+
+ def __init__(self, args):
+ super(PrepareBluestore, self).__init__(args)
+ if args.dmcrypt:
+ self.lockbox = Lockbox(args)
+ self.data = PrepareBluestoreData(args)
+ self.block = PrepareBluestoreBlock(args)
+ self.blockdb = PrepareBluestoreBlockDB(args)
+ self.blockwal = PrepareBluestoreBlockWAL(args)
+
+ @staticmethod
+ def parser():
+ parser = argparse.ArgumentParser(add_help=False)
+ parser.add_argument(
+ '--bluestore',
+ dest='bluestore',
+ action='store_true', default=True,
+ help='bluestore objectstore',
+ )
+ parser.add_argument(
+ '--filestore',
+ dest='bluestore',
+ action='store_false',
+ help='filestore objectstore',
+ )
+ return parser
+
+ @staticmethod
+ def parent_parsers():
+ return [
+ PrepareBluestore.parser(),
+ PrepareBluestoreBlock.parser(),
+ PrepareBluestoreBlockDB.parser(),
+ PrepareBluestoreBlockWAL.parser(),
+ ]
+
+ def _prepare(self):
+ if self.data.args.dmcrypt:
+ self.lockbox.prepare()
+ to_prepare_list = []
+ if getattr(self.data.args, 'block.db'):
+ to_prepare_list.append(self.blockdb)
+ if getattr(self.data.args, 'block.wal'):
+ to_prepare_list.append(self.blockwal)
+ to_prepare_list.append(self.block)
+ self.data.prepare(*to_prepare_list)
+
+
+class Space(object):
+
+ NAMES = ('block', 'journal', 'block.db', 'block.wal')
+
+
+class PrepareSpace(object):
+
+ NONE = 0
+ FILE = 1
+ DEVICE = 2
+
+ def __init__(self, args):
+ self.args = args
+ self.set_type()
+ self.space_size = self.get_space_size()
+ if getattr(self.args, self.name + '_uuid') is None:
+ setattr(self.args, self.name + '_uuid', str(uuid.uuid4()))
+ self.space_symlink = None
+ self.space_dmcrypt = None
+
+ def set_type(self):
+ name = self.name
+ args = self.args
+ if (self.wants_space() and
+ dev_is_diskdevice(args.data) and
+ not is_partition(args.data) and
+ getattr(args, name) is None and
+ getattr(args, name + '_file') is None):
+ LOG.info('Will colocate %s with data on %s',
+ name, args.data)
+ setattr(args, name, args.data)
+
+ if getattr(args, name) is None:
+ if getattr(args, name + '_dev'):
+ raise Error('%s is unspecified; not a block device' %
+ name.capitalize(), getattr(args, name))
+ self.type = self.NONE
+ return
+
+ if not os.path.exists(getattr(args, name)):
+ if getattr(args, name + '_dev'):
+ raise Error('%s does not exist; not a block device' %
+ name.capitalize(), getattr(args, name))
+ self.type = self.FILE
+ return
+
+ mode = os.stat(getattr(args, name)).st_mode
+ if stmode_is_diskdevice(mode):
+ if getattr(args, name + '_file'):
+ raise Error('%s is not a regular file' % name.capitalize,
+ getattr(args, name))
+ self.type = self.DEVICE
+ return
+
+ if stat.S_ISREG(mode):
+ if getattr(args, name + '_dev'):
+ raise Error('%s is not a block device' % name.capitalize,
+ getattr(args, name))
+ self.type = self.FILE
+ return
+
+ raise Error('%s %s is neither a block device nor regular file' %
+ (name.capitalize, getattr(args, name)))
+
+ def is_none(self):
+ return self.type == self.NONE
+
+ def is_file(self):
+ return self.type == self.FILE
+
+ def is_device(self):
+ return self.type == self.DEVICE
+
+ @staticmethod
+ def parser(name, positional=True):
+ parser = argparse.ArgumentParser(add_help=False)
+ parser.add_argument(
+ '--%s-uuid' % name,
+ metavar='UUID',
+ help='unique uuid to assign to the %s' % name,
+ )
+ parser.add_argument(
+ '--%s-file' % name,
+ action='store_true', default=None,
+ help='verify that %s is a file' % name.upper(),
+ )
+ parser.add_argument(
+ '--%s-dev' % name,
+ action='store_true', default=None,
+ help='verify that %s is a block device' % name.upper(),
+ )
+
+ if positional:
+ parser.add_argument(
+ name,
+ metavar=name.upper(),
+ nargs='?',
+ help=('path to OSD %s disk block device;' % name +
+ ' leave out to store %s in file' % name),
+ )
+ return parser
+
+ def wants_space(self):
+ return True
+
+ def populate_data_path(self, path):
+ if self.type == self.DEVICE:
+ self.populate_data_path_device(path)
+ elif self.type == self.FILE:
+ self.populate_data_path_file(path)
+ elif self.type == self.NONE:
+ pass
+ else:
+ raise Error('unexpected type ', self.type)
+
+ def populate_data_path_file(self, path):
+ space_uuid = self.name + '_uuid'
+ if getattr(self.args, space_uuid) is not None:
+ write_one_line(path, space_uuid,
+ getattr(self.args, space_uuid))
+ if self.space_symlink is not None:
+ adjust_symlink(self.space_symlink,
+ os.path.join(path, self.name))
+
+ def populate_data_path_device(self, path):
+ self.populate_data_path_file(path)
+
+ if self.space_dmcrypt is not None:
+ adjust_symlink(self.space_dmcrypt,
+ os.path.join(path, self.name + '_dmcrypt'))
+ else:
+ try:
+ os.unlink(os.path.join(path, self.name + '_dmcrypt'))
+ except OSError:
+ pass
+
+ def prepare(self):
+ if self.type == self.DEVICE:
+ self.prepare_device()
+ elif self.type == self.FILE:
+ self.prepare_file()
+ elif self.type == self.NONE:
+ pass
+ else:
+ raise Error('unexpected type ', self.type)
+
+ def prepare_file(self):
+ space_filename = getattr(self.args, self.name)
+ if not os.path.exists(space_filename):
+ LOG.debug('Creating %s file %s with size 0'
+ ' (ceph-osd will resize and allocate)',
+ self.name,
+ space_filename)
+ space_file = open(space_filename, 'wb')
+ space_file.close()
+ path_set_context(space_filename)
+
+ LOG.debug('%s is file %s',
+ self.name.capitalize(),
+ space_filename)
+ LOG.warning('OSD will not be hot-swappable if %s is '
+ 'not the same device as the osd data' %
+ self.name)
+ self.space_symlink = space_filename
+
+ def prepare_device(self):
+ reusing_partition = False
+
+ if is_partition(getattr(self.args, self.name)):
+ LOG.debug('%s %s is a partition',
+ self.name.capitalize(), getattr(self.args, self.name))
+ partition = DevicePartition.factory(
+ path=None, dev=getattr(self.args, self.name), args=self.args)
+ if isinstance(partition, DevicePartitionCrypt):
+ raise Error(getattr(self.args, self.name) +
+ ' partition already exists'
+ ' and --dmcrypt specified')
+ LOG.warning('OSD will not be hot-swappable' +
+ ' if ' + self.name + ' is not' +
+ ' the same device as the osd data')
+ if partition.get_ptype() == partition.ptype_for_name(self.name):
+ LOG.debug('%s %s was previously prepared with '
+ 'ceph-disk. Reusing it.',
+ self.name.capitalize(),
+ getattr(self.args, self.name))
+ reusing_partition = True
+ # Read and reuse the partition uuid from this journal's
+ # previous life. We reuse the uuid instead of changing it
+ # because udev does not reliably notice changes to an
+ # existing partition's GUID. See
+ # http://tracker.ceph.com/issues/10146
+ setattr(self.args, self.name + '_uuid', partition.get_uuid())
+ LOG.debug('Reusing %s with uuid %s',
+ self.name,
+ getattr(self.args, self.name + '_uuid'))
+ else:
+ LOG.warning('%s %s was not prepared with '
+ 'ceph-disk. Symlinking directly.',
+ self.name.capitalize(),
+ getattr(self.args, self.name))
+ self.space_symlink = getattr(self.args, self.name)
+ return
+
+ self.space_symlink = '/dev/disk/by-partuuid/{uuid}'.format(
+ uuid=getattr(self.args, self.name + '_uuid'))
+
+ if self.args.dmcrypt:
+ self.space_dmcrypt = self.space_symlink
+ self.space_symlink = '/dev/mapper/{uuid}'.format(
+ uuid=getattr(self.args, self.name + '_uuid'))
+
+ if reusing_partition:
+ # confirm that the space_symlink exists. It should since
+ # this was an active space
+ # in the past. Continuing otherwise would be futile.
+ assert os.path.exists(self.space_symlink)
+ return
+
+ num = self.desired_partition_number()
+
+ if num == 0:
+ LOG.warning('OSD will not be hot-swappable if %s '
+ 'is not the same device as the osd data',
+ self.name)
+
+ device = Device.factory(getattr(self.args, self.name), self.args)
+ num = device.create_partition(
+ uuid=getattr(self.args, self.name + '_uuid'),
+ name=self.name,
+ size=self.space_size,
+ num=num)
+
+ partition = device.get_partition(num)
+
+ LOG.debug('%s is GPT partition %s',
+ self.name.capitalize(),
+ self.space_symlink)
+
+ if isinstance(partition, DevicePartitionCrypt):
+ partition.format()
+ partition.map()
+
+ command_check_call(
+ [
+ 'sgdisk',
+ '--typecode={num}:{uuid}'.format(
+ num=num,
+ uuid=partition.ptype_for_name(self.name),
+ ),
+ '--',
+ getattr(self.args, self.name),
+ ],
+ )
+ update_partition(getattr(self.args, self.name), 'prepared')
+
+ LOG.debug('%s is GPT partition %s',
+ self.name.capitalize(),
+ self.space_symlink)
+
+
+class PrepareJournal(PrepareSpace):
+
+ def __init__(self, args):
+ self.name = 'journal'
+ (self.allows_journal,
+ self.wants_journal,
+ self.needs_journal) = check_journal_reqs(args)
+
+ if args.journal and not self.allows_journal:
+ raise Error('journal specified but not allowed by osd backend')
+
+ super(PrepareJournal, self).__init__(args)
+
+ def wants_space(self):
+ return self.wants_journal
+
+ def get_space_size(self):
+ return int(get_conf_with_default(
+ cluster=self.args.cluster,
+ variable='osd_journal_size',
+ ))
+
+ def desired_partition_number(self):
+ if self.args.journal == self.args.data:
+ # we're sharing the disk between osd data and journal;
+ # make journal be partition number 2
+ num = 2
+ else:
+ num = 0
+ return num
+
+ @staticmethod
+ def parser():
+ return PrepareSpace.parser('journal')
+
+
+class PrepareBluestoreBlock(PrepareSpace):
+
+ def __init__(self, args):
+ self.name = 'block'
+ super(PrepareBluestoreBlock, self).__init__(args)
+
+ def get_space_size(self):
+ block_size = get_conf(
+ cluster=self.args.cluster,
+ variable='bluestore_block_size',
+ )
+
+ if block_size is None:
+ return 0 # get as much space as possible
+ else:
+ return int(block_size) / 1048576 # MB
+
+ def desired_partition_number(self):
+ if self.args.block == self.args.data:
+ num = 2
+ else:
+ num = 0
+ return num
+
+ @staticmethod
+ def parser():
+ return PrepareSpace.parser('block')
+
+
+class PrepareBluestoreBlockDB(PrepareSpace):
+
+ def __init__(self, args):
+ self.name = 'block.db'
+ super(PrepareBluestoreBlockDB, self).__init__(args)
+
+ def get_space_size(self):
+ block_db_size = get_conf(
+ cluster=self.args.cluster,
+ variable='bluestore_block_db_size',
+ )
+
+ if block_db_size is None or int(block_db_size) == 0:
+ block_size = get_conf(
+ cluster=self.args.cluster,
+ variable='bluestore_block_size',
+ )
+ if block_size is None:
+ return 1024 # MB
+ size = int(block_size) / 100 / 1048576
+ return max(size, 1024) # MB
+ else:
+ return int(block_db_size) / 1048576 # MB
+
+ def desired_partition_number(self):
+ if getattr(self.args, 'block.db') == self.args.data:
+ num = 3
+ else:
+ num = 0
+ return num
+
+ def wants_space(self):
+ return False
+
+ @staticmethod
+ def parser():
+ parser = PrepareSpace.parser('block.db', positional=False)
+ parser.add_argument(
+ '--block.db',
+ metavar='BLOCKDB',
+ help='path to the device or file for bluestore block.db',
+ )
+ return parser
+
+
+class PrepareBluestoreBlockWAL(PrepareSpace):
+
+ def __init__(self, args):
+ self.name = 'block.wal'
+ super(PrepareBluestoreBlockWAL, self).__init__(args)
+
+ def get_space_size(self):
+ block_size = get_conf(
+ cluster=self.args.cluster,
+ variable='bluestore_block_wal_size',
+ )
+
+ if block_size is None:
+ return 576 # MB, default value
+ else:
+ return int(block_size) / 1048576 # MB
+
+ def desired_partition_number(self):
+ if getattr(self.args, 'block.wal') == self.args.data:
+ num = 4
+ else:
+ num = 0
+ return num
+
+ def wants_space(self):
+ return False
+
+ @staticmethod
+ def parser():
+ parser = PrepareSpace.parser('block.wal', positional=False)
+ parser.add_argument(
+ '--block.wal',
+ metavar='BLOCKWAL',
+ help='path to the device or file for bluestore block.wal',
+ )
+ return parser
+
+
+class CryptHelpers(object):
+
+ @staticmethod
+ def get_cryptsetup_parameters(args):
+ cryptsetup_parameters_str = get_conf(
+ cluster=args.cluster,
+ variable='osd_cryptsetup_parameters',
+ )
+ if cryptsetup_parameters_str is None:
+ return []
+ else:
+ return shlex.split(cryptsetup_parameters_str)
+
+ @staticmethod
+ def get_dmcrypt_keysize(args):
+ dmcrypt_keysize_str = get_conf(
+ cluster=args.cluster,
+ variable='osd_dmcrypt_key_size',
+ )
+ dmcrypt_type = CryptHelpers.get_dmcrypt_type(args)
+ if dmcrypt_type == 'luks':
+ if dmcrypt_keysize_str is None:
+ # As LUKS will hash the 'passphrase' in .luks.key
+ # into a key, set a large default
+ # so if not updated for some time, it is still a
+ # reasonable value.
+ #
+ return 1024
+ else:
+ return int(dmcrypt_keysize_str)
+ elif dmcrypt_type == 'plain':
+ if dmcrypt_keysize_str is None:
+ # This value is hard-coded in the udev script
+ return 256
+ else:
+ LOG.warning('ensure the 95-ceph-osd.rules file has '
+ 'been copied to /etc/udev/rules.d '
+ 'and modified to call cryptsetup '
+ 'with --key-size=%s' % dmcrypt_keysize_str)
+ return int(dmcrypt_keysize_str)
+ else:
+ return 0
+
+ @staticmethod
+ def get_dmcrypt_type(args):
+ if hasattr(args, 'dmcrypt') and args.dmcrypt:
+ dmcrypt_type = get_conf(
+ cluster=args.cluster,
+ variable='osd_dmcrypt_type',
+ )
+
+ if dmcrypt_type is None or dmcrypt_type == 'luks':
+ return 'luks'
+ elif dmcrypt_type == 'plain':
+ return 'plain'
+ else:
+ raise Error('invalid osd_dmcrypt_type parameter '
+ '(must be luks or plain): ', dmcrypt_type)
+ else:
+ return None
+
+
+class Secrets(object):
+
+ def __init__(self):
+ secret, stderr, ret = command(['ceph-authtool', '--gen-print-key'])
+ LOG.debug("stderr " + stderr)
+ assert ret == 0
+ self.keys = {
+ 'cephx_secret': secret.strip(),
+ }
+
+ def write_osd_keyring(self, keyring, osd_id):
+ command_check_call(
+ [
+ 'ceph-authtool', keyring,
+ '--create-keyring',
+ '--name', 'osd.' + str(osd_id),
+ '--add-key', self.keys['cephx_secret'],
+ ])
+ path_set_context(keyring)
+
+ def get_json(self):
+ return bytearray(json.dumps(self.keys), 'ascii')
+
+
+class LockboxSecrets(Secrets):
+
+ def __init__(self, args):
+ super(LockboxSecrets, self).__init__()
+
+ key_size = CryptHelpers.get_dmcrypt_keysize(args)
+ key = open('/dev/urandom', 'rb').read(key_size / 8)
+ base64_key = base64.b64encode(key).decode('ascii')
+
+ secret, stderr, ret = command(['ceph-authtool', '--gen-print-key'])
+ LOG.debug("stderr " + stderr)
+ assert ret == 0
+
+ self.keys.update({
+ 'dmcrypt_key': base64_key,
+ 'cephx_lockbox_secret': secret.strip(),
+ })
+
+ def write_lockbox_keyring(self, path, osd_uuid):
+ keyring = os.path.join(path, 'keyring')
+ command_check_call(
+ [
+ 'ceph-authtool', keyring,
+ '--create-keyring',
+ '--name', 'client.osd-lockbox.' + osd_uuid,
+ '--add-key', self.keys['cephx_lockbox_secret'],
+ ])
+ path_set_context(keyring)
+
+
+class Lockbox(object):
+
+ def __init__(self, args):
+ self.args = args
+ self.partition = None
+ self.device = None
+
+ if hasattr(self.args, 'lockbox') and self.args.lockbox is None:
+ self.args.lockbox = self.args.data
+
+ def set_partition(self, partition):
+ self.partition = partition
+
+ @staticmethod
+ def parser():
+ parser = argparse.ArgumentParser(add_help=False)
+ parser.add_argument(
+ '--lockbox',
+ help='path to the device to store the lockbox',
+ )
+ parser.add_argument(
+ '--lockbox-uuid',
+ metavar='UUID',
+ help='unique lockbox uuid',
+ )
+ return parser
+
+ def create_partition(self):
+ self.device = Device.factory(self.args.lockbox, argparse.Namespace())
+ partition_number = 5
+ self.device.create_partition(uuid=self.args.lockbox_uuid,
+ name='lockbox',
+ num=partition_number,
+ size=10) # MB
+ return self.device.get_partition(partition_number)
+
+ def set_or_create_partition(self):
+ if is_partition(self.args.lockbox):
+ LOG.debug('OSD lockbox device %s is a partition',
+ self.args.lockbox)
+ self.partition = DevicePartition.factory(
+ path=None, dev=self.args.lockbox, args=self.args)
+ ptype = self.partition.get_ptype()
+ ready = Ptype.get_ready_by_name('lockbox')
+ if ptype not in ready:
+ LOG.warning('incorrect partition UUID: %s, expected %s'
+ % (ptype, str(ready)))
+ else:
+ LOG.debug('Creating osd partition on %s',
+ self.args.lockbox)
+ self.partition = self.create_partition()
+
+ def create_key(self):
+ cluster = self.args.cluster
+ bootstrap = self.args.prepare_key_template.format(cluster=cluster,
+ statedir=STATEDIR)
+ path = self.get_mount_point()
+ secrets = LockboxSecrets(self.args)
+ id_arg = self.args.osd_id and [self.args.osd_id] or []
+ osd_id = command_with_stdin(
+ [
+ 'ceph',
+ '--cluster', cluster,
+ '--name', 'client.bootstrap-osd',
+ '--keyring', bootstrap,
+ '-i', '-',
+ 'osd', 'new', self.args.osd_uuid,
+ ] + id_arg,
+ secrets.get_json()
+ )
+ secrets.write_lockbox_keyring(path, self.args.osd_uuid)
+ osd_id = must_be_one_line(osd_id)
+ check_osd_id(osd_id)
+ write_one_line(path, 'whoami', osd_id)
+ secrets.write_osd_keyring(os.path.join(path, 'osd_keyring'), osd_id)
+ write_one_line(path, 'key-management-mode', KEY_MANAGEMENT_MODE_V1)
+
+ def symlink_spaces(self, path):
+ target = self.get_mount_point()
+ for name in Space.NAMES:
+ if (hasattr(self.args, name + '_uuid') and
+ getattr(self.args, name + '_uuid')):
+ uuid = getattr(self.args, name + '_uuid')
+ symlink = os.path.join(STATEDIR, 'osd-lockbox', uuid)
+ adjust_symlink(target, symlink)
+ write_one_line(path, name + '-uuid', uuid)
+
+ def populate(self):
+ maybe_mkdir(os.path.join(STATEDIR, 'osd-lockbox'))
+ args = ['mkfs', '-t', 'ext4', self.partition.get_dev()]
+ LOG.debug('Creating lockbox fs on %s: ' + str(" ".join(args)))
+ command_check_call(args)
+ path = self.get_mount_point()
+ maybe_mkdir(path)
+ args = ['mount', '-t', 'ext4', self.partition.get_dev(), path]
+ LOG.debug('Mounting lockbox ' + str(" ".join(args)))
+ command_check_call(args)
+ write_one_line(path, 'osd-uuid', self.args.osd_uuid)
+ if self.args.cluster_uuid is None:
+ self.args.cluster_uuid = get_fsid(cluster=self.args.cluster)
+ write_one_line(path, 'ceph_fsid', self.args.cluster_uuid)
+ self.create_key()
+ self.symlink_spaces(path)
+ write_one_line(path, 'magic', CEPH_LOCKBOX_ONDISK_MAGIC)
+ if self.device is not None:
+ command_check_call(
+ [
+ 'sgdisk',
+ '--typecode={num}:{uuid}'.format(
+ num=self.partition.get_partition_number(),
+ uuid=self.partition.ptype_for_name('lockbox'),
+ ),
+ '--',
+ get_partition_base(self.partition.get_dev()),
+ ],
+ )
+
+ def get_mount_point(self):
+ return os.path.join(STATEDIR, 'osd-lockbox', self.args.osd_uuid)
+
+ def get_osd_uuid(self):
+ return self.args.osd_uuid
+
+ def activate(self):
+ path = is_mounted(self.partition.get_dev())
+ if path:
+ LOG.info("Lockbox already mounted at " + path)
+ return
+
+ path = tempfile.mkdtemp(
+ prefix='mnt.',
+ dir=STATEDIR + '/tmp',
+ )
+ args = ['mount', '-t', 'ext4', '-o', 'ro',
+ self.partition.get_dev(),
+ path]
+ LOG.debug('Mounting lockbox temporarily ' + str(" ".join(args)))
+ command_check_call(args)
+ self.args.osd_uuid = get_oneliner(path, 'osd-uuid')
+ command_check_call(['umount', path])
+ LOG.debug('Mounting lockbox readonly ' + str(" ".join(args)))
+ args = ['mount', '-t', 'ext4', '-o', 'ro',
+ self.partition.get_dev(),
+ self.get_mount_point()]
+ command_check_call(args)
+ for name in Space.NAMES + ('osd',):
+ uuid_path = os.path.join(self.get_mount_point(), name + '-uuid')
+ if os.path.exists(uuid_path):
+ uuid = get_oneliner(self.get_mount_point(), name + '-uuid')
+ dev = os.path.join('/dev/disk/by-partuuid/', uuid.lower())
+ args = ['ceph-disk', 'trigger', dev]
+ command_check_call(args)
+
+ def prepare(self):
+ verify_not_in_use(self.args.lockbox, check_partitions=True)
+ self.set_or_create_partition()
+ self.populate()
+
+
+class PrepareData(object):
+
+ FILE = 1
+ DEVICE = 2
+
+ def __init__(self, args):
+
+ self.args = args
+ self.partition = None
+ self.set_type()
+ if self.args.cluster_uuid is None:
+ self.args.cluster_uuid = get_fsid(cluster=self.args.cluster)
+
+ if self.args.osd_uuid is None:
+ self.args.osd_uuid = str(uuid.uuid4())
+
+ def set_type(self):
+ dmode = os.stat(self.args.data).st_mode
+
+ if stat.S_ISDIR(dmode):
+ self.type = self.FILE
+ elif stmode_is_diskdevice(dmode):
+ self.type = self.DEVICE
+ else:
+ raise Error('not a dir or block device', self.args.data)
+
+ def is_file(self):
+ return self.type == self.FILE
+
+ def is_device(self):
+ return self.type == self.DEVICE
+
+ @staticmethod
+ def parser():
+ parser = argparse.ArgumentParser(add_help=False)
+ parser.add_argument(
+ '--fs-type',
+ help='file system type to use (e.g. "ext4")',
+ )
+ parser.add_argument(
+ '--zap-disk',
+ action='store_true', default=None,
+ help='destroy the partition table (and content) of a disk',
+ )
+ parser.add_argument(
+ '--data-dir',
+ action='store_true', default=None,
+ help='verify that DATA is a dir',
+ )
+ parser.add_argument(
+ '--data-dev',
+ action='store_true', default=None,
+ help='verify that DATA is a block device',
+ )
+ parser.add_argument(
+ 'data',
+ metavar='DATA',
+ help='path to OSD data (a disk block device or directory)',
+ )
+ return parser
+
+ def populate_data_path_file(self, path, *to_prepare_list):
+ self.populate_data_path(path, *to_prepare_list)
+
+ def populate_data_path(self, path, *to_prepare_list):
+ if os.path.exists(os.path.join(path, 'magic')):
+ LOG.debug('Data dir %s already exists', path)
+ return
+ else:
+ LOG.debug('Preparing osd data dir %s', path)
+
+ if self.args.osd_uuid is None:
+ self.args.osd_uuid = str(uuid.uuid4())
+
+ write_one_line(path, 'ceph_fsid', self.args.cluster_uuid)
+ write_one_line(path, 'fsid', self.args.osd_uuid)
+ if self.args.osd_id:
+ write_one_line(path, 'wanttobe', self.args.osd_id)
+ if self.args.crush_device_class:
+ write_one_line(path, 'crush_device_class',
+ self.args.crush_device_class)
+ write_one_line(path, 'magic', CEPH_OSD_ONDISK_MAGIC)
+
+ for to_prepare in to_prepare_list:
+ to_prepare.populate_data_path(path)
+
+ def prepare(self, *to_prepare_list):
+ if self.type == self.DEVICE:
+ self.prepare_device(*to_prepare_list)
+ elif self.type == self.FILE:
+ self.prepare_file(*to_prepare_list)
+ else:
+ raise Error('unexpected type ', self.type)
+
+ def prepare_file(self, *to_prepare_list):
+
+ if not os.path.exists(self.args.data):
+ raise Error('data path for directory does not exist',
+ self.args.data)
+
+ if self.args.data_dev:
+ raise Error('data path is not a block device', self.args.data)
+
+ for to_prepare in to_prepare_list:
+ to_prepare.prepare()
+
+ self.populate_data_path_file(self.args.data, *to_prepare_list)
+
+ def sanity_checks(self):
+ if not os.path.exists(self.args.data):
+ raise Error('data path for device does not exist',
+ self.args.data)
+ verify_not_in_use(self.args.data,
+ check_partitions=not self.args.dmcrypt)
+
+ def set_variables(self):
+ if self.args.fs_type is None:
+ self.args.fs_type = get_conf(
+ cluster=self.args.cluster,
+ variable='osd_mkfs_type',
+ )
+ if self.args.fs_type is None:
+ self.args.fs_type = get_conf(
+ cluster=self.args.cluster,
+ variable='osd_fs_type',
+ )
+ if self.args.fs_type is None:
+ self.args.fs_type = DEFAULT_FS_TYPE
+
+ self.mkfs_args = get_conf(
+ cluster=self.args.cluster,
+ variable='osd_mkfs_options_{fstype}'.format(
+ fstype=self.args.fs_type,
+ ),
+ )
+ if self.mkfs_args is None:
+ self.mkfs_args = get_conf(
+ cluster=self.args.cluster,
+ variable='osd_fs_mkfs_options_{fstype}'.format(
+ fstype=self.args.fs_type,
+ ),
+ )
+
+ self.mount_options = get_mount_options(cluster=self.args.cluster,
+ fs_type=self.args.fs_type)
+
+ if self.args.osd_uuid is None:
+ self.args.osd_uuid = str(uuid.uuid4())
+
+ def prepare_device(self, *to_prepare_list):
+ self.sanity_checks()
+ self.set_variables()
+ if self.args.zap_disk is not None:
+ zap(self.args.data)
+
+ def create_data_partition(self):
+ device = Device.factory(self.args.data, self.args)
+ partition_number = 1
+ device.create_partition(uuid=self.args.osd_uuid,
+ name='data',
+ num=partition_number,
+ size=self.get_space_size())
+ return device.get_partition(partition_number)
+
+ def set_data_partition(self):
+ if is_partition(self.args.data):
+ LOG.debug('OSD data device %s is a partition',
+ self.args.data)
+ self.partition = DevicePartition.factory(
+ path=None, dev=self.args.data, args=self.args)
+ ptype = self.partition.get_ptype()
+ ready = Ptype.get_ready_by_name('osd')
+ if ptype not in ready:
+ LOG.warning('incorrect partition UUID: %s, expected %s'
+ % (ptype, str(ready)))
+ else:
+ LOG.debug('Creating osd partition on %s',
+ self.args.data)
+ self.partition = self.create_data_partition()
+
+ def populate_data_path_device(self, *to_prepare_list):
+ partition = self.partition
+
+ if isinstance(partition, DevicePartitionCrypt):
+ partition.map()
+
+ try:
+ args = [
+ 'mkfs',
+ '-t',
+ self.args.fs_type,
+ ]
+ if self.mkfs_args is not None:
+ args.extend(self.mkfs_args.split())
+ if self.args.fs_type == 'xfs':
+ args.extend(['-f']) # always force
+ else:
+ args.extend(MKFS_ARGS.get(self.args.fs_type, []))
+ args.extend([
+ '--',
+ partition.get_dev(),
+ ])
+ LOG.debug('Creating %s fs on %s',
+ self.args.fs_type, partition.get_dev())
+ command_check_call(args, exit=True)
+
+ path = mount(dev=partition.get_dev(),
+ fstype=self.args.fs_type,
+ options=self.mount_options)
+
+ try:
+ self.populate_data_path(path, *to_prepare_list)
+ finally:
+ path_set_context(path)
+ unmount(path)
+ finally:
+ if isinstance(partition, DevicePartitionCrypt):
+ partition.unmap()
+
+ if not is_partition(self.args.data):
+ command_check_call(
+ [
+ 'sgdisk',
+ '--typecode=%d:%s' % (partition.get_partition_number(),
+ partition.ptype_for_name('osd')),
+ '--',
+ self.args.data,
+ ],
+ exit=True,
+ )
+ update_partition(self.args.data, 'prepared')
+ command_check_call(['udevadm', 'trigger',
+ '--action=add',
+ '--sysname-match',
+ os.path.basename(partition.rawdev)])
+
+
+class PrepareFilestoreData(PrepareData):
+
+ def get_space_size(self):
+ return 0 # get as much space as possible
+
+ def prepare_device(self, *to_prepare_list):
+ super(PrepareFilestoreData, self).prepare_device(*to_prepare_list)
+ for to_prepare in to_prepare_list:
+ to_prepare.prepare()
+ self.set_data_partition()
+ self.populate_data_path_device(*to_prepare_list)
+
+ def populate_data_path(self, path, *to_prepare_list):
+ super(PrepareFilestoreData, self).populate_data_path(path,
+ *to_prepare_list)
+ write_one_line(path, 'type', 'filestore')
+
+
+class PrepareBluestoreData(PrepareData):
+
+ def get_space_size(self):
+ return 100 # MB
+
+ def prepare_device(self, *to_prepare_list):
+ super(PrepareBluestoreData, self).prepare_device(*to_prepare_list)
+ self.set_data_partition()
+ for to_prepare in to_prepare_list:
+ to_prepare.prepare()
+ self.populate_data_path_device(*to_prepare_list)
+
+ def populate_data_path(self, path, *to_prepare_list):
+ super(PrepareBluestoreData, self).populate_data_path(path,
+ *to_prepare_list)
+ write_one_line(path, 'type', 'bluestore')
+
+
+def mkfs(
+ path,
+ cluster,
+ osd_id,
+ fsid,
+ keyring,
+):
+ monmap = os.path.join(path, 'activate.monmap')
+ command_check_call(
+ [
+ 'ceph',
+ '--cluster', cluster,
+ '--name', 'client.bootstrap-osd',
+ '--keyring', keyring,
+ 'mon', 'getmap', '-o', monmap,
+ ],
+ )
+
+ osd_type = read_one_line(path, 'type')
+
+ if osd_type == 'bluestore':
+ command_check_call(
+ [
+ 'ceph-osd',
+ '--cluster', cluster,
+ '--mkfs',
+ '-i', osd_id,
+ '--monmap', monmap,
+ '--osd-data', path,
+ '--osd-uuid', fsid,
+ '--setuser', get_ceph_user(),
+ '--setgroup', get_ceph_group(),
+ ],
+ )
+ elif osd_type == 'filestore':
+ command_check_call(
+ [
+ 'ceph-osd',
+ '--cluster', cluster,
+ '--mkfs',
+ '-i', osd_id,
+ '--monmap', monmap,
+ '--osd-data', path,
+ '--osd-journal', os.path.join(path, 'journal'),
+ '--osd-uuid', fsid,
+ '--setuser', get_ceph_user(),
+ '--setgroup', get_ceph_group(),
+ ],
+ )
+ else:
+ raise Error('unrecognized objectstore type %s' % osd_type)
+
+
+def get_mount_point(cluster, osd_id):
+ parent = STATEDIR + '/osd'
+ return os.path.join(
+ parent,
+ '{cluster}-{osd_id}'.format(cluster=cluster, osd_id=osd_id),
+ )
+
+
+def move_mount(
+ dev,
+ path,
+ cluster,
+ osd_id,
+ fstype,
+ mount_options,
+):
+ LOG.debug('Moving mount to final location...')
+ osd_data = get_mount_point(cluster, osd_id)
+ maybe_mkdir(osd_data)
+
+ # pick best-of-breed mount options based on fs type
+ if mount_options is None:
+ mount_options = MOUNT_OPTIONS.get(fstype, '')
+
+ # we really want to mount --move, but that is not supported when
+ # the parent mount is shared, as it is by default on RH, Fedora,
+ # and probably others. Also, --bind doesn't properly manipulate
+ # /etc/mtab, which *still* isn't a symlink to /proc/mounts despite
+ # this being 2013. Instead, mount the original device at the final
+ # location.
+ command_check_call(
+ [
+ '/bin/mount',
+ '-o',
+ mount_options,
+ '--',
+ dev,
+ osd_data,
+ ],
+ )
+ command_check_call(
+ [
+ '/bin/umount',
+ '-l', # lazy, in case someone else is peeking at the
+ # wrong moment
+ '--',
+ path,
+ ],
+ )
+
+
+#
+# For upgrade purposes, to make sure there are no competing units,
+# both --runtime unit and the default should be disabled. There can be
+# two units at the same time: one with --runtime and another without
+# it. If, for any reason (manual or ceph-disk) the two units co-exist
+# they will compete with each other.
+#
+def systemd_disable(
+ path,
+ osd_id,
+):
+ # ensure there is no duplicate ceph-osd@.service
+ for style in ([], ['--runtime']):
+ command_check_call(
+ [
+ 'systemctl',
+ 'disable',
+ 'ceph-osd@{osd_id}'.format(osd_id=osd_id),
+ ] + style,
+ )
+
+
+def systemd_start(
+ path,
+ osd_id,
+):
+ systemd_disable(path, osd_id)
+ if os.path.ismount(path):
+ style = ['--runtime']
+ else:
+ style = []
+ command_check_call(
+ [
+ 'systemctl',
+ 'enable',
+ 'ceph-osd@{osd_id}'.format(osd_id=osd_id),
+ ] + style,
+ )
+ command_check_call(
+ [
+ 'systemctl',
+ 'start',
+ 'ceph-osd@{osd_id}'.format(osd_id=osd_id),
+ ],
+ )
+
+
+def systemd_stop(
+ path,
+ osd_id,
+):
+ systemd_disable(path, osd_id)
+ command_check_call(
+ [
+ 'systemctl',
+ 'stop',
+ 'ceph-osd@{osd_id}'.format(osd_id=osd_id),
+ ],
+ )
+
+
+def start_daemon(
+ cluster,
+ osd_id,
+):
+ LOG.debug('Starting %s osd.%s...', cluster, osd_id)
+
+ path = (STATEDIR + '/osd/{cluster}-{osd_id}').format(
+ cluster=cluster, osd_id=osd_id)
+
+ try:
+ if os.path.exists(os.path.join(path, 'upstart')):
+ command_check_call(
+ [
+ '/sbin/initctl',
+ # use emit, not start, because start would fail if the
+ # instance was already running
+ 'emit',
+ # since the daemon starting doesn't guarantee much about
+ # the service being operational anyway, don't bother
+ # waiting for it
+ '--no-wait',
+ '--',
+ 'ceph-osd',
+ 'cluster={cluster}'.format(cluster=cluster),
+ 'id={osd_id}'.format(osd_id=osd_id),
+ ],
+ )
+ elif os.path.exists(os.path.join(path, 'sysvinit')):
+ if os.path.exists('/usr/sbin/service'):
+ svc = '/usr/sbin/service'
+ else:
+ svc = '/sbin/service'
+ command_check_call(
+ [
+ svc,
+ 'ceph',
+ '--cluster',
+ '{cluster}'.format(cluster=cluster),
+ 'start',
+ 'osd.{osd_id}'.format(osd_id=osd_id),
+ ],
+ )
+ elif os.path.exists(os.path.join(path, 'systemd')):
+ systemd_start(path, osd_id)
+ elif os.path.exists(os.path.join(path, 'openrc')):
+ base_script = '/etc/init.d/ceph-osd'
+ osd_script = '{base}.{osd_id}'.format(
+ base=base_script,
+ osd_id=osd_id
+ )
+ if not os.path.exists(osd_script):
+ os.symlink(base_script, osd_script)
+ command_check_call(
+ [
+ osd_script,
+ 'start',
+ ],
+ )
+ elif os.path.exists(os.path.join(path, 'bsdrc')):
+ command_check_call(
+ [
+ '/usr/sbin/service', 'ceph', 'start',
+ 'osd.{osd_id}'.format(osd_id=osd_id),
+ ],
+ )
+ else:
+ raise Error('{cluster} osd.{osd_id} '
+ 'is not tagged with an init system'
+ .format(
+ cluster=cluster,
+ osd_id=osd_id,
+ ))
+ except subprocess.CalledProcessError as e:
+ raise Error('ceph osd start failed', e)
+
+
+def stop_daemon(
+ cluster,
+ osd_id,
+):
+ LOG.debug('Stoping %s osd.%s...', cluster, osd_id)
+
+ path = (STATEDIR + '/osd/{cluster}-{osd_id}').format(
+ cluster=cluster, osd_id=osd_id)
+
+ try:
+ if os.path.exists(os.path.join(path, 'upstart')):
+ command_check_call(
+ [
+ '/sbin/initctl',
+ 'stop',
+ 'ceph-osd',
+ 'cluster={cluster}'.format(cluster=cluster),
+ 'id={osd_id}'.format(osd_id=osd_id),
+ ],
+ )
+ elif os.path.exists(os.path.join(path, 'sysvinit')):
+ svc = which('service')
+ command_check_call(
+ [
+ svc,
+ 'ceph',
+ '--cluster',
+ '{cluster}'.format(cluster=cluster),
+ 'stop',
+ 'osd.{osd_id}'.format(osd_id=osd_id),
+ ],
+ )
+ elif os.path.exists(os.path.join(path, 'systemd')):
+ systemd_stop(path, osd_id)
+ elif os.path.exists(os.path.join(path, 'openrc')):
+ command_check_call(
+ [
+ '/etc/init.d/ceph-osd.{osd_id}'.format(osd_id=osd_id),
+ 'stop',
+ ],
+ )
+ elif os.path.exists(os.path.join(path, 'bsdrc')):
+ command_check_call(
+ [
+ '/usr/local/etc/rc.d/ceph stop osd.{osd_id}'
+ .format(osd_id=osd_id),
+ ],
+ )
+ else:
+ raise Error('{cluster} osd.{osd_id} '
+ 'is not tagged with an init system'
+ .format(cluster=cluster, osd_id=osd_id))
+ except subprocess.CalledProcessError as e:
+ raise Error('ceph osd stop failed', e)
+
+
+def detect_fstype(dev):
+ if FREEBSD:
+ fstype = _check_output(
+ args=[
+ 'fstyp',
+ '-u',
+ dev,
+ ],
+ )
+ else:
+ fstype = _check_output(
+ args=[
+ '/sbin/blkid',
+ # we don't want stale cached results
+ '-p',
+ '-s', 'TYPE',
+ '-o', 'value',
+ '--',
+ dev,
+ ],
+ )
+ fstype = must_be_one_line(fstype)
+ return fstype
+
+
+def dmcrypt_is_mapped(uuid):
+ path = os.path.join('/dev/mapper', uuid)
+ if os.path.exists(path):
+ return path
+ else:
+ return None
+
+
+def dmcrypt_map(dev, dmcrypt_key_dir):
+ ptype = get_partition_type(dev)
+ if ptype in Ptype.get_ready_by_type('plain'):
+ luks = False
+ cryptsetup_parameters = ['--key-size', '256']
+ elif ptype in Ptype.get_ready_by_type('luks'):
+ luks = True
+ cryptsetup_parameters = []
+ else:
+ raise Error('--dmcrypt called for dev %s with invalid ptype %s'
+ % (dev, ptype))
+ part_uuid = get_partition_uuid(dev)
+ dmcrypt_key = get_dmcrypt_key(part_uuid, dmcrypt_key_dir, luks)
+ return _dmcrypt_map(
+ rawdev=dev,
+ key=dmcrypt_key,
+ _uuid=part_uuid,
+ cryptsetup_parameters=cryptsetup_parameters,
+ luks=luks,
+ format_dev=False,
+ )
+
+
+def mount_activate(
+ dev,
+ activate_key_template,
+ init,
+ dmcrypt,
+ dmcrypt_key_dir,
+ reactivate=False,
+):
+
+ if dmcrypt:
+ part_uuid = get_partition_uuid(dev)
+ dev = dmcrypt_map(dev, dmcrypt_key_dir)
+ try:
+ fstype = detect_fstype(dev=dev)
+ except (subprocess.CalledProcessError,
+ TruncatedLineError,
+ TooManyLinesError) as e:
+ raise FilesystemTypeError(
+ 'device {dev}'.format(dev=dev),
+ e,
+ )
+
+ # TODO always using mount options from cluster=ceph for
+ # now; see http://tracker.newdream.net/issues/3253
+ mount_options = get_mount_options(cluster='ceph', fs_type=fstype)
+
+ path = mount(dev=dev, fstype=fstype, options=mount_options)
+
+ # check if the disk is deactive, change the journal owner, group
+ # mode for correct user and group.
+ if os.path.exists(os.path.join(path, 'deactive')):
+ # logging to syslog will help us easy to know udev triggered failure
+ if not reactivate:
+ unmount(path)
+ # we need to unmap again because dmcrypt map will create again
+ # on bootup stage (due to deactivate)
+ if '/dev/mapper/' in dev:
+ part_uuid = dev.replace('/dev/mapper/', '')
+ dmcrypt_unmap(part_uuid)
+ LOG.info('OSD deactivated! reactivate with: --reactivate')
+ raise Error('OSD deactivated! reactivate with: --reactivate')
+ # flag to activate a deactive osd.
+ deactive = True
+ else:
+ deactive = False
+
+ osd_id = None
+ cluster = None
+ try:
+ (osd_id, cluster) = activate(path, activate_key_template, init)
+
+ # Now active successfully
+ # If we got reactivate and deactive, remove the deactive file
+ if deactive and reactivate:
+ os.remove(os.path.join(path, 'deactive'))
+ LOG.info('Remove `deactive` file.')
+
+ # check if the disk is already active, or if something else is already
+ # mounted there
+ active = False
+ other = False
+ src_dev = os.stat(path).st_dev
+ try:
+ dst_dev = os.stat((STATEDIR + '/osd/{cluster}-{osd_id}').format(
+ cluster=cluster,
+ osd_id=osd_id)).st_dev
+ if src_dev == dst_dev:
+ active = True
+ else:
+ parent_dev = os.stat(STATEDIR + '/osd').st_dev
+ if dst_dev != parent_dev:
+ other = True
+ elif os.listdir(get_mount_point(cluster, osd_id)):
+ LOG.info(get_mount_point(cluster, osd_id) +
+ " is not empty, won't override")
+ other = True
+
+ except OSError:
+ pass
+
+ if active:
+ LOG.info('%s osd.%s already mounted in position; unmounting ours.'
+ % (cluster, osd_id))
+ unmount(path)
+ elif other:
+ raise Error('another %s osd.%s already mounted in position '
+ '(old/different cluster instance?); unmounting ours.'
+ % (cluster, osd_id))
+ else:
+ move_mount(
+ dev=dev,
+ path=path,
+ cluster=cluster,
+ osd_id=osd_id,
+ fstype=fstype,
+ mount_options=mount_options,
+ )
+ return cluster, osd_id
+
+ except:
+ LOG.error('Failed to activate')
+ unmount(path)
+ raise
+ finally:
+ # remove our temp dir
+ if os.path.exists(path):
+ os.rmdir(path)
+
+
+def activate_dir(
+ path,
+ activate_key_template,
+ init,
+):
+
+ if not os.path.exists(path):
+ raise Error(
+ 'directory %s does not exist' % path
+ )
+
+ (osd_id, cluster) = activate(path, activate_key_template, init)
+
+ if init not in (None, 'none'):
+ canonical = (STATEDIR + '/osd/{cluster}-{osd_id}').format(
+ cluster=cluster,
+ osd_id=osd_id)
+ if path != canonical:
+ # symlink it from the proper location
+ create = True
+ if os.path.lexists(canonical):
+ old = os.readlink(canonical)
+ if old != path:
+ LOG.debug('Removing old symlink %s -> %s', canonical, old)
+ try:
+ os.unlink(canonical)
+ except:
+ raise Error('unable to remove old symlink', canonical)
+ else:
+ create = False
+ if create:
+ LOG.debug('Creating symlink %s -> %s', canonical, path)
+ try:
+ os.symlink(path, canonical)
+ except:
+ raise Error('unable to create symlink %s -> %s'
+ % (canonical, path))
+
+ return cluster, osd_id
+
+
+def find_cluster_by_uuid(_uuid):
+ """
+ Find a cluster name by searching /etc/ceph/*.conf for a conf file
+ with the right uuid.
+ """
+ _uuid = _uuid.lower()
+ no_fsid = []
+ if not os.path.exists(SYSCONFDIR):
+ return None
+ for conf_file in os.listdir(SYSCONFDIR):
+ if not conf_file.endswith('.conf'):
+ continue
+ cluster = conf_file[:-5]
+ try:
+ fsid = get_fsid(cluster)
+ except Error as e:
+ if 'getting cluster uuid from configuration failed' not in str(e):
+ raise e
+ no_fsid.append(cluster)
+ else:
+ if fsid == _uuid:
+ return cluster
+ # be tolerant of /etc/ceph/ceph.conf without an fsid defined.
+ if len(no_fsid) == 1 and no_fsid[0] == 'ceph':
+ LOG.warning('No fsid defined in ' + SYSCONFDIR +
+ '/ceph.conf; using anyway')
+ return 'ceph'
+ return None
+
+
+def activate(
+ path,
+ activate_key_template,
+ init,
+):
+
+ check_osd_magic(path)
+
+ ceph_fsid = read_one_line(path, 'ceph_fsid')
+ if ceph_fsid is None:
+ raise Error('No cluster uuid assigned.')
+ LOG.debug('Cluster uuid is %s', ceph_fsid)
+
+ cluster = find_cluster_by_uuid(ceph_fsid)
+ if cluster is None:
+ raise Error('No cluster conf found in ' + SYSCONFDIR +
+ ' with fsid %s' % ceph_fsid)
+ LOG.debug('Cluster name is %s', cluster)
+
+ fsid = read_one_line(path, 'fsid')
+ if fsid is None:
+ raise Error('No OSD uuid assigned.')
+ LOG.debug('OSD uuid is %s', fsid)
+
+ keyring = activate_key_template.format(cluster=cluster,
+ statedir=STATEDIR)
+
+ osd_id = get_osd_id(path)
+ if osd_id is None:
+ osd_id = allocate_osd_id(
+ cluster=cluster,
+ fsid=fsid,
+ keyring=keyring,
+ path=path,
+ )
+ write_one_line(path, 'whoami', osd_id)
+ LOG.debug('OSD id is %s', osd_id)
+
+ if not os.path.exists(os.path.join(path, 'ready')):
+ LOG.debug('Initializing OSD...')
+ # re-running mkfs is safe, so just run until it completes
+ mkfs(
+ path=path,
+ cluster=cluster,
+ osd_id=osd_id,
+ fsid=fsid,
+ keyring=keyring,
+ )
+
+ if init not in (None, 'none'):
+ if init == 'auto':
+ conf_val = get_conf(
+ cluster=cluster,
+ variable='init'
+ )
+ if conf_val is not None:
+ init = conf_val
+ else:
+ init = init_get()
+
+ LOG.debug('Marking with init system %s', init)
+ init_path = os.path.join(path, init)
+ with open(init_path, 'w'):
+ path_set_context(init_path)
+
+ # remove markers for others, just in case.
+ for other in INIT_SYSTEMS:
+ if other != init:
+ try:
+ os.unlink(os.path.join(path, other))
+ except OSError:
+ pass
+
+ if not os.path.exists(os.path.join(path, 'active')):
+ write_one_line(path, 'active', 'ok')
+ LOG.debug('%s osd.%s data dir is ready at %s', cluster, osd_id, path)
+ return (osd_id, cluster)
+
+
+def main_activate(args):
+ cluster = None
+ osd_id = None
+
+ LOG.info('path = ' + str(args.path))
+ if not os.path.exists(args.path):
+ raise Error('%s does not exist' % args.path)
+
+ if is_suppressed(args.path):
+ LOG.info('suppressed activate request on %s', args.path)
+ return
+
+ with activate_lock:
+ mode = os.stat(args.path).st_mode
+ if stmode_is_diskdevice(mode):
+ if (is_partition(args.path) and
+ (get_partition_type(args.path) ==
+ PTYPE['mpath']['osd']['ready']) and
+ not is_mpath(args.path)):
+ raise Error('%s is not a multipath block device' %
+ args.path)
+ (cluster, osd_id) = mount_activate(
+ dev=args.path,
+ activate_key_template=args.activate_key_template,
+ init=args.mark_init,
+ dmcrypt=args.dmcrypt,
+ dmcrypt_key_dir=args.dmcrypt_key_dir,
+ reactivate=args.reactivate,
+ )
+ osd_data = get_mount_point(cluster, osd_id)
+
+ args.cluster = cluster
+ if args.dmcrypt:
+ for name in Space.NAMES:
+ # Check if encrypted device in journal
+ dev_path = os.path.join(osd_data, name + '_dmcrypt')
+ if not os.path.exists(dev_path):
+ continue
+ partition = DevicePartition.factory(
+ path=None,
+ dev=dev_path,
+ args=args)
+ partition.rawdev = args.path
+ partition.map()
+
+ elif stat.S_ISDIR(mode):
+ (cluster, osd_id) = activate_dir(
+ path=args.path,
+ activate_key_template=args.activate_key_template,
+ init=args.mark_init,
+ )
+ osd_data = args.path
+
+ else:
+ raise Error('%s is not a directory or block device' % args.path)
+
+ # exit with 0 if the journal device is not up, yet
+ # journal device will do the activation
+ osd_journal = '{path}/journal'.format(path=osd_data)
+ if os.path.islink(osd_journal) and not os.access(osd_journal, os.F_OK):
+ LOG.info("activate: Journal not present, not starting, yet")
+ return
+
+ if (not args.no_start_daemon and args.mark_init == 'none'):
+ command_check_call(
+ [
+ 'ceph-osd',
+ '--cluster={cluster}'.format(cluster=cluster),
+ '--id={osd_id}'.format(osd_id=osd_id),
+ '--osd-data={path}'.format(path=osd_data),
+ '--osd-journal={journal}'.format(journal=osd_journal),
+ ],
+ )
+
+ if (not args.no_start_daemon and
+ args.mark_init not in (None, 'none')):
+
+ start_daemon(
+ cluster=cluster,
+ osd_id=osd_id,
+ )
+
+
+def main_activate_lockbox(args):
+ with activate_lock:
+ main_activate_lockbox_protected(args)
+
+
+def main_activate_lockbox_protected(args):
+ partition = DevicePartition.factory(
+ path=None, dev=args.path, args=args)
+
+ lockbox = Lockbox(args)
+ lockbox.set_partition(partition)
+ lockbox.activate()
+
+
+###########################
+
+def _mark_osd_out(cluster, osd_id):
+ LOG.info('Prepare to mark osd.%d out...', osd_id)
+ command([
+ 'ceph',
+ 'osd',
+ 'out',
+ 'osd.%d' % osd_id,
+ ])
+
+
+def _check_osd_status(cluster, osd_id):
+ """
+ report the osd status:
+ 00(0) : means OSD OUT AND DOWN
+ 01(1) : means OSD OUT AND UP
+ 10(2) : means OSD IN AND DOWN
+ 11(3) : means OSD IN AND UP
+ """
+ LOG.info("Checking osd id: %s ..." % osd_id)
+ found = False
+ status_code = 0
+ out, err, ret = command([
+ 'ceph',
+ 'osd',
+ 'dump',
+ '--cluster={cluster}'.format(
+ cluster=cluster,
+ ),
+ '--format',
+ 'json',
+ ])
+ out_json = json.loads(out)
+ for item in out_json[u'osds']:
+ if item.get(u'osd') == int(osd_id):
+ found = True
+ if item.get(u'in') is 1:
+ status_code += 2
+ if item.get(u'up') is 1:
+ status_code += 1
+ if not found:
+ raise Error('Could not osd.%s in osd tree!' % osd_id)
+ return status_code
+
+
+def _remove_osd_directory_files(mounted_path, cluster):
+ """
+ To remove the 'ready', 'active', INIT-specific files.
+ """
+ if os.path.exists(os.path.join(mounted_path, 'ready')):
+ os.remove(os.path.join(mounted_path, 'ready'))
+ LOG.info('Remove `ready` file.')
+ else:
+ LOG.info('`ready` file is already removed.')
+
+ if os.path.exists(os.path.join(mounted_path, 'active')):
+ os.remove(os.path.join(mounted_path, 'active'))
+ LOG.info('Remove `active` file.')
+ else:
+ LOG.info('`active` file is already removed.')
+
+ # Just check `upstart` and `sysvinit` directly if filename is init-spec.
+ conf_val = get_conf(
+ cluster=cluster,
+ variable='init'
+ )
+ if conf_val is not None:
+ init = conf_val
+ else:
+ init = init_get()
+ os.remove(os.path.join(mounted_path, init))
+ LOG.info('Remove `%s` file.', init)
+ return
+
+
+def main_deactivate(args):
+ with activate_lock:
+ main_deactivate_locked(args)
+
+
+def main_deactivate_locked(args):
+ osd_id = args.deactivate_by_id
+ path = args.path
+ target_dev = None
+ dmcrypt = False
+ devices = list_devices()
+
+ # list all devices and found we need
+ for device in devices:
+ if 'partitions' in device:
+ for dev_part in device.get('partitions'):
+ if (osd_id and
+ 'whoami' in dev_part and
+ dev_part['whoami'] == osd_id):
+ target_dev = dev_part
+ elif (path and
+ 'path' in dev_part and
+ dev_part['path'] == path):
+ target_dev = dev_part
+ if not target_dev:
+ raise Error('Cannot find any match device!!')
+
+ # set up all we need variable
+ osd_id = target_dev['whoami']
+ part_type = target_dev['ptype']
+ mounted_path = target_dev['mount']
+ if Ptype.is_dmcrypt(part_type, 'osd'):
+ dmcrypt = True
+
+ # Do not do anything if osd is already down.
+ status_code = _check_osd_status(args.cluster, osd_id)
+ if status_code == OSD_STATUS_IN_UP:
+ if args.mark_out is True:
+ _mark_osd_out(args.cluster, int(osd_id))
+ stop_daemon(args.cluster, osd_id)
+ elif status_code == OSD_STATUS_IN_DOWN:
+ if args.mark_out is True:
+ _mark_osd_out(args.cluster, int(osd_id))
+ LOG.info("OSD already out/down. Do not do anything now.")
+ return
+ elif status_code == OSD_STATUS_OUT_UP:
+ stop_daemon(args.cluster, osd_id)
+ elif status_code == OSD_STATUS_OUT_DOWN:
+ LOG.info("OSD already out/down. Do not do anything now.")
+ return
+
+ if not args.once:
+ # remove 'ready', 'active', and INIT-specific files.
+ _remove_osd_directory_files(mounted_path, args.cluster)
+
+ # Write deactivate to osd directory!
+ with open(os.path.join(mounted_path, 'deactive'), 'w'):
+ path_set_context(os.path.join(mounted_path, 'deactive'))
+
+ unmount(mounted_path, do_rm=not args.once)
+ LOG.info("Umount `%s` successfully.", mounted_path)
+
+ if dmcrypt:
+ lockbox = os.path.join(STATEDIR, 'osd-lockbox')
+ command(['umount', os.path.join(lockbox, target_dev['uuid'])])
+
+ dmcrypt_unmap(target_dev['uuid'])
+ for name in Space.NAMES:
+ if name + '_uuid' in target_dev:
+ dmcrypt_unmap(target_dev[name + '_uuid'])
+
+###########################
+
+
+def _remove_lockbox(uuid, cluster):
+ lockbox = os.path.join(STATEDIR, 'osd-lockbox')
+ if not os.path.exists(lockbox):
+ return
+ canonical = os.path.join(lockbox, uuid)
+ command(['umount', canonical])
+ for name in os.listdir(lockbox):
+ path = os.path.join(lockbox, name)
+ if os.path.islink(path) and os.readlink(path) == canonical:
+ os.unlink(path)
+
+
+def destroy_lookup_device(args, predicate, description):
+ devices = list_devices()
+ for device in devices:
+ for partition in device.get('partitions', []):
+ if partition['type'] == 'lockbox':
+ if not is_mounted(partition['path']):
+ main_activate_lockbox_protected(
+ argparse.Namespace(verbose=args.verbose,
+ path=partition['path']))
+ for device in devices:
+ for partition in device.get('partitions', []):
+ if partition['dmcrypt']:
+ dmcrypt_path = dmcrypt_is_mapped(partition['uuid'])
+ if dmcrypt_path:
+ unmap = False
+ else:
+ dmcrypt_path = dmcrypt_map(partition['path'],
+ args.dmcrypt_key_dir)
+ unmap = True
+ list_dev_osd(dmcrypt_path, {}, partition)
+ if unmap:
+ dmcrypt_unmap(partition['uuid'])
+ dmcrypt = True
+ else:
+ dmcrypt = False
+ if predicate(partition):
+ return dmcrypt, partition
+ raise Error('found no device matching ', description)
+
+
+def main_destroy(args):
+ with activate_lock:
+ main_destroy_locked(args)
+
+
+def main_destroy_locked(args):
+ osd_id = args.destroy_by_id
+ path = args.path
+ target_dev = None
+
+ if path:
+ if not is_partition(path):
+ raise Error(path + " must be a partition device")
+ path = os.path.realpath(path)
+
+ if path:
+ (dmcrypt, target_dev) = destroy_lookup_device(
+ args, lambda x: x.get('path') == path,
+ path)
+ elif osd_id:
+ (dmcrypt, target_dev) = destroy_lookup_device(
+ args, lambda x: x.get('whoami') == osd_id,
+ 'osd id ' + str(osd_id))
+
+ osd_id = target_dev['whoami']
+ dev_path = target_dev['path']
+ if target_dev['ptype'] == PTYPE['mpath']['osd']['ready']:
+ base_dev = get_partition_base_mpath(dev_path)
+ else:
+ base_dev = get_partition_base(dev_path)
+
+ # Before osd deactivate, we cannot destroy it
+ status_code = _check_osd_status(args.cluster, osd_id)
+ if status_code != OSD_STATUS_OUT_DOWN and \
+ status_code != OSD_STATUS_IN_DOWN:
+ raise Error("Could not destroy the active osd. (osd-id: %s)" %
+ osd_id)
+
+ if args.purge:
+ action = 'purge'
+ else:
+ action = 'destroy'
+ LOG.info("Prepare to %s osd.%s" % (action, osd_id))
+ command([
+ 'ceph',
+ 'osd',
+ action,
+ 'osd.%s' % osd_id,
+ '--yes-i-really-mean-it',
+ ])
+
+ # we remove the crypt map and device mapper (if dmcrypt is True)
+ if dmcrypt:
+ for name in Space.NAMES:
+ if target_dev.get(name + '_uuid'):
+ dmcrypt_unmap(target_dev[name + '_uuid'])
+ _remove_lockbox(target_dev['uuid'], args.cluster)
+
+ # Check zap flag. If we found zap flag, we need to find device for
+ # destroy this osd data.
+ if args.zap is True:
+ # erase the osd data
+ LOG.info("Prepare to zap the device %s" % base_dev)
+ zap(base_dev)
+
+
+def get_space_osd_uuid(name, path):
+ if not os.path.exists(path):
+ raise Error('%s does not exist' % path)
+
+ if not path_is_diskdevice(path):
+ raise Error('%s is not a block device' % path)
+
+ if (is_partition(path) and
+ get_partition_type(path) in (PTYPE['mpath']['journal']['ready'],
+ PTYPE['mpath']['block']['ready']) and
+ not is_mpath(path)):
+ raise Error('%s is not a multipath block device' %
+ path)
+
+ try:
+ out = _check_output(
+ args=[
+ 'ceph-osd',
+ '--get-device-fsid',
+ path,
+ ],
+ close_fds=True,
+ )
+ except subprocess.CalledProcessError as e:
+ raise Error(
+ 'failed to get osd uuid/fsid from %s' % name,
+ e,
+ )
+ value = str(out).split('\n', 1)[0]
+ LOG.debug('%s %s has OSD UUID %s', name.capitalize(), path, value)
+ return value
+
+
+def main_activate_space(name, args):
+ if not os.path.exists(args.dev):
+ raise Error('%s does not exist' % args.dev)
+
+ if is_suppressed(args.dev):
+ LOG.info('suppressed activate request on space %s', args.dev)
+ return
+
+ cluster = None
+ osd_id = None
+ osd_uuid = None
+ dev = None
+ with activate_lock:
+ if args.dmcrypt:
+ dev = dmcrypt_map(args.dev, args.dmcrypt_key_dir)
+ else:
+ dev = args.dev
+ # FIXME: For an encrypted journal dev, does this return the
+ # cyphertext or plaintext dev uuid!? Also, if the journal is
+ # encrypted, is the data partition also always encrypted, or
+ # are mixed pairs supported!?
+ osd_uuid = get_space_osd_uuid(name, dev)
+ path = os.path.join('/dev/disk/by-partuuid/', osd_uuid.lower())
+
+ if is_suppressed(path):
+ LOG.info('suppressed activate request on %s', path)
+ return
+
+ # warn and exit with 0 if the data device is not up, yet
+ # data device will do the activation
+ if not os.access(path, os.F_OK):
+ LOG.info("activate: OSD device not present, not starting, yet")
+ return
+
+ (cluster, osd_id) = mount_activate(
+ dev=path,
+ activate_key_template=args.activate_key_template,
+ init=args.mark_init,
+ dmcrypt=args.dmcrypt,
+ dmcrypt_key_dir=args.dmcrypt_key_dir,
+ reactivate=args.reactivate,
+ )
+
+ start_daemon(
+ cluster=cluster,
+ osd_id=osd_id,
+ )
+
+
+###########################
+
+
+def main_activate_all(args):
+ dir = '/dev/disk/by-parttypeuuid'
+ LOG.debug('Scanning %s', dir)
+ if not os.path.exists(dir):
+ return
+ err = False
+ for name in os.listdir(dir):
+ if name.find('.') < 0:
+ continue
+ (tag, uuid) = name.split('.')
+
+ if tag in Ptype.get_ready_by_name('osd'):
+
+ if Ptype.is_dmcrypt(tag, 'osd'):
+ path = os.path.join('/dev/mapper', uuid)
+ else:
+ path = os.path.join(dir, name)
+
+ if is_suppressed(path):
+ LOG.info('suppressed activate request on %s', path)
+ continue
+
+ LOG.info('Activating %s', path)
+ with activate_lock:
+ try:
+ # never map dmcrypt cyphertext devices
+ (cluster, osd_id) = mount_activate(
+ dev=path,
+ activate_key_template=args.activate_key_template,
+ init=args.mark_init,
+ dmcrypt=False,
+ dmcrypt_key_dir='',
+ )
+ start_daemon(
+ cluster=cluster,
+ osd_id=osd_id,
+ )
+
+ except Exception as e:
+ print(
+ '{prog}: {msg}'.format(prog=args.prog, msg=e),
+ file=sys.stderr
+ )
+
+ err = True
+
+ if err:
+ raise Error('One or more partitions failed to activate')
+
+
+###########################
+
+def is_swap(dev):
+ dev = os.path.realpath(dev)
+ with open(PROCDIR + '/swaps', 'rb') as proc_swaps:
+ for line in proc_swaps.readlines()[1:]:
+ fields = line.split()
+ if len(fields) < 3:
+ continue
+ swaps_dev = fields[0]
+ if os.path.isabs(swaps_dev) and os.path.exists(swaps_dev):
+ swaps_dev = os.path.realpath(swaps_dev)
+ if swaps_dev == dev:
+ return True
+ return False
+
+
+def get_oneliner(base, name):
+ path = os.path.join(base, name)
+ if os.path.isfile(path):
+ with open(path, 'rb') as _file:
+ return _bytes2str(_file.readline().rstrip())
+ return None
+
+
+def get_dev_fs(dev):
+ if FREEBSD:
+ fstype, _, ret = command(
+ [
+ 'fstyp',
+ '-u',
+ dev,
+ ],
+ )
+ if ret == 0:
+ return fstype
+ else:
+ fscheck, _, _ = command(
+ [
+ 'blkid',
+ '-s',
+ 'TYPE',
+ dev,
+ ],
+ )
+ if 'TYPE' in fscheck:
+ fstype = fscheck.split()[1].split('"')[1]
+ return fstype
+ return None
+
+
+def split_dev_base_partnum(dev):
+ if is_mpath(dev):
+ partnum = partnum_mpath(dev)
+ base = get_partition_base_mpath(dev)
+ else:
+ b = block_path(dev)
+ partnum = open(os.path.join(b, 'partition')).read().strip()
+ base = get_partition_base(dev)
+ return base, partnum
+
+
+def get_partition_type(part):
+ return get_blkid_partition_info(part, 'ID_PART_ENTRY_TYPE')
+
+
+def get_partition_uuid(part):
+ return get_blkid_partition_info(part, 'ID_PART_ENTRY_UUID')
+
+
+def get_blkid_partition_info(dev, what=None):
+ out, _, _ = command(
+ [
+ 'blkid',
+ '-o',
+ 'udev',
+ '-p',
+ dev,
+ ]
+ )
+ p = {}
+ for line in out.splitlines():
+ (key, value) = line.split('=')
+ p[key] = value
+ if what:
+ return p.get(what)
+ else:
+ return p
+
+
+def more_osd_info(path, uuid_map, desc):
+ desc['ceph_fsid'] = get_oneliner(path, 'ceph_fsid')
+ if desc['ceph_fsid']:
+ desc['cluster'] = find_cluster_by_uuid(desc['ceph_fsid'])
+ desc['whoami'] = get_oneliner(path, 'whoami')
+ for name in Space.NAMES:
+ uuid = get_oneliner(path, name + '_uuid')
+ if uuid:
+ desc[name + '_uuid'] = uuid.lower()
+ if desc[name + '_uuid'] in uuid_map:
+ desc[name + '_dev'] = uuid_map[desc[name + '_uuid']]
+
+
+def list_dev_osd(dev, uuid_map, desc):
+ desc['mount'] = is_mounted(dev)
+ desc['fs_type'] = get_dev_fs(dev)
+ desc['state'] = 'unprepared'
+ if desc['mount']:
+ desc['state'] = 'active'
+ more_osd_info(desc['mount'], uuid_map, desc)
+ elif desc['fs_type']:
+ try:
+ tpath = mount(dev=dev, fstype=desc['fs_type'], options='')
+ if tpath:
+ try:
+ magic = get_oneliner(tpath, 'magic')
+ if magic is not None:
+ desc['magic'] = magic
+ desc['state'] = 'prepared'
+ more_osd_info(tpath, uuid_map, desc)
+ finally:
+ unmount(tpath)
+ except MountError:
+ pass
+
+
+def list_dev_lockbox(dev, uuid_map, desc):
+ desc['mount'] = is_mounted(dev)
+ desc['fs_type'] = get_dev_fs(dev)
+ desc['state'] = 'unprepared'
+ if desc['mount']:
+ desc['state'] = 'active'
+ desc['osd_uuid'] = get_oneliner(desc['mount'], 'osd-uuid')
+ elif desc['fs_type']:
+ try:
+ tpath = tempfile.mkdtemp(prefix='mnt.', dir=STATEDIR + '/tmp')
+ args = ['mount', '-t', 'ext4', dev, tpath]
+ LOG.debug('Mounting lockbox ' + str(" ".join(args)))
+ command_check_call(args)
+ magic = get_oneliner(tpath, 'magic')
+ if magic is not None:
+ desc['magic'] = magic
+ desc['state'] = 'prepared'
+ desc['osd_uuid'] = get_oneliner(tpath, 'osd-uuid')
+ unmount(tpath)
+ except subprocess.CalledProcessError:
+ pass
+ if desc.get('osd_uuid') in uuid_map:
+ desc['lockbox_for'] = uuid_map[desc['osd_uuid']]
+
+
+def list_format_lockbox_plain(dev):
+ desc = []
+ if dev.get('lockbox_for'):
+ desc.append('for ' + dev['lockbox_for'])
+ elif dev.get('osd_uuid'):
+ desc.append('for osd ' + dev['osd_uuid'])
+ return desc
+
+
+def list_format_more_osd_info_plain(dev):
+ desc = []
+ if dev.get('ceph_fsid'):
+ if dev.get('cluster'):
+ desc.append('cluster ' + dev['cluster'])
+ else:
+ desc.append('unknown cluster ' + dev['ceph_fsid'])
+ if dev.get('whoami'):
+ desc.append('osd.%s' % dev['whoami'])
+ for name in Space.NAMES:
+ if dev.get(name + '_dev'):
+ desc.append(name + ' %s' % dev[name + '_dev'])
+ return desc
+
+
+def list_format_dev_plain(dev, prefix=''):
+ desc = []
+ if dev['ptype'] == PTYPE['regular']['osd']['ready']:
+ desc = (['ceph data', dev['state']] +
+ list_format_more_osd_info_plain(dev))
+ elif dev['ptype'] in (PTYPE['regular']['lockbox']['ready'],
+ PTYPE['mpath']['lockbox']['ready']):
+ desc = (['ceph lockbox', dev['state']] +
+ list_format_lockbox_plain(dev))
+ elif Ptype.is_dmcrypt(dev['ptype'], 'osd'):
+ dmcrypt = dev['dmcrypt']
+ if not dmcrypt['holders']:
+ desc = ['ceph data (dmcrypt %s)' % dmcrypt['type'],
+ 'not currently mapped']
+ elif len(dmcrypt['holders']) == 1:
+ holder = get_dev_path(dmcrypt['holders'][0])
+ desc = ['ceph data (dmcrypt %s %s)' %
+ (dmcrypt['type'], holder)]
+ desc += list_format_more_osd_info_plain(dev)
+ else:
+ desc = ['ceph data (dmcrypt %s)' % dmcrypt['type'],
+ 'holders: ' + ','.join(dmcrypt['holders'])]
+ elif Ptype.is_regular_space(dev['ptype']):
+ name = Ptype.space_ptype_to_name(dev['ptype'])
+ desc.append('ceph ' + name)
+ if dev.get(name + '_for'):
+ desc.append('for %s' % dev[name + '_for'])
+ elif Ptype.is_dmcrypt_space(dev['ptype']):
+ name = Ptype.space_ptype_to_name(dev['ptype'])
+ dmcrypt = dev['dmcrypt']
+ if dmcrypt['holders'] and len(dmcrypt['holders']) == 1:
+ holder = get_dev_path(dmcrypt['holders'][0])
+ desc = ['ceph ' + name + ' (dmcrypt %s %s)' %
+ (dmcrypt['type'], holder)]
+ else:
+ desc = ['ceph ' + name + ' (dmcrypt %s)' % dmcrypt['type']]
+ if dev.get(name + '_for'):
+ desc.append('for %s' % dev[name + '_for'])
+ else:
+ desc.append(dev['type'])
+ if dev.get('fs_type'):
+ desc.append(dev['fs_type'])
+ elif dev.get('ptype'):
+ desc.append(dev['ptype'])
+ if dev.get('mount'):
+ desc.append('mounted on %s' % dev['mount'])
+ return '%s%s %s' % (prefix, dev['path'], ', '.join(desc))
+
+
+def list_format_plain(devices):
+ lines = []
+ for device in devices:
+ if device.get('partitions'):
+ lines.append('%s :' % device['path'])
+ for p in sorted(device['partitions'], key=lambda x: x['path']):
+ lines.append(list_format_dev_plain(dev=p,
+ prefix=' '))
+ else:
+ lines.append(list_format_dev_plain(dev=device,
+ prefix=''))
+ return "\n".join(lines)
+
+
+def list_dev(dev, uuid_map, space_map):
+ info = {
+ 'path': dev,
+ 'dmcrypt': {},
+ }
+
+ info['is_partition'] = is_partition(dev)
+ if info['is_partition']:
+ ptype = get_partition_type(dev)
+ info['uuid'] = get_partition_uuid(dev)
+ else:
+ ptype = 'unknown'
+ info['ptype'] = ptype
+ LOG.info("list_dev(dev = " + dev + ", ptype = " + str(ptype) + ")")
+ if ptype in (PTYPE['regular']['osd']['ready'],
+ PTYPE['mpath']['osd']['ready']):
+ info['type'] = 'data'
+ if ptype == PTYPE['mpath']['osd']['ready']:
+ info['multipath'] = True
+ list_dev_osd(dev, uuid_map, info)
+ elif ptype in (PTYPE['regular']['lockbox']['ready'],
+ PTYPE['mpath']['lockbox']['ready']):
+ info['type'] = 'lockbox'
+ if ptype == PTYPE['mpath']['osd']['ready']:
+ info['multipath'] = True
+ list_dev_lockbox(dev, uuid_map, info)
+ elif ptype == PTYPE['plain']['osd']['ready']:
+ holders = is_held(dev)
+ info['type'] = 'data'
+ info['dmcrypt']['holders'] = holders
+ info['dmcrypt']['type'] = 'plain'
+ if len(holders) == 1:
+ list_dev_osd(get_dev_path(holders[0]), uuid_map, info)
+ elif ptype == PTYPE['luks']['osd']['ready']:
+ holders = is_held(dev)
+ info['type'] = 'data'
+ info['dmcrypt']['holders'] = holders
+ info['dmcrypt']['type'] = 'LUKS'
+ if len(holders) == 1:
+ list_dev_osd(get_dev_path(holders[0]), uuid_map, info)
+ elif Ptype.is_regular_space(ptype) or Ptype.is_mpath_space(ptype):
+ name = Ptype.space_ptype_to_name(ptype)
+ info['type'] = name
+ if ptype == PTYPE['mpath'][name]['ready']:
+ info['multipath'] = True
+ if info.get('uuid') in space_map:
+ info[name + '_for'] = space_map[info['uuid']]
+ elif Ptype.is_plain_space(ptype):
+ name = Ptype.space_ptype_to_name(ptype)
+ holders = is_held(dev)
+ info['type'] = name
+ info['dmcrypt']['type'] = 'plain'
+ info['dmcrypt']['holders'] = holders
+ if info.get('uuid') in space_map:
+ info[name + '_for'] = space_map[info['uuid']]
+ elif Ptype.is_luks_space(ptype):
+ name = Ptype.space_ptype_to_name(ptype)
+ holders = is_held(dev)
+ info['type'] = name
+ info['dmcrypt']['type'] = 'LUKS'
+ info['dmcrypt']['holders'] = holders
+ if info.get('uuid') in space_map:
+ info[name + '_for'] = space_map[info['uuid']]
+ else:
+ path = is_mounted(dev)
+ fs_type = get_dev_fs(dev)
+ if is_swap(dev):
+ info['type'] = 'swap'
+ else:
+ info['type'] = 'other'
+ if fs_type:
+ info['fs_type'] = fs_type
+ if path:
+ info['mount'] = path
+
+ return info
+
+
+def list_devices():
+ partmap = list_all_partitions()
+
+ uuid_map = {}
+ space_map = {}
+ for base, parts in sorted(partmap.items()):
+ for p in parts:
+ dev = get_dev_path(p)
+ part_uuid = get_partition_uuid(dev)
+ if part_uuid:
+ uuid_map[part_uuid] = dev
+ ptype = get_partition_type(dev)
+ LOG.debug("main_list: " + dev +
+ " ptype = " + str(ptype) +
+ " uuid = " + str(part_uuid))
+ if ptype in Ptype.get_ready_by_name('osd'):
+ if Ptype.is_dmcrypt(ptype, 'osd'):
+ holders = is_held(dev)
+ if len(holders) != 1:
+ continue
+ dev_to_mount = get_dev_path(holders[0])
+ else:
+ dev_to_mount = dev
+
+ fs_type = get_dev_fs(dev_to_mount)
+ if fs_type is not None:
+ mount_options = get_mount_options(cluster='ceph',
+ fs_type=fs_type)
+ try:
+ tpath = mount(dev=dev_to_mount,
+ fstype=fs_type, options=mount_options)
+ try:
+ for name in Space.NAMES:
+ space_uuid = get_oneliner(tpath,
+ name + '_uuid')
+ if space_uuid:
+ space_map[space_uuid.lower()] = dev
+ finally:
+ unmount(tpath)
+ except MountError:
+ pass
+
+ LOG.debug("main_list: " + str(partmap) + ", uuid_map = " +
+ str(uuid_map) + ", space_map = " + str(space_map))
+
+ devices = []
+ for base, parts in sorted(partmap.items()):
+ if parts:
+ disk = {'path': get_dev_path(base)}
+ partitions = []
+ for p in sorted(parts):
+ partitions.append(list_dev(get_dev_path(p),
+ uuid_map,
+ space_map))
+ disk['partitions'] = partitions
+ devices.append(disk)
+ else:
+ device = list_dev(get_dev_path(base), uuid_map, space_map)
+ device['path'] = get_dev_path(base)
+ devices.append(device)
+ LOG.debug("list_devices: " + str(devices))
+ return devices
+
+
+def list_zfs():
+ try:
+ out, err, ret = command(
+ [
+ 'zfs',
+ 'list',
+ '-o', 'name,mountpoint'
+ ]
+ )
+ except subprocess.CalledProcessError as e:
+ LOG.info('zfs list -o name,mountpoint '
+ 'fails.\n (Error: %s)' % e)
+ raise
+ lines = out.splitlines()
+ for line in lines[1:]:
+ vdevline = line.split()
+ if os.path.exists(os.path.join(vdevline[1], 'active')):
+ elems = os.path.split(vdevline[1])
+ print(vdevline[0], "ceph data, active, cluster ceph,", elems[1],
+ "mounted on:", vdevline[1])
+ else:
+ print(vdevline[0] + " other, zfs, mounted on: " + vdevline[1])
+
+
+def main_list(args):
+ with activate_lock:
+ if FREEBSD:
+ main_list_freebsd(args)
+ else:
+ main_list_protected(args)
+
+
+def main_list_protected(args):
+ devices = list_devices()
+ if args.path:
+ paths = []
+ for path in args.path:
+ if os.path.exists(path):
+ paths.append(os.path.realpath(path))
+ else:
+ paths.append(path)
+ selected_devices = []
+ for device in devices:
+ for path in paths:
+ if re.search(path + '$', device['path']):
+ selected_devices.append(device)
+ else:
+ selected_devices = devices
+ if args.format == 'json':
+ print(json.dumps(selected_devices))
+ else:
+ output = list_format_plain(selected_devices)
+ if output:
+ print(output)
+
+
+def main_list_freebsd(args):
+ # Currently accomodate only ZFS Filestore partitions
+ # return a list of VDEVs and mountpoints
+ # > zfs list
+ # NAME USED AVAIL REFER MOUNTPOINT
+ # osd0 1.01G 1.32T 1.01G /var/lib/ceph/osd/osd.0
+ # osd1 1.01G 1.32T 1.01G /var/lib/ceph/osd/osd.1
+ list_zfs()
+
+
+###########################
+#
+# Mark devices that we want to suppress activates on with a
+# file like
+#
+# /var/lib/ceph/tmp/suppress-activate.sdb
+#
+# where the last bit is the sanitized device name (/dev/X without the
+# /dev/ prefix) and the is_suppress() check matches a prefix. That
+# means suppressing sdb will stop activate on sdb1, sdb2, etc.
+#
+
+def is_suppressed(path):
+ disk = os.path.realpath(path)
+ try:
+ if (not disk.startswith('/dev/') or
+ not ldev_is_diskdevice(disk)):
+ return False
+ base = get_dev_name(disk)
+ while len(base):
+ if os.path.exists(SUPPRESS_PREFIX + base): # noqa
+ return True
+ base = base[:-1]
+ except:
+ return False
+
+
+def set_suppress(path):
+ disk = os.path.realpath(path)
+ if not os.path.exists(disk):
+ raise Error('does not exist', path)
+ if not ldev_is_diskdevice(path):
+ raise Error('not a block device', path)
+ base = get_dev_name(disk)
+
+ with open(SUPPRESS_PREFIX + base, 'w') as f: # noqa
+ pass
+ LOG.info('set suppress flag on %s', base)
+
+
+def unset_suppress(path):
+ disk = os.path.realpath(path)
+ if not os.path.exists(disk):
+ raise Error('does not exist', path)
+ if not ldev_is_diskdevice(path):
+ raise Error('not a block device', path)
+ assert disk.startswith('/dev/')
+ base = get_dev_name(disk)
+
+ fn = SUPPRESS_PREFIX + base # noqa
+ if not os.path.exists(fn):
+ raise Error('not marked as suppressed', path)
+
+ try:
+ os.unlink(fn)
+ LOG.info('unset suppress flag on %s', base)
+ except OSError as e:
+ raise Error('failed to unsuppress', e)
+
+
+def main_suppress(args):
+ set_suppress(args.path)
+
+
+def main_unsuppress(args):
+ unset_suppress(args.path)
+
+
+def main_zap(args):
+ for dev in args.dev:
+ zap(dev)
+
+
+def main_trigger(args):
+ LOG.debug("main_trigger: " + str(args))
+ if is_systemd() and not args.sync:
+ # http://www.freedesktop.org/software/systemd/man/systemd-escape.html
+ escaped_dev = args.dev[1:].replace('-', '\\x2d')
+ service = 'ceph-disk@{dev}.service'.format(dev=escaped_dev)
+ LOG.info('systemd detected, triggering %s' % service)
+ command(
+ [
+ 'systemctl',
+ '--no-block',
+ 'restart',
+ service,
+ ]
+ )
+ return
+ if is_upstart() and not args.sync:
+ LOG.info('upstart detected, triggering ceph-disk task')
+ command(
+ [
+ 'initctl',
+ 'emit',
+ 'ceph-disk',
+ 'dev={dev}'.format(dev=args.dev),
+ 'pid={pid}'.format(pid=os.getpid()),
+ ]
+ )
+ return
+
+ if get_ceph_user() == 'ceph':
+ command_check_call(['chown', 'ceph:ceph', args.dev])
+ parttype = get_partition_type(args.dev)
+ partid = get_partition_uuid(args.dev)
+
+ LOG.info('trigger {dev} parttype {parttype} uuid {partid}'.format(
+ dev=args.dev,
+ parttype=parttype,
+ partid=partid,
+ ))
+
+ ceph_disk = ['ceph-disk']
+ if args.verbose:
+ ceph_disk.append('--verbose')
+
+ if parttype in (PTYPE['regular']['osd']['ready'],
+ PTYPE['mpath']['osd']['ready']):
+ out, err, ret = command(
+ ceph_disk +
+ [
+ 'activate',
+ args.dev,
+ ]
+ )
+
+ elif parttype in (PTYPE['plain']['osd']['ready'],
+ PTYPE['luks']['osd']['ready']):
+ out, err, ret = command(
+ ceph_disk +
+ [
+ 'activate',
+ '--dmcrypt',
+ args.dev,
+ ]
+ )
+
+ elif parttype in (PTYPE['regular']['journal']['ready'],
+ PTYPE['mpath']['journal']['ready']):
+ out, err, ret = command(
+ ceph_disk +
+ [
+ 'activate-journal',
+ args.dev,
+ ]
+ )
+
+ elif parttype in (PTYPE['plain']['journal']['ready'],
+ PTYPE['luks']['journal']['ready']):
+ out, err, ret = command(
+ ceph_disk +
+ [
+ 'activate-journal',
+ '--dmcrypt',
+ args.dev,
+ ]
+ )
+
+ elif parttype in (PTYPE['regular']['block']['ready'],
+ PTYPE['regular']['block.db']['ready'],
+ PTYPE['regular']['block.wal']['ready'],
+ PTYPE['mpath']['block']['ready'],
+ PTYPE['mpath']['block.db']['ready'],
+ PTYPE['mpath']['block.wal']['ready']):
+ out, err, ret = command(
+ ceph_disk +
+ [
+ 'activate-block',
+ args.dev,
+ ]
+ )
+
+ elif parttype in (PTYPE['plain']['block']['ready'],
+ PTYPE['plain']['block.db']['ready'],
+ PTYPE['plain']['block.wal']['ready'],
+ PTYPE['luks']['block']['ready'],
+ PTYPE['luks']['block.db']['ready'],
+ PTYPE['luks']['block.wal']['ready']):
+ out, err, ret = command(
+ ceph_disk +
+ [
+ 'activate-block',
+ '--dmcrypt',
+ args.dev,
+ ]
+ )
+
+ elif parttype in (PTYPE['regular']['lockbox']['ready'],
+ PTYPE['mpath']['lockbox']['ready']):
+ out, err, ret = command(
+ ceph_disk +
+ [
+ 'activate-lockbox',
+ args.dev,
+ ]
+ )
+
+ else:
+ raise Error('unrecognized partition type %s' % parttype)
+
+ if ret != 0:
+ LOG.info(out)
+ LOG.error(err)
+ raise Error('return code ' + str(ret))
+ else:
+ LOG.debug(out)
+ LOG.debug(err)
+
+
+def main_fix(args):
+ # A hash table containing 'path': ('uid', 'gid', blocking, recursive)
+ fix_table = [
+ ('/usr/bin/ceph-mon', 'root', ROOTGROUP, True, False),
+ ('/usr/bin/ceph-mds', 'root', ROOTGROUP, True, False),
+ ('/usr/bin/ceph-osd', 'root', ROOTGROUP, True, False),
+ ('/usr/bin/radosgw', 'root', ROOTGROUP, True, False),
+ ('/etc/ceph', 'root', ROOTGROUP, True, True),
+ ('/var/run/ceph', 'ceph', 'ceph', True, True),
+ ('/var/log/ceph', 'ceph', 'ceph', True, True),
+ ('/var/log/radosgw', 'ceph', 'ceph', True, True),
+ ('/var/lib/ceph', 'ceph', 'ceph', True, False),
+ ]
+
+ # Relabel/chown all files under /var/lib/ceph/ recursively (except for osd)
+ for directory in glob.glob('/var/lib/ceph/*'):
+ if directory == '/var/lib/ceph/osd':
+ fix_table.append((directory, 'ceph', 'ceph', True, False))
+ else:
+ fix_table.append((directory, 'ceph', 'ceph', True, True))
+
+ # Relabel/chown the osds recursively and in parallel
+ for directory in glob.glob('/var/lib/ceph/osd/*'):
+ fix_table.append((directory, 'ceph', 'ceph', False, True))
+
+ LOG.debug("fix_table: " + str(fix_table))
+
+ # The lists of background processes
+ all_processes = []
+ permissions_processes = []
+ selinux_processes = []
+
+ # Preliminary checks
+ if args.selinux or args.all:
+ out, err, ret = command(['selinuxenabled'])
+ if ret:
+ LOG.error('SELinux is not enabled, please enable it, first.')
+ raise Error('no SELinux')
+
+ for daemon in ['ceph-mon', 'ceph-osd', 'ceph-mds', 'radosgw', 'ceph-mgr']:
+ out, err, ret = command(['pgrep', daemon])
+ if ret == 0:
+ LOG.error(daemon + ' is running, please stop it, first')
+ raise Error(daemon + ' running')
+
+ # Relabel the basic system data without the ceph files
+ if args.system or args.all:
+ c = ['restorecon', '-R', '/']
+ for directory, _, _, _, _ in fix_table:
+ # Skip /var/lib/ceph subdirectories
+ if directory.startswith('/var/lib/ceph/'):
+ continue
+ c.append('-e')
+ c.append(directory)
+
+ out, err, ret = command(c)
+
+ if ret:
+ LOG.error("Failed to restore labels of the underlying system")
+ LOG.error(err)
+ raise Error("basic restore failed")
+
+ # Use find to relabel + chown ~simultaenously
+ if args.all:
+ for directory, uid, gid, blocking, recursive in fix_table:
+ # Skip directories/files that are not installed
+ if not os.access(directory, os.F_OK):
+ continue
+
+ c = [
+ 'find',
+ directory,
+ '-exec',
+ 'chown',
+ ':'.join((uid, gid)),
+ '{}',
+ '+',
+ '-exec',
+ 'restorecon',
+ '{}',
+ '+',
+ ]
+
+ # Just pass -maxdepth 0 for non-recursive calls
+ if not recursive:
+ c += ['-maxdepth', '0']
+
+ if blocking:
+ out, err, ret = command(c)
+
+ if ret:
+ LOG.error("Failed to fix " + directory)
+ LOG.error(err)
+ raise Error(directory + " fix failed")
+ else:
+ all_processes.append(command_init(c))
+
+ LOG.debug("all_processes: " + str(all_processes))
+ for process in all_processes:
+ out, err, ret = command_wait(process)
+ if ret:
+ LOG.error("A background find process failed")
+ LOG.error(err)
+ raise Error("background failed")
+
+ # Fix permissions
+ if args.permissions:
+ for directory, uid, gid, blocking, recursive in fix_table:
+ # Skip directories/files that are not installed
+ if not os.access(directory, os.F_OK):
+ continue
+
+ if recursive:
+ c = [
+ 'chown',
+ '-R',
+ ':'.join((uid, gid)),
+ directory
+ ]
+ else:
+ c = [
+ 'chown',
+ ':'.join((uid, gid)),
+ directory
+ ]
+
+ if blocking:
+ out, err, ret = command(c)
+
+ if ret:
+ LOG.error("Failed to chown " + directory)
+ LOG.error(err)
+ raise Error(directory + " chown failed")
+ else:
+ permissions_processes.append(command_init(c))
+
+ LOG.debug("permissions_processes: " + str(permissions_processes))
+ for process in permissions_processes:
+ out, err, ret = command_wait(process)
+ if ret:
+ LOG.error("A background permissions process failed")
+ LOG.error(err)
+ raise Error("background failed")
+
+ # Fix SELinux labels
+ if args.selinux:
+ for directory, uid, gid, blocking, recursive in fix_table:
+ # Skip directories/files that are not installed
+ if not os.access(directory, os.F_OK):
+ continue
+
+ if recursive:
+ c = [
+ 'restorecon',
+ '-R',
+ directory
+ ]
+ else:
+ c = [
+ 'restorecon',
+ directory
+ ]
+
+ if blocking:
+ out, err, ret = command(c)
+
+ if ret:
+ LOG.error("Failed to restore labels for " + directory)
+ LOG.error(err)
+ raise Error(directory + " relabel failed")
+ else:
+ selinux_processes.append(command_init(c))
+
+ LOG.debug("selinux_processes: " + str(selinux_processes))
+ for process in selinux_processes:
+ out, err, ret = command_wait(process)
+ if ret:
+ LOG.error("A background selinux process failed")
+ LOG.error(err)
+ raise Error("background failed")
+
+ LOG.info(
+ "The ceph files has been fixed, please reboot "
+ "the system for the changes to take effect."
+ )
+
+
+def setup_statedir(dir):
+ # XXX The following use of globals makes linting
+ # really hard. Global state in Python is iffy and
+ # should be avoided.
+ global STATEDIR
+ STATEDIR = dir
+
+ if not os.path.exists(STATEDIR):
+ os.mkdir(STATEDIR)
+ if not os.path.exists(STATEDIR + "/tmp"):
+ os.mkdir(STATEDIR + "/tmp")
+
+ global prepare_lock
+ prepare_lock = FileLock(STATEDIR + '/tmp/ceph-disk.prepare.lock')
+
+ global activate_lock
+ activate_lock = FileLock(STATEDIR + '/tmp/ceph-disk.activate.lock')
+
+ global SUPPRESS_PREFIX
+ SUPPRESS_PREFIX = STATEDIR + '/tmp/suppress-activate.'
+
+
+def setup_sysconfdir(dir):
+ global SYSCONFDIR
+ SYSCONFDIR = dir
+
+
+def parse_args(argv):
+ parser = argparse.ArgumentParser(
+ 'ceph-disk',
+ )
+ parser.add_argument(
+ '-v', '--verbose',
+ action='store_true', default=None,
+ help='be more verbose',
+ )
+ parser.add_argument(
+ '--log-stdout',
+ action='store_true', default=None,
+ help='log to stdout',
+ )
+ parser.add_argument(
+ '--prepend-to-path',
+ metavar='PATH',
+ default='/usr/bin',
+ help=('prepend PATH to $PATH for backward compatibility '
+ '(default /usr/bin)'),
+ )
+ parser.add_argument(
+ '--statedir',
+ metavar='PATH',
+ default='/var/lib/ceph',
+ help=('directory in which ceph state is preserved '
+ '(default /var/lib/ceph)'),
+ )
+ parser.add_argument(
+ '--sysconfdir',
+ metavar='PATH',
+ default='/etc/ceph',
+ help=('directory in which ceph configuration files are found '
+ '(default /etc/ceph)'),
+ )
+ parser.add_argument(
+ '--setuser',
+ metavar='USER',
+ default=None,
+ help='use the given user for subprocesses, rather than ceph or root'
+ )
+ parser.add_argument(
+ '--setgroup',
+ metavar='GROUP',
+ default=None,
+ help='use the given group for subprocesses, rather than ceph or root'
+ )
+ parser.set_defaults(
+ # we want to hold on to this, for later
+ prog=parser.prog,
+ )
+
+ subparsers = parser.add_subparsers(
+ title='subcommands',
+ description='valid subcommands',
+ help='sub-command help',
+ )
+
+ Prepare.set_subparser(subparsers)
+ make_activate_parser(subparsers)
+ make_activate_lockbox_parser(subparsers)
+ make_activate_block_parser(subparsers)
+ make_activate_journal_parser(subparsers)
+ make_activate_all_parser(subparsers)
+ make_list_parser(subparsers)
+ make_suppress_parser(subparsers)
+ make_deactivate_parser(subparsers)
+ make_destroy_parser(subparsers)
+ make_zap_parser(subparsers)
+ make_trigger_parser(subparsers)
+ make_fix_parser(subparsers)
+
+ args = parser.parse_args(argv)
+ return args
+
+
+def make_fix_parser(subparsers):
+ fix_parser = subparsers.add_parser(
+ 'fix',
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\
+ """)),
+ help='fix SELinux labels and/or file permissions')
+
+ fix_parser.add_argument(
+ '--system',
+ action='store_true',
+ default=False,
+ help='fix SELinux labels for the non-ceph system data'
+ )
+ fix_parser.add_argument(
+ '--selinux',
+ action='store_true',
+ default=False,
+ help='fix SELinux labels for ceph data'
+ )
+ fix_parser.add_argument(
+ '--permissions',
+ action='store_true',
+ default=False,
+ help='fix file permissions for ceph data'
+ )
+ fix_parser.add_argument(
+ '--all',
+ action='store_true',
+ default=False,
+ help='perform all the fix-related operations'
+ )
+ fix_parser.set_defaults(
+ func=main_fix,
+ )
+ return fix_parser
+
+
+def make_trigger_parser(subparsers):
+ trigger_parser = subparsers.add_parser(
+ 'trigger',
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\
+ The partition given in argument is activated. The type of the
+ partition (data, lockbox, journal etc.) is detected by its
+ type. If the init system is upstart or systemd, the activation is
+ delegated to it and runs asynchronously, which
+ helps reduce the execution time of udev actions.
+ """)),
+ help='activate any device (called by udev)')
+ trigger_parser.add_argument(
+ 'dev',
+ help=('device'),
+ )
+ trigger_parser.add_argument(
+ '--cluster',
+ metavar='NAME',
+ default='ceph',
+ help='cluster name to assign this disk to',
+ )
+ trigger_parser.add_argument(
+ '--dmcrypt',
+ action='store_true', default=None,
+ help='map devices with dm-crypt',
+ )
+ trigger_parser.add_argument(
+ '--dmcrypt-key-dir',
+ metavar='KEYDIR',
+ default='/etc/ceph/dmcrypt-keys',
+ help='directory where dm-crypt keys are stored',
+ )
+ trigger_parser.add_argument(
+ '--sync',
+ action='store_true', default=None,
+ help='do operation synchronously; do not trigger systemd',
+ )
+ trigger_parser.set_defaults(
+ func=main_trigger,
+ )
+ return trigger_parser
+
+
+def make_activate_parser(subparsers):
+ activate_parser = subparsers.add_parser(
+ 'activate',
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\
+ Activate the OSD found at PATH (can be a directory
+ or a device partition, possibly encrypted). When
+ activated for the first time, a unique OSD id is obtained
+ from the cluster. If PATH is a directory, a symbolic
+ link is added in {statedir}/osd/ceph-$id. If PATH is
+ a partition, it is mounted on {statedir}/osd/ceph-$id.
+ Finally, the OSD daemon is run.
+
+ If the OSD depends on auxiliary partitions (journal, block, ...)
+ they need to be available otherwise activation will fail. It
+ may happen if a journal is encrypted and cryptsetup was not
+ run yet.
+ """.format(statedir=STATEDIR))),
+ help='Activate a Ceph OSD')
+ activate_parser.add_argument(
+ '--mount',
+ action='store_true', default=None,
+ help='mount a block device [deprecated, ignored]',
+ )
+ activate_parser.add_argument(
+ '--activate-key',
+ metavar='PATH',
+ help='bootstrap-osd keyring path template (%(default)s)',
+ dest='activate_key_template',
+ )
+ activate_parser.add_argument(
+ '--mark-init',
+ metavar='INITSYSTEM',
+ help='init system to manage this dir',
+ default='auto',
+ choices=INIT_SYSTEMS,
+ )
+ activate_parser.add_argument(
+ '--no-start-daemon',
+ action='store_true', default=None,
+ help='do not start the daemon',
+ )
+ activate_parser.add_argument(
+ 'path',
+ metavar='PATH',
+ help='path to block device or directory',
+ )
+ activate_parser.add_argument(
+ '--dmcrypt',
+ action='store_true', default=None,
+ help='map DATA and/or JOURNAL devices with dm-crypt',
+ )
+ activate_parser.add_argument(
+ '--dmcrypt-key-dir',
+ metavar='KEYDIR',
+ default='/etc/ceph/dmcrypt-keys',
+ help='directory where dm-crypt keys are stored',
+ )
+ activate_parser.add_argument(
+ '--reactivate',
+ action='store_true', default=False,
+ help='activate the deactived OSD',
+ )
+ activate_parser.set_defaults(
+ activate_key_template='{statedir}/bootstrap-osd/{cluster}.keyring',
+ func=main_activate,
+ )
+ return activate_parser
+
+
+def make_activate_lockbox_parser(subparsers):
+ parser = subparsers.add_parser(
+ 'activate-lockbox',
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\
+ Mount the partition found at PATH on {statedir}/osd-lockbox/$uuid
+ where $uuid uniquely identifies the OSD that needs this lockbox
+ to retrieve keys from the monitor and unlock its partitions.
+
+ If the OSD has one or more auxiliary devices (journal, block, ...)
+ symbolic links are created at {statedir}/osd-lockbox/$other_uuid
+ and point to {statedir}/osd-lockbox/$uuid. This will, for instance,
+ allow a journal encrypted in a partition identified by $other_uuid to
+ fetch the keys it needs from the monitor.
+
+ Finally the OSD is activated, as it would be with ceph-disk activate.
+ """.format(statedir=STATEDIR))),
+ help='Activate a Ceph lockbox')
+ parser.add_argument(
+ '--activate-key',
+ help='bootstrap-osd keyring path template (%(default)s)',
+ dest='activate_key_template',
+ )
+ parser.add_argument(
+ '--dmcrypt-key-dir',
+ metavar='KEYDIR',
+ default='/etc/ceph/dmcrypt-keys',
+ help='directory where dm-crypt keys are stored',
+ )
+ parser.add_argument(
+ 'path',
+ metavar='PATH',
+ help='path to block device',
+ )
+ parser.set_defaults(
+ activate_key_template='{statedir}/bootstrap-osd/{cluster}.keyring',
+ func=main_activate_lockbox,
+ )
+ return parser
+
+
+def make_activate_block_parser(subparsers):
+ return make_activate_space_parser('block', subparsers)
+
+
+def make_activate_journal_parser(subparsers):
+ return make_activate_space_parser('journal', subparsers)
+
+
+def make_activate_space_parser(name, subparsers):
+ activate_space_parser = subparsers.add_parser(
+ 'activate-%s' % name,
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\
+ Activating a {name} partition is only meaningfull
+ if it is encrypted and it will map it using
+ cryptsetup.
+
+ Finally the corresponding OSD is activated,
+ as it would be with ceph-disk activate.
+ """.format(name=name))),
+ help='Activate an OSD via its %s device' % name)
+ activate_space_parser.add_argument(
+ 'dev',
+ metavar='DEV',
+ help='path to %s block device' % name,
+ )
+ activate_space_parser.add_argument(
+ '--activate-key',
+ metavar='PATH',
+ help='bootstrap-osd keyring path template (%(default)s)',
+ dest='activate_key_template',
+ )
+ activate_space_parser.add_argument(
+ '--mark-init',
+ metavar='INITSYSTEM',
+ help='init system to manage this dir',
+ default='auto',
+ choices=INIT_SYSTEMS,
+ )
+ activate_space_parser.add_argument(
+ '--dmcrypt',
+ action='store_true', default=None,
+ help=('map data and/or auxiliariy (journal, etc.) '
+ 'devices with dm-crypt'),
+ )
+ activate_space_parser.add_argument(
+ '--dmcrypt-key-dir',
+ metavar='KEYDIR',
+ default='/etc/ceph/dmcrypt-keys',
+ help='directory where dm-crypt keys are stored',
+ )
+ activate_space_parser.add_argument(
+ '--reactivate',
+ action='store_true', default=False,
+ help='activate the deactived OSD',
+ )
+ activate_space_parser.set_defaults(
+ activate_key_template='{statedir}/bootstrap-osd/{cluster}.keyring',
+ func=lambda args: main_activate_space(name, args),
+ )
+ return activate_space_parser
+
+
+def make_activate_all_parser(subparsers):
+ activate_all_parser = subparsers.add_parser(
+ 'activate-all',
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\
+ Activate all OSD partitions found in /dev/disk/by-parttypeuuid.
+ The partitions containing auxiliary devices (journal, block, ...)
+ are not activated.
+ """)),
+ help='Activate all tagged OSD partitions')
+ activate_all_parser.add_argument(
+ '--activate-key',
+ metavar='PATH',
+ help='bootstrap-osd keyring path template (%(default)s)',
+ dest='activate_key_template',
+ )
+ activate_all_parser.add_argument(
+ '--mark-init',
+ metavar='INITSYSTEM',
+ help='init system to manage this dir',
+ default='auto',
+ choices=INIT_SYSTEMS,
+ )
+ activate_all_parser.set_defaults(
+ activate_key_template='{statedir}/bootstrap-osd/{cluster}.keyring',
+ func=main_activate_all,
+ )
+ return activate_all_parser
+
+
+def make_list_parser(subparsers):
+ list_parser = subparsers.add_parser(
+ 'list',
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\
+ Display all partitions on the system and their
+ associated Ceph information, if any.
+ """)),
+ help='List disks, partitions, and Ceph OSDs')
+ list_parser.add_argument(
+ '--format',
+ help='output format',
+ default='plain',
+ choices=['json', 'plain'],
+ )
+ list_parser.add_argument(
+ 'path',
+ metavar='PATH',
+ nargs='*',
+ help='path to block devices, relative to /sys/block',
+ )
+ list_parser.set_defaults(
+ func=main_list,
+ )
+ return list_parser
+
+
+def make_suppress_parser(subparsers):
+ suppress_parser = subparsers.add_parser(
+ 'suppress-activate',
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\
+ Add a prefix to the list of suppressed device names
+ so that they are ignored by all activate* subcommands.
+ """)),
+ help='Suppress activate on a device (prefix)')
+ suppress_parser.add_argument(
+ 'path',
+ metavar='PATH',
+ help='path to block device or directory',
+ )
+ suppress_parser.set_defaults(
+ func=main_suppress,
+ )
+
+ unsuppress_parser = subparsers.add_parser(
+ 'unsuppress-activate',
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\
+ Remove a prefix from the list of suppressed device names
+ so that they are no longer ignored by all
+ activate* subcommands.
+ """)),
+ help='Stop suppressing activate on a device (prefix)')
+ unsuppress_parser.add_argument(
+ 'path',
+ metavar='PATH',
+ help='path to block device or directory',
+ )
+ unsuppress_parser.set_defaults(
+ func=main_unsuppress,
+ )
+ return suppress_parser
+
+
+def make_deactivate_parser(subparsers):
+ deactivate_parser = subparsers.add_parser(
+ 'deactivate',
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\
+ Deactivate the OSD located at PATH. It stops the OSD daemon
+ and optionally marks it out (with --mark-out). The content of
+ the OSD is left untouched.
+
+ By default, the, ready, active, INIT-specific files are
+ removed (so that it is not automatically re-activated by the
+ udev rules or ceph-disk trigger) and the file deactive is
+ created to remember the OSD is deactivated.
+
+ If the --once option is given, the ready, active, INIT-specific
+ files are not removed and the OSD will reactivate whenever
+ ceph-disk trigger is run on one of the devices (journal, data,
+ block, lockbox, ...).
+
+ If the OSD is dmcrypt, remove the data dmcrypt map. When
+ deactivate finishes, the OSD is down.
+ """)),
+ help='Deactivate a Ceph OSD')
+ deactivate_parser.add_argument(
+ '--cluster',
+ metavar='NAME',
+ default='ceph',
+ help='cluster name to assign this disk to',
+ )
+ deactivate_parser.add_argument(
+ 'path',
+ metavar='PATH',
+ nargs='?',
+ help='path to block device or directory',
+ )
+ deactivate_parser.add_argument(
+ '--deactivate-by-id',
+ metavar='<id>',
+ help='ID of OSD to deactive'
+ )
+ deactivate_parser.add_argument(
+ '--mark-out',
+ action='store_true', default=False,
+ help='option to mark the osd out',
+ )
+ deactivate_parser.add_argument(
+ '--once',
+ action='store_true', default=False,
+ help='does not need --reactivate to activate again',
+ )
+ deactivate_parser.set_defaults(
+ func=main_deactivate,
+ )
+
+
+def make_destroy_parser(subparsers):
+ destroy_parser = subparsers.add_parser(
+ 'destroy',
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\ Destroy the OSD located at PATH. It removes the OSD from the
+ cluster and marks it destroyed. An OSD must be down before it
+ can be destroyed. Once it is destroyed, a new OSD can be created
+ in its place, reusing the same OSD id and position (e.g. after
+ a failed HDD or SSD is replaced). Alternatively, if the
+ --purge option is also specified, the OSD is removed from the
+ CRUSH map and the OSD id is deallocated.""")),
+ help='Destroy a Ceph OSD')
+ destroy_parser.add_argument(
+ '--cluster',
+ metavar='NAME',
+ default='ceph',
+ help='cluster name to assign this disk to',
+ )
+ destroy_parser.add_argument(
+ 'path',
+ metavar='PATH',
+ nargs='?',
+ help='path to block device or directory',
+ )
+ destroy_parser.add_argument(
+ '--destroy-by-id',
+ metavar='<id>',
+ help='ID of OSD to destroy'
+ )
+ destroy_parser.add_argument(
+ '--dmcrypt-key-dir',
+ metavar='KEYDIR',
+ default='/etc/ceph/dmcrypt-keys',
+ help=('directory where dm-crypt keys are stored '
+ '(If you don\'t know how it work, '
+ 'dont use it. we have default value)'),
+ )
+ destroy_parser.add_argument(
+ '--zap',
+ action='store_true', default=False,
+ help='option to erase data and partition',
+ )
+ destroy_parser.add_argument(
+ '--purge',
+ action='store_true', default=False,
+ help='option to remove OSD from CRUSH map and deallocate the id',
+ )
+ destroy_parser.set_defaults(
+ func=main_destroy,
+ )
+
+
+def make_zap_parser(subparsers):
+ zap_parser = subparsers.add_parser(
+ 'zap',
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.fill(textwrap.dedent("""\
+ Zap/erase/destroy a device's partition table and contents. It
+ actually uses sgdisk and it's option --zap-all to
+ destroy both GPT and MBR data structures so that the disk
+ becomes suitable for repartitioning.
+ """)),
+ help='Zap/erase/destroy a device\'s partition table (and contents)')
+ zap_parser.add_argument(
+ 'dev',
+ metavar='DEV',
+ nargs='+',
+ help='path to block device',
+ )
+ zap_parser.set_defaults(
+ func=main_zap,
+ )
+ return zap_parser
+
+
+def main(argv):
+ args = parse_args(argv)
+
+ setup_logging(args.verbose, args.log_stdout)
+
+ if args.prepend_to_path != '':
+ path = os.environ.get('PATH', os.defpath)
+ os.environ['PATH'] = args.prepend_to_path + ":" + path
+
+ if args.func.__name__ != 'main_trigger':
+ # trigger may run when statedir is unavailable and does not use it
+ setup_statedir(args.statedir)
+ setup_sysconfdir(args.sysconfdir)
+
+ global CEPH_PREF_USER
+ CEPH_PREF_USER = args.setuser
+ global CEPH_PREF_GROUP
+ CEPH_PREF_GROUP = args.setgroup
+
+ if args.verbose:
+ args.func(args)
+ else:
+ main_catch(args.func, args)
+
+
+def setup_logging(verbose, log_stdout):
+ loglevel = logging.WARNING
+ if verbose:
+ loglevel = logging.DEBUG
+
+ if log_stdout:
+ ch = logging.StreamHandler(stream=sys.stdout)
+ ch.setLevel(loglevel)
+ formatter = logging.Formatter('%(funcName)s: %(message)s')
+ ch.setFormatter(formatter)
+ LOG.addHandler(ch)
+ LOG.setLevel(loglevel)
+ else:
+ logging.basicConfig(
+ level=loglevel,
+ format='%(funcName)s: %(message)s',
+ )
+
+
+def main_catch(func, args):
+
+ try:
+ func(args)
+
+ except Error as e:
+ raise SystemExit(
+ '{prog}: {msg}'.format(
+ prog=args.prog,
+ msg=e,
+ )
+ )
+
+ except CephDiskException as error:
+ exc_name = error.__class__.__name__
+ raise SystemExit(
+ '{prog} {exc_name}: {msg}'.format(
+ prog=args.prog,
+ exc_name=exc_name,
+ msg=error,
+ )
+ )
+
+
+def run():
+ main(sys.argv[1:])
+
+
+if __name__ == '__main__':
+ main(sys.argv[1:])
+ warned_about = {}
Code Review / stor4nfv.git / blobdiff