3 # Copyright (C) 2015 Red Hat <contact@redhat.com>
4 # Copyright (C) 2014 Inktank <info@inktank.com>
5 # Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
6 # Copyright (C) 2014 Catalyst.net Ltd
8 # Author: Loic Dachary <loic@dachary.org>
10 # This program is free software; you can redistribute it and/or modify
11 # it under the terms of the GNU Library Public License as published by
12 # the Free Software Foundation; either version 2, or (at your option)
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU Library Public License for more details.
20 # THIS IS ceph-disk AS OF dc5a9053ce69c0630091774f16ce421da67d26fb v10.0.3-2247-gdc5a905
21 # PRIOR TO THE INTRODUCTION OF THE LOCKBOX VOLUME TO STORE KEY FETCHING
42 CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026'
47 # identical because creating a journal is atomic
48 'ready': '45b0969e-9b03-4f30-b4c6-b4b80ceff106',
49 'tobe': '45b0969e-9b03-4f30-b4c6-b4b80ceff106',
52 # identical because creating a block is atomic
53 'ready': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
54 'tobe': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
57 'ready': '4fbd7e29-9d25-41b8-afd0-062c0ceff05d',
58 'tobe': '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be',
63 'ready': '45b0969e-9b03-4f30-b4c6-35865ceff106',
64 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
67 'ready': 'cafecafe-9b03-4f30-b4c6-35865ceff106',
68 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
71 'ready': '4fbd7e29-9d25-41b8-afd0-35865ceff05d',
72 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
77 'ready': '45b0969e-9b03-4f30-b4c6-5ec00ceff106',
78 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
81 'ready': 'cafecafe-9b03-4f30-b4c6-5ec00ceff106',
82 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
85 'ready': '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d',
86 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
91 'ready': '45b0969e-8ae0-4982-bf9d-5a8d867af560',
92 'tobe': '45b0969e-8ae0-4982-bf9d-5a8d867af560',
95 'ready': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
96 'tobe': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
99 'ready': '4fbd7e29-8ae0-4982-bf9d-5a8d867af560',
100 'tobe': '89c57f98-8ae0-4982-bf9d-5a8d867af560',
109 def get_ready_by_type(what):
110 return [x['ready'] for x in PTYPE[what].values()]
113 def get_ready_by_name(name):
114 return [x[name]['ready'] for x in PTYPE.values()]
117 def is_regular_space(ptype):
118 return Ptype.is_what_space('regular', ptype)
121 def is_mpath_space(ptype):
122 return Ptype.is_what_space('mpath', ptype)
125 def is_plain_space(ptype):
126 return Ptype.is_what_space('plain', ptype)
129 def is_luks_space(ptype):
130 return Ptype.is_what_space('luks', ptype)
133 def is_what_space(what, ptype):
134 for name in Space.NAMES:
135 if ptype == PTYPE[what][name]['ready']:
140 def space_ptype_to_name(ptype):
141 for what in PTYPE.values():
142 for name in Space.NAMES:
143 if ptype == what[name]['ready']:
145 raise ValueError('ptype ' + ptype + ' not found')
148 def is_dmcrypt_space(ptype):
149 for name in Space.NAMES:
150 if Ptype.is_dmcrypt(ptype, name):
155 def is_dmcrypt(ptype, name):
156 for what in ('plain', 'luks'):
157 if ptype == PTYPE[what][name]['ready']:
161 DEFAULT_FS_TYPE = 'xfs'
165 OSD STATUS Definition
167 OSD_STATUS_OUT_DOWN = 0
168 OSD_STATUS_OUT_UP = 1
169 OSD_STATUS_IN_DOWN = 2
172 MOUNT_OPTIONS = dict(
173 btrfs='noatime,user_subvol_rm_allowed',
174 # user_xattr is default ever since linux 2.6.39 / 3.0, but we'll
175 # delay a moment before removing it fully because we did have some
176 # issues with ext4 before the xatts-in-leveldb work, and it seemed
177 # that user_xattr helped
178 ext4='noatime,user_xattr',
179 xfs='noatime,inode64',
184 # btrfs requires -f, for the same reason as xfs (see comment below)
191 # xfs insists on not overwriting previous fs; even if we wipe
192 # partition table, we often recreate it exactly the same way,
193 # so we'll see ghosts of filesystems past
207 STATEDIR = '/var/lib/ceph'
209 SYSCONFDIR = '/etc/ceph'
213 SUPPRESS_PREFIX = None
215 # only warn once about some things
218 # Nuke the TERM variable to avoid confusing any subprocesses we call.
219 # For example, libreadline will print weird control sequences for some
221 if 'TERM' in os.environ:
222 del os.environ['TERM']
225 if LOG_NAME == '__main__':
226 LOG_NAME = os.path.basename(sys.argv[0])
227 LOG = logging.getLogger(LOG_NAME)
229 # Allow user-preferred values for subprocess user and group
230 CEPH_PREF_USER = None
231 CEPH_PREF_GROUP = None
234 class filelock(object):
235 def __init__(self, fn):
241 self.fd = file(self.fn, 'w')
242 fcntl.lockf(self.fd, fcntl.LOCK_EX)
246 fcntl.lockf(self.fd, fcntl.LOCK_UN)
250 class Error(Exception):
256 doc = self.__doc__.strip()
257 return ': '.join([doc] + [str(a) for a in self.args])
260 class MountError(Error):
262 Mounting filesystem failed
266 class UnmountError(Error):
268 Unmounting filesystem failed
272 class BadMagicError(Error):
274 Does not look like a Ceph OSD, or incompatible version
278 class TruncatedLineError(Error):
284 class TooManyLinesError(Error):
290 class FilesystemTypeError(Error):
292 Cannot discover filesystem type
296 class CephDiskException(Exception):
298 A base exception for ceph-disk to provide custom (ad-hoc) messages that
299 will be caught and dealt with when main() is executed
304 class ExecutableNotFound(CephDiskException):
306 Exception to report on executables not available in PATH
313 Detect whether systemd is running
315 with file('/proc/1/comm', 'rb') as i:
317 if 'systemd' in line:
324 Detect whether upstart is running
326 (out, err, _) = command(['init', '--version'])
332 def maybe_mkdir(*a, **kw):
334 Creates a new directory if it doesn't exist, removes
335 existing symlink before creating the directory.
337 # remove any symlink, if it is there..
338 if os.path.exists(*a) and stat.S_ISLNK(os.lstat(*a).st_mode):
339 LOG.debug('Removing old symlink at %s', *a)
344 if e.errno == errno.EEXIST:
350 def which(executable):
351 """find the location of an executable"""
352 if 'PATH' in os.environ:
353 envpath = os.environ['PATH']
356 PATH = envpath.split(os.pathsep)
367 for location in locations:
368 executable_path = os.path.join(location, executable)
369 if (os.path.isfile(executable_path) and
370 os.access(executable_path, os.X_OK)):
371 return executable_path
374 def _get_command_executable(arguments):
376 Return the full path for an executable, raise if the executable is not
377 found. If the executable has already a full path do not perform any checks.
379 if arguments[0].startswith('/'): # an absolute path
381 executable = which(arguments[0])
383 command_msg = 'Could not run command: %s' % ' '.join(arguments)
384 executable_msg = '%s not in path.' % arguments[0]
385 raise ExecutableNotFound('%s %s' % (executable_msg, command_msg))
387 # swap the old executable for the new one
388 arguments[0] = executable
392 def command(arguments, **kwargs):
394 Safely execute a ``subprocess.Popen`` call making sure that the
395 executable exists and raising a helpful error message
398 .. note:: This should be the preferred way of calling ``subprocess.Popen``
399 since it provides the caller with the safety net of making sure that
400 executables *will* be found and will error nicely otherwise.
402 This returns the output of the command and the return code of the
403 process in a tuple: (output, returncode).
405 arguments = _get_command_executable(arguments)
406 LOG.info('Running command: %s' % ' '.join(arguments))
407 process = subprocess.Popen(
409 stdout=subprocess.PIPE,
410 stderr=subprocess.PIPE,
412 out, err = process.communicate()
413 return out, err, process.returncode
416 def command_check_call(arguments):
418 Safely execute a ``subprocess.check_call`` call making sure that the
419 executable exists and raising a helpful error message if it does not.
421 .. note:: This should be the preferred way of calling
422 ``subprocess.check_call`` since it provides the caller with the safety net
423 of making sure that executables *will* be found and will error nicely
426 arguments = _get_command_executable(arguments)
427 LOG.info('Running command: %s', ' '.join(arguments))
428 return subprocess.check_call(arguments)
431 def platform_distro():
433 Returns a normalized, lower case string without any leading nor trailing
434 whitespace that represents the distribution name of the current machine.
436 distro = platform_information()[0] or ''
437 return distro.strip().lower()
440 def platform_information():
441 distro, release, codename = platform.linux_distribution()
442 # this could be an empty string in Debian
443 if not codename and 'debian' in distro.lower():
449 major_version = release.split('.')[0]
450 codename = debian_codenames.get(major_version, '')
452 # In order to support newer jessie/sid or wheezy/sid strings we test
453 # this if sid is buried in the minor, we should use sid anyway.
454 if not codename and '/' in release:
455 major, minor = release.split('/')
463 str(release).strip(),
464 str(codename).strip()
468 # An alternative block_path implementation would be
470 # name = basename(dev)
471 # return /sys/devices/virtual/block/$name
473 # It is however more fragile because it relies on the fact
474 # that the basename of the device the user will use always
475 # matches the one the driver will use. On Ubuntu 14.04, for
476 # instance, when multipath creates a partition table on
478 # /dev/mapper/353333330000007d0 -> ../dm-0
480 # it will create partition devices named
482 # /dev/mapper/353333330000007d0-part1
484 # which is the same device as /dev/dm-1 but not a symbolic
487 # ubuntu@other:~$ ls -l /dev/mapper /dev/dm-1
488 # brw-rw---- 1 root disk 252, 1 Aug 15 17:52 /dev/dm-1
489 # lrwxrwxrwx 1 root root 7 Aug 15 17:52 353333330000007d0 -> ../dm-0
490 # brw-rw---- 1 root disk 252, 1 Aug 15 17:52 353333330000007d0-part1
492 # Using the basename in this case fails.
497 path = os.path.realpath(dev)
498 rdev = os.stat(path).st_rdev
499 (M, m) = (os.major(rdev), os.minor(rdev))
500 return "{sysfs}/dev/block/{M}:{m}".format(sysfs=SYSFS, M=M, m=m)
503 def get_dm_uuid(dev):
504 uuid_path = os.path.join(block_path(dev), 'dm', 'uuid')
505 LOG.debug("get_dm_uuid " + dev + " uuid path is " + uuid_path)
506 if not os.path.exists(uuid_path):
508 uuid = open(uuid_path, 'r').read()
509 LOG.debug("get_dm_uuid " + dev + " uuid is " + uuid)
515 True if the path is managed by multipath
517 uuid = get_dm_uuid(dev)
519 (re.match('part\d+-mpath-', uuid) or
520 re.match('mpath-', uuid)))
523 def get_dev_name(path):
525 get device name from path. e.g.::
527 /dev/sda -> sdas, /dev/cciss/c0d1 -> cciss!c0d1
529 a device "name" is something like::
535 assert path.startswith('/dev/')
537 return base.replace('/', '!')
540 def get_dev_path(name):
542 get a path (/dev/...) from a name (cciss!c0d1)
543 a device "path" is something like::
549 return '/dev/' + name.replace('!', '/')
552 def get_dev_relpath(name):
554 get a relative path to /dev from a name (cciss!c0d1)
556 return name.replace('!', '/')
559 def get_dev_size(dev, size='megabytes'):
561 Attempt to get the size of a device so that we can prevent errors
562 from actions to devices that are smaller, and improve error reporting.
564 Because we want to avoid breakage in case this approach is not robust, we
565 will issue a warning if we failed to get the size.
567 :param size: bytes or megabytes
568 :param dev: the device to calculate the size
570 fd = os.open(dev, os.O_RDONLY)
571 dividers = {'bytes': 1, 'megabytes': 1024 * 1024}
573 device_size = os.lseek(fd, 0, os.SEEK_END)
574 divider = dividers.get(size, 1024 * 1024) # default to megabytes
575 return device_size / divider
576 except Exception as error:
577 LOG.warning('failed to get size of %s: %s' % (dev, str(error)))
582 def get_partition_mpath(dev, pnum):
583 part_re = "part{pnum}-mpath-".format(pnum=pnum)
584 partitions = list_partitions_mpath(dev, part_re)
591 def get_partition_dev(dev, pnum):
593 get the device name for a partition
595 assume that partitions are named like the base dev,
596 with a number, and optionally
597 some intervening characters (like 'p'). e.g.,
600 cciss/c0d1 1 -> cciss!c0d1p1
604 partname = get_partition_mpath(dev, pnum)
606 name = get_dev_name(os.path.realpath(dev))
607 for f in os.listdir(os.path.join('/sys/block', name)):
608 if f.startswith(name) and f.endswith(str(pnum)):
609 # we want the shortest name that starts with the base name
610 # and ends with the partition number
611 if not partname or len(f) < len(partname):
614 return get_dev_path(partname)
616 raise Error('partition %d for %s does not appear to exist' %
620 def list_all_partitions():
622 Return a list of devices and partitions
624 names = os.listdir('/sys/block')
627 # /dev/fd0 may hang http://tracker.ceph.com/issues/6827
628 if re.match(r'^fd\d$', name):
630 dev_part_list[name] = list_partitions(get_dev_path(name))
634 def list_partitions(dev):
635 dev = os.path.realpath(dev)
637 return list_partitions_mpath(dev)
639 return list_partitions_device(dev)
642 def list_partitions_mpath(dev, part_re="part\d+-mpath-"):
645 holders = os.path.join(p, 'holders')
646 for holder in os.listdir(holders):
647 uuid_path = os.path.join(holders, holder, 'dm', 'uuid')
648 uuid = open(uuid_path, 'r').read()
649 LOG.debug("list_partitions_mpath: " + uuid_path + " uuid = " + uuid)
650 if re.match(part_re, uuid):
651 partitions.append(holder)
655 def list_partitions_device(dev):
657 Return a list of partitions on the given device name
660 basename = get_dev_name(dev)
661 for name in os.listdir(block_path(dev)):
662 if name.startswith(basename):
663 partitions.append(name)
667 def get_partition_base(dev):
669 Get the base device for a partition
671 dev = os.path.realpath(dev)
672 if not stat.S_ISBLK(os.lstat(dev).st_mode):
673 raise Error('not a block device', dev)
675 name = get_dev_name(dev)
676 if os.path.exists(os.path.join('/sys/block', name)):
677 raise Error('not a partition', dev)
680 for basename in os.listdir('/sys/block'):
681 if os.path.exists(os.path.join('/sys/block', basename, name)):
682 return get_dev_path(basename)
683 raise Error('no parent device for partition', dev)
686 def is_partition_mpath(dev):
687 uuid = get_dm_uuid(dev)
688 return bool(re.match('part\d+-mpath-', uuid))
691 def partnum_mpath(dev):
692 uuid = get_dm_uuid(dev)
693 return re.findall('part(\d+)-mpath-', uuid)[0]
696 def get_partition_base_mpath(dev):
697 slave_path = os.path.join(block_path(dev), 'slaves')
698 slaves = os.listdir(slave_path)
700 name_path = os.path.join(slave_path, slaves[0], 'dm', 'name')
701 name = open(name_path, 'r').read().strip()
702 return os.path.join('/dev/mapper', name)
705 def is_partition(dev):
707 Check whether a given device path is a partition or a full disk.
710 return is_partition_mpath(dev)
712 dev = os.path.realpath(dev)
714 if not stat.S_ISBLK(st.st_mode):
715 raise Error('not a block device', dev)
717 name = get_dev_name(dev)
718 if os.path.exists(os.path.join('/sys/block', name)):
721 # make sure it is a partition of something else
722 major = os.major(st.st_rdev)
723 minor = os.minor(st.st_rdev)
724 if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)):
727 raise Error('not a disk or partition', dev)
732 Check if the given device is mounted.
734 dev = os.path.realpath(dev)
735 with file('/proc/mounts', 'rb') as proc_mounts:
736 for line in proc_mounts:
737 fields = line.split()
740 mounts_dev = fields[0]
742 if mounts_dev.startswith('/') and os.path.exists(mounts_dev):
743 mounts_dev = os.path.realpath(mounts_dev)
744 if mounts_dev == dev:
751 Check if a device is held by another device (e.g., a dm-crypt mapping)
753 assert os.path.exists(dev)
757 dev = os.path.realpath(dev)
758 base = get_dev_name(dev)
761 directory = '/sys/block/{base}/holders'.format(base=base)
762 if os.path.exists(directory):
763 return os.listdir(directory)
768 directory = '/sys/block/{base}/{part}/holders'.format(
769 part=part, base=base)
770 if os.path.exists(directory):
771 return os.listdir(directory)
776 def verify_not_in_use(dev, check_partitions=False):
778 Verify if a given device (path) is in use (e.g. mounted or
779 in use by device-mapper).
781 :raises: Error if device is in use.
783 assert os.path.exists(dev)
785 raise Error('Device is mounted', dev)
786 holders = is_held(dev)
788 raise Error('Device %s is in use by a device-mapper '
789 'mapping (dm-crypt?)' % dev, ','.join(holders))
791 if check_partitions and not is_partition(dev):
792 for partname in list_partitions(dev):
793 partition = get_dev_path(partname)
794 if is_mounted(partition):
795 raise Error('Device is mounted', partition)
796 holders = is_held(partition)
798 raise Error('Device %s is in use by a device-mapper '
799 'mapping (dm-crypt?)'
800 % partition, ','.join(holders))
803 def must_be_one_line(line):
805 Checks if given line is really one single line.
807 :raises: TruncatedLineError or TooManyLinesError
808 :return: Content of the line, or None if line isn't valid.
810 if line[-1:] != '\n':
811 raise TruncatedLineError(line)
814 raise TooManyLinesError(line)
818 def read_one_line(parent, name):
820 Read a file whose sole contents are a single line.
824 :return: Contents of the line, or None if file did not exist.
826 path = os.path.join(parent, name)
828 line = file(path, 'rb').read()
830 if e.errno == errno.ENOENT:
836 line = must_be_one_line(line)
837 except (TruncatedLineError, TooManyLinesError) as e:
839 'File is corrupt: {path}: {msg}'.format(
847 def write_one_line(parent, name, text):
849 Write a file whose sole contents are a single line.
853 path = os.path.join(parent, name)
854 tmp = '{path}.{pid}.tmp'.format(path=path, pid=os.getpid())
855 with file(tmp, 'wb') as tmp_file:
856 tmp_file.write(text + '\n')
857 os.fsync(tmp_file.fileno())
858 path_set_context(tmp)
864 Get a init system using 'ceph-detect-init'
866 init = _check_output(
869 '--default', 'sysvinit',
872 init = must_be_one_line(init)
876 def check_osd_magic(path):
878 Check that this path has the Ceph OSD magic.
880 :raises: BadMagicError if this does not look like a Ceph OSD data
883 magic = read_one_line(path, 'magic')
885 # probably not mkfs'ed yet
886 raise BadMagicError(path)
887 if magic != CEPH_OSD_ONDISK_MAGIC:
888 raise BadMagicError(path)
891 def check_osd_id(osd_id):
893 Ensures osd id is numeric.
895 if not re.match(r'^[0-9]+$', osd_id):
896 raise Error('osd id is not numeric', osd_id)
905 Accocates an OSD id on the given cluster.
907 :raises: Error if the call to allocate the OSD id fails.
908 :return: The allocated OSD id.
911 LOG.debug('Allocating OSD id...')
913 osd_id = _check_output(
916 '--cluster', cluster,
917 '--name', 'client.bootstrap-osd',
918 '--keyring', keyring,
919 'osd', 'create', '--concise',
923 except subprocess.CalledProcessError as e:
924 raise Error('ceph osd create failed', e, e.output)
925 osd_id = must_be_one_line(osd_id)
930 def get_osd_id(path):
932 Gets the OSD id of the OSD at the given path.
934 osd_id = read_one_line(path, 'whoami')
935 if osd_id is not None:
941 global CEPH_PREF_USER
943 if CEPH_PREF_USER is not None:
945 pwd.getpwnam(CEPH_PREF_USER)
946 return CEPH_PREF_USER
948 print "No such user: " + CEPH_PREF_USER
958 def get_ceph_group():
959 global CEPH_PREF_GROUP
961 if CEPH_PREF_GROUP is not None:
963 grp.getgrnam(CEPH_PREF_GROUP)
964 return CEPH_PREF_GROUP
966 print "No such group: " + CEPH_PREF_GROUP
976 def path_set_context(path):
977 # restore selinux context to default policy values
978 if which('restorecon'):
979 command(['restorecon', '-R', path])
981 # if ceph user exists, set owner to ceph
982 if get_ceph_user() == 'ceph':
983 command(['chown', '-R', 'ceph:ceph', path])
986 def _check_output(args=None, **kwargs):
987 out, err, ret = command(args, **kwargs)
990 error = subprocess.CalledProcessError(ret, cmd)
991 error.output = out + err
996 def get_conf(cluster, variable):
998 Get the value of the given configuration variable from the
1001 :raises: Error if call to ceph-conf fails.
1002 :return: The variable value or None.
1005 out, err, ret = command(
1008 '--cluster={cluster}'.format(
1017 except OSError as e:
1018 raise Error('error executing ceph-conf', e, err)
1020 # config entry not found
1023 raise Error('getting variable from configuration failed')
1024 value = out.split('\n', 1)[0]
1025 # don't differentiate between "var=" and no var set
1031 def get_conf_with_default(cluster, variable):
1033 Get a config value that is known to the C++ code.
1035 This will fail if called on variables that are not defined in
1036 common config options.
1039 out = _check_output(
1042 '--cluster={cluster}'.format(
1045 '--show-config-value={variable}'.format(
1051 except subprocess.CalledProcessError as e:
1053 'getting variable from configuration failed',
1057 value = str(out).split('\n', 1)[0]
1061 def get_fsid(cluster):
1063 Get the fsid of the cluster.
1065 :return: The fsid or raises Error.
1067 fsid = get_conf_with_default(cluster=cluster, variable='fsid')
1069 raise Error('getting cluster uuid from configuration failed')
1073 def get_dmcrypt_key_path(
1079 Get path to dmcrypt key file.
1081 :return: Path to the dmcrypt key file, callers should check for existence.
1084 path = os.path.join(key_dir, _uuid + ".luks.key")
1086 path = os.path.join(key_dir, _uuid)
1091 def get_or_create_dmcrypt_key(
1098 Get path to existing dmcrypt key or create a new key file.
1100 :return: Path to the dmcrypt key file.
1102 path = get_dmcrypt_key_path(_uuid, key_dir, luks)
1103 if os.path.exists(path):
1108 if not os.path.exists(key_dir):
1109 os.makedirs(key_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
1110 with file('/dev/urandom', 'rb') as i:
1111 key = i.read(key_size / 8)
1112 fd = os.open(path, os.O_WRONLY | os.O_CREAT,
1113 stat.S_IRUSR | stat.S_IWUSR)
1114 assert os.write(fd, key) == len(key)
1118 raise Error('unable to read or create dm-crypt key', path)
1125 cryptsetup_parameters,
1130 Maps a device to a dmcrypt device.
1132 :return: Path to the dmcrypt device.
1134 dev = '/dev/mapper/' + _uuid
1142 ] + cryptsetup_parameters
1160 ] + cryptsetup_parameters
1165 command_check_call(luksFormat_args)
1166 command_check_call(luksOpen_args)
1168 # Plain mode has no format function, nor any validation
1169 # that the key is correct.
1170 command_check_call(create_args)
1171 # set proper ownership of mapped device
1172 command_check_call(['chown', 'ceph:ceph', dev])
1175 except subprocess.CalledProcessError as e:
1176 raise Error('unable to map device', rawdev, e)
1183 Removes the dmcrypt device with the given UUID.
1188 command_check_call(['cryptsetup', 'remove', _uuid])
1190 except subprocess.CalledProcessError as e:
1192 raise Error('unable to unmap device', _uuid, e)
1194 time.sleep(0.5 + retries * 1.0)
1204 Mounts a device with given filessystem type and
1205 mount options to a tempfile path under /var/lib/ceph/tmp.
1207 # sanity check: none of the arguments are None
1209 raise ValueError('dev may not be None')
1211 raise ValueError('fstype may not be None')
1213 # pick best-of-breed mount options based on fs type
1215 options = MOUNT_OPTIONS.get(fstype, '')
1218 path = tempfile.mkdtemp(
1220 dir=STATEDIR + '/tmp',
1223 LOG.debug('Mounting %s on %s with options %s', dev, path, options)
1234 if which('restorecon'):
1241 except subprocess.CalledProcessError as e:
1244 except (OSError, IOError):
1255 Unmount and removes the given mount point.
1260 LOG.debug('Unmounting %s', path)
1269 except subprocess.CalledProcessError as e:
1270 # on failure, retry 3 times with incremental backoff
1272 raise UnmountError(e)
1274 time.sleep(0.5 + retries * 1.0)
1280 ###########################################
1282 def extract_parted_partition_numbers(partitions):
1283 numbers_as_strings = re.findall('^\d+', partitions, re.MULTILINE)
1284 return map(int, numbers_as_strings)
1287 def get_free_partition_index(dev):
1289 Get the next free partition index on a given device.
1291 :return: Index number (> 1 if there is already a partition on the device)
1292 or 1 if there is no partition table.
1295 lines = _check_output(
1304 except subprocess.CalledProcessError as e:
1305 LOG.info('cannot read partition index; assume it '
1306 'isn\'t present\n (Error: %s)' % e)
1310 raise Error('parted failed to output anything')
1311 LOG.debug('get_free_partition_index: analyzing ' + lines)
1312 if ('CHS;' not in lines and
1313 'CYL;' not in lines and
1314 'BYT;' not in lines):
1315 raise Error('parted output expected to contain one of ' +
1316 'CHH; CYL; or BYT; : ' + lines)
1317 if os.path.realpath(dev) not in lines:
1318 raise Error('parted output expected to contain ' + dev + ': ' + lines)
1319 _, partitions = lines.split(os.path.realpath(dev))
1320 partition_numbers = extract_parted_partition_numbers(partitions)
1321 if partition_numbers:
1322 return max(partition_numbers) + 1
1327 def check_journal_reqs(args):
1328 _, _, allows_journal = command([
1329 'ceph-osd', '--check-allows-journal',
1331 '--cluster', args.cluster,
1333 _, _, wants_journal = command([
1334 'ceph-osd', '--check-wants-journal',
1336 '--cluster', args.cluster,
1338 _, _, needs_journal = command([
1339 'ceph-osd', '--check-needs-journal',
1341 '--cluster', args.cluster,
1343 return (not allows_journal, not wants_journal, not needs_journal)
1346 def update_partition(dev, description):
1348 Must be called after modifying a partition table so the kernel
1349 know about the change and fire udev events accordingly. A side
1350 effect of partprobe is to remove partitions and add them again.
1351 The first udevadm settle waits for ongoing udev events to
1352 complete, just in case one of them rely on an existing partition
1353 on dev. The second udevadm settle guarantees to the caller that
1354 all udev events related to the partition table change have been
1355 processed, i.e. the 95-ceph-osd.rules actions and mode changes,
1356 group changes etc. are complete.
1358 LOG.debug('Calling partprobe on %s device %s', description, dev)
1359 partprobe_ok = False
1360 error = 'unknown error'
1361 for i in (1, 2, 3, 4, 5):
1362 command_check_call(['udevadm', 'settle', '--timeout=600'])
1364 _check_output(['partprobe', dev])
1367 except subprocess.CalledProcessError as e:
1369 if ('unable to inform the kernel' not in error and
1370 'Device or resource busy' not in error):
1372 LOG.debug('partprobe %s failed : %s (ignored, waiting 60s)'
1375 if not partprobe_ok:
1376 raise Error('partprobe %s failed : %s' % (dev, error))
1377 command_check_call(['udevadm', 'settle', '--timeout=600'])
1382 Destroy the partition table and content of a given disk.
1384 dev = os.path.realpath(dev)
1385 dmode = os.stat(dev).st_mode
1386 if not stat.S_ISBLK(dmode) or is_partition(dev):
1387 raise Error('not full block device; cannot zap', dev)
1389 LOG.debug('Zapping partition table on %s', dev)
1391 # try to wipe out any GPT partition table backups. sgdisk
1392 # isn't too thorough.
1394 size = 33 * lba_size
1395 with file(dev, 'wb') as dev_file:
1396 dev_file.seek(-size, os.SEEK_END)
1397 dev_file.write(size * '\0')
1417 update_partition(dev, 'zapped')
1419 except subprocess.CalledProcessError as e:
1423 def adjust_symlink(target, path):
1425 if os.path.lexists(path):
1427 mode = os.lstat(path).st_mode
1428 if stat.S_ISREG(mode):
1429 LOG.debug('Removing old file %s', path)
1431 elif stat.S_ISLNK(mode):
1432 old = os.readlink(path)
1434 LOG.debug('Removing old symlink %s -> %s', path, old)
1439 raise Error('unable to remove (or adjust) old file (symlink)',
1442 LOG.debug('Creating symlink %s -> %s', path, target)
1444 os.symlink(target, path)
1446 raise Error('unable to create symlink %s -> %s' % (path, target))
1449 class Device(object):
1451 def __init__(self, path, args):
1454 self.dev_size = None
1455 self.partitions = {}
1456 self.ptype_map = None
1457 assert not is_partition(self.path)
1459 def create_partition(self, uuid, name, size=0, num=0):
1460 ptype = self.ptype_tobe_for_name(name)
1462 num = get_free_partition_index(dev=self.path)
1464 new = '--new={num}:0:+{size}M'.format(num=num, size=size)
1465 if size > self.get_dev_size():
1466 LOG.error('refusing to create %s on %s' % (name, self.path))
1467 LOG.error('%s size (%sM) is bigger than device (%sM)'
1468 % (name, size, self.get_dev_size()))
1469 raise Error('%s device size (%sM) is not big enough for %s'
1470 % (self.path, self.get_dev_size(), name))
1472 new = '--largest-new={num}'.format(num=num)
1474 LOG.debug('Creating %s partition num %d size %d on %s',
1475 name, num, size, self.path)
1480 '--change-name={num}:ceph {name}'.format(num=num, name=name),
1481 '--partition-guid={num}:{uuid}'.format(num=num, uuid=uuid),
1482 '--typecode={num}:{uuid}'.format(num=num, uuid=ptype),
1488 update_partition(self.path, 'created')
1491 def ptype_tobe_for_name(self, name):
1494 if self.ptype_map is None:
1495 partition = DevicePartition.factory(
1496 path=self.path, dev=None, args=self.args)
1497 self.ptype_map = partition.ptype_map
1498 return self.ptype_map[name]['tobe']
1500 def get_partition(self, num):
1501 if num not in self.partitions:
1502 dev = get_partition_dev(self.path, num)
1503 partition = DevicePartition.factory(
1504 path=self.path, dev=dev, args=self.args)
1505 partition.set_partition_number(num)
1506 self.partitions[num] = partition
1507 return self.partitions[num]
1509 def get_dev_size(self):
1510 if self.dev_size is None:
1511 self.dev_size = get_dev_size(self.path)
1512 return self.dev_size
1515 def factory(path, args):
1516 return Device(path, args)
1519 class DevicePartition(object):
1521 def __init__(self, args):
1527 self.ptype_map = None
1529 self.set_variables_ptype()
1532 if self.uuid is None:
1533 self.uuid = get_partition_uuid(self.rawdev)
1536 def get_ptype(self):
1537 if self.ptype is None:
1538 self.ptype = get_partition_type(self.rawdev)
1541 def set_partition_number(self, num):
1544 def get_partition_number(self):
1547 def set_dev(self, dev):
1554 def get_rawdev(self):
1557 def set_variables_ptype(self):
1558 self.ptype_map = PTYPE['regular']
1560 def ptype_for_name(self, name):
1561 return self.ptype_map[name]['ready']
1564 def factory(path, dev, args):
1565 dmcrypt_type = CryptHelpers.get_dmcrypt_type(args)
1566 if ((path is not None and is_mpath(path)) or
1567 (dev is not None and is_mpath(dev))):
1568 partition = DevicePartitionMultipath(args)
1569 elif dmcrypt_type == 'luks':
1570 partition = DevicePartitionCryptLuks(args)
1571 elif dmcrypt_type == 'plain':
1572 partition = DevicePartitionCryptPlain(args)
1574 partition = DevicePartition(args)
1575 partition.set_dev(dev)
1579 class DevicePartitionMultipath(DevicePartition):
1581 def set_variables_ptype(self):
1582 self.ptype_map = PTYPE['mpath']
1585 class DevicePartitionCrypt(DevicePartition):
1587 def __init__(self, args):
1588 super(DevicePartitionCrypt, self).__init__(args)
1589 self.osd_dm_keypath = None
1590 self.cryptsetup_parameters = CryptHelpers.get_cryptsetup_parameters(
1592 self.dmcrypt_type = CryptHelpers.get_dmcrypt_type(self.args)
1593 self.dmcrypt_keysize = CryptHelpers.get_dmcrypt_keysize(self.args)
1595 def setup_crypt(self):
1600 self.dev = _dmcrypt_map(
1602 keypath=self.osd_dm_keypath,
1603 _uuid=self.get_uuid(),
1604 cryptsetup_parameters=self.cryptsetup_parameters,
1611 dmcrypt_unmap(self.get_uuid())
1612 self.dev = self.rawdev
1620 class DevicePartitionCryptPlain(DevicePartitionCrypt):
1625 def setup_crypt(self):
1626 if self.osd_dm_keypath is not None:
1629 self.cryptsetup_parameters += ['--key-size', str(self.dmcrypt_keysize)]
1631 self.osd_dm_keypath = get_or_create_dmcrypt_key(
1632 self.get_uuid(), self.args.dmcrypt_key_dir,
1633 self.dmcrypt_keysize, False)
1635 def set_variables_ptype(self):
1636 self.ptype_map = PTYPE['plain']
1639 class DevicePartitionCryptLuks(DevicePartitionCrypt):
1644 def setup_crypt(self):
1645 if self.osd_dm_keypath is not None:
1648 if self.dmcrypt_keysize == 1024:
1649 # We don't force this into the cryptsetup_parameters,
1650 # as we want the cryptsetup defaults
1651 # to prevail for the actual LUKS key lengths.
1654 self.cryptsetup_parameters += ['--key-size',
1655 str(self.dmcrypt_keysize)]
1657 self.osd_dm_keypath = get_or_create_dmcrypt_key(
1658 self.get_uuid(), self.args.dmcrypt_key_dir,
1659 self.dmcrypt_keysize, True)
1661 def set_variables_ptype(self):
1662 self.ptype_map = PTYPE['luks']
1665 class Prepare(object):
1669 parser = argparse.ArgumentParser(add_help=False)
1670 parser.add_argument(
1674 help='cluster name to assign this disk to',
1676 parser.add_argument(
1679 help='cluster uuid to assign this disk to',
1681 parser.add_argument(
1684 help='unique OSD uuid to assign this disk to',
1686 parser.add_argument(
1688 action='store_true', default=None,
1689 help='encrypt DATA and/or JOURNAL devices with dm-crypt',
1691 parser.add_argument(
1692 '--dmcrypt-key-dir',
1694 default='/etc/ceph/dmcrypt-keys',
1695 help='directory where dm-crypt keys are stored',
1700 def set_subparser(subparsers):
1703 PrepareData.parser(),
1705 parents.extend(PrepareFilestore.parent_parsers())
1706 parents.extend(PrepareBluestore.parent_parsers())
1707 parser = subparsers.add_parser(
1710 help='Prepare a directory or disk for a Ceph OSD',
1712 parser.set_defaults(
1718 prepare_lock.acquire()
1719 self.prepare_locked()
1720 prepare_lock.release()
1725 return PrepareBluestore(args)
1727 return PrepareFilestore(args)
1731 Prepare.factory(args).prepare()
1734 class PrepareFilestore(Prepare):
1736 def __init__(self, args):
1737 self.data = PrepareFilestoreData(args)
1738 self.journal = PrepareJournal(args)
1741 def parent_parsers():
1743 PrepareJournal.parser(),
1746 def prepare_locked(self):
1747 self.data.prepare(self.journal)
1750 class PrepareBluestore(Prepare):
1752 def __init__(self, args):
1753 self.data = PrepareBluestoreData(args)
1754 self.block = PrepareBluestoreBlock(args)
1758 parser = argparse.ArgumentParser(add_help=False)
1759 parser.add_argument(
1761 action='store_true', default=None,
1762 help='bluestore objectstore',
1764 parser.add_argument(
1766 action='store_true', default=True,
1767 help='IGNORED FORWARD COMPATIBIILTY HACK',
1772 def parent_parsers():
1774 PrepareBluestore.parser(),
1775 PrepareBluestoreBlock.parser(),
1778 def prepare_locked(self):
1779 self.data.prepare(self.block)
1782 class Space(object):
1784 NAMES = ('block', 'journal')
1787 class PrepareSpace(object):
1793 def __init__(self, args):
1796 self.space_size = self.get_space_size()
1797 if (getattr(self.args, self.name) and
1798 getattr(self.args, self.name + '_uuid') is None):
1799 setattr(self.args, self.name + '_uuid', str(uuid.uuid4()))
1800 self.space_symlink = None
1801 self.space_dmcrypt = None
1806 dmode = os.stat(args.data).st_mode
1807 if (self.wants_space() and
1808 stat.S_ISBLK(dmode) and
1809 not is_partition(args.data) and
1810 getattr(args, name) is None and
1811 getattr(args, name + '_file') is None):
1812 LOG.info('Will colocate %s with data on %s',
1814 setattr(args, name, args.data)
1816 if getattr(args, name) is None:
1817 if getattr(args, name + '_dev'):
1818 raise Error('%s is unspecified; not a block device' %
1819 name.capitalize(), getattr(args, name))
1820 self.type = self.NONE
1823 if not os.path.exists(getattr(args, name)):
1824 if getattr(args, name + '_dev'):
1825 raise Error('%s does not exist; not a block device' %
1826 name.capitalize(), getattr(args, name))
1827 self.type = self.FILE
1830 mode = os.stat(getattr(args, name)).st_mode
1831 if stat.S_ISBLK(mode):
1832 if getattr(args, name + '_file'):
1833 raise Error('%s is not a regular file' % name.capitalize,
1835 self.type = self.DEVICE
1838 if stat.S_ISREG(mode):
1839 if getattr(args, name + '_dev'):
1840 raise Error('%s is not a block device' % name.capitalize,
1842 self.type = self.FILE
1844 raise Error('%s %s is neither a block device nor regular file' %
1845 (name.capitalize, geattr(args, name)))
1848 return self.type == self.NONE
1851 return self.type == self.FILE
1853 def is_device(self):
1854 return self.type == self.DEVICE
1858 parser = argparse.ArgumentParser(add_help=False)
1859 parser.add_argument(
1862 help='unique uuid to assign to the %s' % name,
1864 parser.add_argument(
1866 action='store_true', default=None,
1867 help='verify that %s is a file' % name.upper(),
1869 parser.add_argument(
1871 action='store_true', default=None,
1872 help='verify that %s is a block device' % name.upper(),
1874 parser.add_argument(
1876 metavar=name.upper(),
1878 help=('path to OSD %s disk block device;' % name +
1879 ' leave out to store %s in file' % name),
1883 def wants_space(self):
1886 def populate_data_path(self, path):
1887 if self.type == self.DEVICE:
1888 self.populate_data_path_device(path)
1889 elif self.type == self.FILE:
1890 self.populate_data_path_file(path)
1891 elif self.type == self.NONE:
1894 raise Error('unexpected type ', self.type)
1896 def populate_data_path_file(self, path):
1897 space_uuid = self.name + '_uuid'
1898 if getattr(self.args, space_uuid) is not None:
1899 write_one_line(path, space_uuid,
1900 getattr(self.args, space_uuid))
1902 def populate_data_path_device(self, path):
1903 self.populate_data_path_file(path)
1904 if self.space_symlink is not None:
1905 adjust_symlink(self.space_symlink,
1906 os.path.join(path, self.name))
1908 if self.space_dmcrypt is not None:
1909 adjust_symlink(self.space_dmcrypt,
1910 os.path.join(path, self.name + '_dmcrypt'))
1913 os.unlink(os.path.join(path, self.name + '_dmcrypt'))
1918 if self.type == self.DEVICE:
1919 self.prepare_device()
1920 elif self.type == self.FILE:
1922 elif self.type == self.NONE:
1925 raise Error('unexpected type ', self.type)
1927 def prepare_file(self):
1928 if not os.path.exists(getattr(self.args, self.name)):
1929 LOG.debug('Creating %s file %s with size 0'
1930 ' (ceph-osd will resize and allocate)',
1932 getattr(self.args, self.name))
1933 with file(getattr(self.args, self.name), 'wb') as space_file:
1936 LOG.debug('%s is file %s',
1937 self.name.capitalize(),
1938 getattr(self.args, self.name))
1939 LOG.warning('OSD will not be hot-swappable if %s is '
1940 'not the same device as the osd data' %
1942 self.space_symlink = space_file
1944 def prepare_device(self):
1945 reusing_partition = False
1947 if is_partition(getattr(self.args, self.name)):
1948 LOG.debug('%s %s is a partition',
1949 self.name.capitalize(), getattr(self.args, self.name))
1950 partition = DevicePartition.factory(
1951 path=None, dev=getattr(self.args, self.name), args=self.args)
1952 if isinstance(partition, DevicePartitionCrypt):
1953 raise Error(getattr(self.args, self.name) +
1954 ' partition already exists'
1955 ' and --dmcrypt specified')
1956 LOG.warning('OSD will not be hot-swappable' +
1957 ' if ' + self.name + ' is not' +
1958 ' the same device as the osd data')
1959 if partition.get_ptype() == partition.ptype_for_name(self.name):
1960 LOG.debug('%s %s was previously prepared with '
1961 'ceph-disk. Reusing it.',
1962 self.name.capitalize(),
1963 getattr(self.args, self.name))
1964 reusing_partition = True
1965 # Read and reuse the partition uuid from this journal's
1966 # previous life. We reuse the uuid instead of changing it
1967 # because udev does not reliably notice changes to an
1968 # existing partition's GUID. See
1969 # http://tracker.ceph.com/issues/10146
1970 setattr(self.args, self.name + '_uuid', partition.get_uuid())
1971 LOG.debug('Reusing %s with uuid %s',
1973 getattr(self.args, self.name + '_uuid'))
1975 LOG.warning('%s %s was not prepared with '
1976 'ceph-disk. Symlinking directly.',
1977 self.name.capitalize(),
1978 getattr(self.args, self.name))
1979 self.space_symlink = getattr(self.args, self.name)
1982 self.space_symlink = '/dev/disk/by-partuuid/{uuid}'.format(
1983 uuid=getattr(self.args, self.name + '_uuid'))
1985 if self.args.dmcrypt:
1986 self.space_dmcrypt = self.space_symlink
1987 self.space_symlink = '/dev/mapper/{uuid}'.format(
1988 uuid=getattr(self.args, self.name + '_uuid'))
1990 if reusing_partition:
1991 # confirm that the space_symlink exists. It should since
1992 # this was an active space
1993 # in the past. Continuing otherwise would be futile.
1994 assert os.path.exists(self.space_symlink)
1997 num = self.desired_partition_number()
2000 LOG.warning('OSD will not be hot-swappable if %s '
2001 'is not the same device as the osd data',
2004 device = Device.factory(getattr(self.args, self.name), self.args)
2005 num = device.create_partition(
2006 uuid=getattr(self.args, self.name + '_uuid'),
2008 size=self.space_size,
2011 partition = device.get_partition(num)
2013 LOG.debug('%s is GPT partition %s',
2014 self.name.capitalize(),
2017 if isinstance(partition, DevicePartitionCrypt):
2023 '--typecode={num}:{uuid}'.format(
2025 uuid=partition.ptype_for_name(self.name),
2028 getattr(self.args, self.name),
2032 LOG.debug('%s is GPT partition %s',
2033 self.name.capitalize(),
2037 class PrepareJournal(PrepareSpace):
2039 def __init__(self, args):
2040 self.name = 'journal'
2041 (self.allows_journal,
2043 self.needs_journal) = check_journal_reqs(args)
2045 if args.journal and not self.allows_journal:
2046 raise Error('journal specified but not allowed by osd backend')
2048 super(PrepareJournal, self).__init__(args)
2050 def wants_space(self):
2051 return self.wants_journal
2053 def get_space_size(self):
2054 return int(get_conf_with_default(
2055 cluster=self.args.cluster,
2056 variable='osd_journal_size',
2059 def desired_partition_number(self):
2060 if self.args.journal == self.args.data:
2061 # we're sharing the disk between osd data and journal;
2062 # make journal be partition number 2
2070 return PrepareSpace.parser('journal')
2073 class PrepareBluestoreBlock(PrepareSpace):
2075 def __init__(self, args):
2077 super(PrepareBluestoreBlock, self).__init__(args)
2079 def get_space_size(self):
2080 return 0 # get as much space as possible
2082 def desired_partition_number(self):
2083 if self.args.block == self.args.data:
2091 return PrepareSpace.parser('block')
2094 class CryptHelpers(object):
2097 def get_cryptsetup_parameters(args):
2098 cryptsetup_parameters_str = get_conf(
2099 cluster=args.cluster,
2100 variable='osd_cryptsetup_parameters',
2102 if cryptsetup_parameters_str is None:
2105 return shlex.split(cryptsetup_parameters_str)
2108 def get_dmcrypt_keysize(args):
2109 dmcrypt_keysize_str = get_conf(
2110 cluster=args.cluster,
2111 variable='osd_dmcrypt_key_size',
2113 dmcrypt_type = CryptHelpers.get_dmcrypt_type(args)
2114 if dmcrypt_type == 'luks':
2115 if dmcrypt_keysize_str is None:
2116 # As LUKS will hash the 'passphrase' in .luks.key
2117 # into a key, set a large default
2118 # so if not updated for some time, it is still a
2123 return int(dmcrypt_keysize_str)
2124 elif dmcrypt_type == 'plain':
2125 if dmcrypt_keysize_str is None:
2126 # This value is hard-coded in the udev script
2129 LOG.warning('ensure the 95-ceph-osd.rules file has '
2130 'been copied to /etc/udev/rules.d '
2131 'and modified to call cryptsetup '
2132 'with --key-size=%s' % dmcrypt_keysize_str)
2133 return int(dmcrypt_keysize_str)
2138 def get_dmcrypt_type(args):
2140 dmcrypt_type = get_conf(
2141 cluster=args.cluster,
2142 variable='osd_dmcrypt_type',
2145 if dmcrypt_type is None or dmcrypt_type == 'luks':
2147 elif dmcrypt_type == 'plain':
2150 raise Error('invalid osd_dmcrypt_type parameter '
2151 '(must be luks or plain): ', dmcrypt_type)
2156 class PrepareData(object):
2161 def __init__(self, args):
2164 self.partition = None
2166 if self.args.cluster_uuid is None:
2167 self.args.cluster_uuid = get_fsid(cluster=self.args.cluster)
2169 if self.args.osd_uuid is None:
2170 self.args.osd_uuid = str(uuid.uuid4())
2173 dmode = os.stat(self.args.data).st_mode
2175 if stat.S_ISDIR(dmode):
2176 self.type = self.FILE
2177 elif stat.S_ISBLK(dmode):
2178 self.type = self.DEVICE
2180 raise Error('not a dir or block device', args.data)
2183 return self.type == self.FILE
2185 def is_device(self):
2186 return self.type == self.DEVICE
2190 parser = argparse.ArgumentParser(add_help=False)
2191 parser.add_argument(
2193 help='file system type to use (e.g. "ext4")',
2195 parser.add_argument(
2197 action='store_true', default=None,
2198 help='destroy the partition table (and content) of a disk',
2200 parser.add_argument(
2202 action='store_true', default=None,
2203 help='verify that DATA is a dir',
2205 parser.add_argument(
2207 action='store_true', default=None,
2208 help='verify that DATA is a block device',
2210 parser.add_argument(
2213 help='path to OSD data (a disk block device or directory)',
2217 def populate_data_path_file(self, path, *to_prepare_list):
2218 self.populate_data_path(path, *to_prepare_list)
2220 def populate_data_path(self, path, *to_prepare_list):
2221 if os.path.exists(os.path.join(path, 'magic')):
2222 LOG.debug('Data dir %s already exists', path)
2225 LOG.debug('Preparing osd data dir %s', path)
2227 if self.args.osd_uuid is None:
2228 self.args.osd_uuid = str(uuid.uuid4())
2230 write_one_line(path, 'ceph_fsid', self.args.cluster_uuid)
2231 write_one_line(path, 'fsid', self.args.osd_uuid)
2232 write_one_line(path, 'magic', CEPH_OSD_ONDISK_MAGIC)
2234 for to_prepare in to_prepare_list:
2235 to_prepare.populate_data_path(path)
2237 def prepare(self, *to_prepare_list):
2238 if self.type == self.DEVICE:
2239 self.prepare_device(*to_prepare_list)
2240 elif self.type == self.FILE:
2241 self.prepare_file(*to_prepare_list)
2243 raise Error('unexpected type ', self.type)
2245 def prepare_file(self, *to_prepare_list):
2247 if not os.path.exists(self.args.data):
2248 raise Error('data path for directory does not exist',
2251 if self.args.data_dev:
2252 raise Error('data path is not a block device', self.args.data)
2254 for to_prepare in to_prepare_list:
2255 to_prepare.prepare()
2257 self.populate_data_path_file(self.args.data, *to_prepare_list)
2259 def sanity_checks(self):
2260 if not os.path.exists(self.args.data):
2261 raise Error('data path for device does not exist',
2263 verify_not_in_use(self.args.data, True)
2265 def set_variables(self):
2266 if self.args.fs_type is None:
2267 self.args.fs_type = get_conf(
2268 cluster=self.args.cluster,
2269 variable='osd_mkfs_type',
2271 if self.args.fs_type is None:
2272 self.args.fs_type = get_conf(
2273 cluster=self.args.cluster,
2274 variable='osd_fs_type',
2276 if self.args.fs_type is None:
2277 self.args.fs_type = DEFAULT_FS_TYPE
2279 self.mkfs_args = get_conf(
2280 cluster=self.args.cluster,
2281 variable='osd_mkfs_options_{fstype}'.format(
2282 fstype=self.args.fs_type,
2285 if self.mkfs_args is None:
2286 self.mkfs_args = get_conf(
2287 cluster=self.args.cluster,
2288 variable='osd_fs_mkfs_options_{fstype}'.format(
2289 fstype=self.args.fs_type,
2293 self.mount_options = get_conf(
2294 cluster=self.args.cluster,
2295 variable='osd_mount_options_{fstype}'.format(
2296 fstype=self.args.fs_type,
2299 if self.mount_options is None:
2300 self.mount_options = get_conf(
2301 cluster=self.args.cluster,
2302 variable='osd_fs_mount_options_{fstype}'.format(
2303 fstype=self.args.fs_type,
2307 # remove whitespaces
2308 self.mount_options = "".join(self.mount_options.split())
2310 if self.args.osd_uuid is None:
2311 self.args.osd_uuid = str(uuid.uuid4())
2313 def prepare_device(self, *to_prepare_list):
2314 self.sanity_checks()
2315 self.set_variables()
2316 if self.args.zap_disk is not None:
2319 def create_data_partition(self):
2320 device = Device.factory(self.args.data, self.args)
2321 partition_number = 1
2322 device.create_partition(uuid=self.args.osd_uuid,
2324 num=partition_number,
2325 size=self.get_space_size())
2326 return device.get_partition(partition_number)
2328 def set_data_partition(self):
2329 if is_partition(self.args.data):
2330 LOG.debug('OSD data device %s is a partition',
2332 self.partition = DevicePartition.factory(
2333 path=None, dev=self.args.data, args=self.args)
2334 ptype = partition.get_ptype()
2335 if ptype != ptype_osd:
2336 LOG.warning('incorrect partition UUID: %s, expected %s'
2337 % (ptype, ptype_osd))
2339 LOG.debug('Creating osd partition on %s',
2341 self.partition = self.create_data_partition()
2343 def populate_data_path_device(self, *to_prepare_list):
2344 partition = self.partition
2346 if isinstance(partition, DevicePartitionCrypt):
2355 if self.mkfs_args is not None:
2356 args.extend(self.mkfs_args.split())
2357 if self.args.fs_type == 'xfs':
2358 args.extend(['-f']) # always force
2360 args.extend(MKFS_ARGS.get(self.args.fs_type, []))
2363 partition.get_dev(),
2366 LOG.debug('Creating %s fs on %s',
2367 self.args.fs_type, partition.get_dev())
2368 command_check_call(args)
2369 except subprocess.CalledProcessError as e:
2372 path = mount(dev=partition.get_dev(),
2373 fstype=self.args.fs_type,
2374 options=self.mount_options)
2377 self.populate_data_path(path, *to_prepare_list)
2379 path_set_context(path)
2382 if isinstance(partition, DevicePartitionCrypt):
2385 if not is_partition(self.args.data):
2390 '--typecode=%d:%s' % (partition.get_partition_number(),
2391 partition.ptype_for_name('osd')),
2396 except subprocess.CalledProcessError as e:
2398 update_partition(self.args.data, 'prepared')
2399 command_check_call(['udevadm', 'trigger',
2402 os.path.basename(partition.rawdev)])
2405 class PrepareFilestoreData(PrepareData):
2407 def get_space_size(self):
2408 return 0 # get as much space as possible
2410 def prepare_device(self, *to_prepare_list):
2411 super(PrepareFilestoreData, self).prepare_device(*to_prepare_list)
2412 for to_prepare in to_prepare_list:
2413 to_prepare.prepare()
2414 self.set_data_partition()
2415 self.populate_data_path_device(*to_prepare_list)
2417 def populate_data_path(self, path, *to_prepare_list):
2418 super(PrepareFilestoreData, self).populate_data_path(path,
2420 write_one_line(path, 'type', 'filestore')
2423 class PrepareBluestoreData(PrepareData):
2425 def get_space_size(self):
2428 def prepare_device(self, *to_prepare_list):
2429 super(PrepareBluestoreData, self).prepare_device(*to_prepare_list)
2430 self.set_data_partition()
2431 for to_prepare in to_prepare_list:
2432 to_prepare.prepare()
2433 self.populate_data_path_device(*to_prepare_list)
2435 def populate_data_path(self, path, *to_prepare_list):
2436 super(PrepareBluestoreData, self).populate_data_path(path,
2438 write_one_line(path, 'type', 'bluestore')
2448 monmap = os.path.join(path, 'activate.monmap')
2452 '--cluster', cluster,
2453 '--name', 'client.bootstrap-osd',
2454 '--keyring', keyring,
2455 'mon', 'getmap', '-o', monmap,
2459 osd_type = read_one_line(path, 'type')
2461 if osd_type == 'bluestore':
2465 '--cluster', cluster,
2472 '--keyring', os.path.join(path, 'keyring'),
2473 '--setuser', get_ceph_user(),
2474 '--setgroup', get_ceph_user(),
2481 '--cluster', cluster,
2487 '--osd-journal', os.path.join(path, 'journal'),
2489 '--keyring', os.path.join(path, 'keyring'),
2490 '--setuser', get_ceph_user(),
2491 '--setgroup', get_ceph_group(),
2503 # try dumpling+ cap scheme
2507 '--cluster', cluster,
2508 '--name', 'client.bootstrap-osd',
2509 '--keyring', keyring,
2510 'auth', 'add', 'osd.{osd_id}'.format(osd_id=osd_id),
2511 '-i', os.path.join(path, 'keyring'),
2513 'mon', 'allow profile osd',
2516 except subprocess.CalledProcessError as err:
2517 if err.returncode == errno.EINVAL:
2518 # try old cap scheme
2522 '--cluster', cluster,
2523 '--name', 'client.bootstrap-osd',
2524 '--keyring', keyring,
2525 'auth', 'add', 'osd.{osd_id}'.format(osd_id=osd_id),
2526 '-i', os.path.join(path, 'keyring'),
2535 def get_mount_point(cluster, osd_id):
2536 parent = STATEDIR + '/osd'
2537 return os.path.join(
2539 '{cluster}-{osd_id}'.format(cluster=cluster, osd_id=osd_id),
2551 LOG.debug('Moving mount to final location...')
2552 osd_data = get_mount_point(cluster, osd_id)
2553 maybe_mkdir(osd_data)
2555 # pick best-of-breed mount options based on fs type
2556 if mount_options is None:
2557 mount_options = MOUNT_OPTIONS.get(fstype, '')
2559 # we really want to mount --move, but that is not supported when
2560 # the parent mount is shared, as it is by default on RH, Fedora,
2561 # and probably others. Also, --bind doesn't properly manipulate
2562 # /etc/mtab, which *still* isn't a symlink to /proc/mounts despite
2563 # this being 2013. Instead, mount the original device at the final
2578 '-l', # lazy, in case someone else is peeking at the
2590 LOG.debug('Starting %s osd.%s...', cluster, osd_id)
2592 path = (STATEDIR + '/osd/{cluster}-{osd_id}').format(
2593 cluster=cluster, osd_id=osd_id)
2596 if os.path.exists(os.path.join(path, 'upstart')):
2600 # use emit, not start, because start would fail if the
2601 # instance was already running
2603 # since the daemon starting doesn't guarantee much about
2604 # the service being operational anyway, don't bother
2609 'cluster={cluster}'.format(cluster=cluster),
2610 'id={osd_id}'.format(osd_id=osd_id),
2613 elif os.path.exists(os.path.join(path, 'sysvinit')):
2614 if os.path.exists('/usr/sbin/service'):
2615 svc = '/usr/sbin/service'
2617 svc = '/sbin/service'
2623 '{cluster}'.format(cluster=cluster),
2625 'osd.{osd_id}'.format(osd_id=osd_id),
2628 elif os.path.exists(os.path.join(path, 'systemd')):
2633 'ceph-osd@{osd_id}'.format(osd_id=osd_id),
2640 'ceph-osd@{osd_id}'.format(osd_id=osd_id),
2644 raise Error('{cluster} osd.{osd_id} is not tagged '
2645 'with an init system'.format(
2649 except subprocess.CalledProcessError as e:
2650 raise Error('ceph osd start failed', e)
2657 LOG.debug('Stoping %s osd.%s...', cluster, osd_id)
2659 path = (STATEDIR + '/osd/{cluster}-{osd_id}').format(
2660 cluster=cluster, osd_id=osd_id)
2663 if os.path.exists(os.path.join(path, 'upstart')):
2669 'cluster={cluster}'.format(cluster=cluster),
2670 'id={osd_id}'.format(osd_id=osd_id),
2673 elif os.path.exists(os.path.join(path, 'sysvinit')):
2674 svc = which('service')
2680 '{cluster}'.format(cluster=cluster),
2682 'osd.{osd_id}'.format(osd_id=osd_id),
2685 elif os.path.exists(os.path.join(path, 'systemd')):
2690 'ceph-osd@{osd_id}'.format(osd_id=osd_id),
2697 'ceph-osd@{osd_id}'.format(osd_id=osd_id),
2701 raise Error('{cluster} osd.{osd_id} is not tagged with an init '
2702 ' system'.format(cluster=cluster, osd_id=osd_id))
2703 except subprocess.CalledProcessError as e:
2704 raise Error('ceph osd stop failed', e)
2710 fstype = _check_output(
2713 # we don't want stale cached results
2721 fstype = must_be_one_line(fstype)
2725 def dmcrypt_map(dev, dmcrypt_key_dir):
2726 ptype = get_partition_type(dev)
2727 if ptype in Ptype.get_ready_by_type('plain'):
2729 cryptsetup_parameters = ['--key-size', '256']
2730 elif ptype in Ptype.get_ready_by_type('luks'):
2732 cryptsetup_parameters = []
2734 raise Error('--dmcrypt called for dev %s with invalid ptype %s'
2736 part_uuid = get_partition_uuid(dev)
2737 dmcrypt_key_path = get_dmcrypt_key_path(part_uuid, dmcrypt_key_dir, luks)
2738 return _dmcrypt_map(
2740 keypath=dmcrypt_key_path,
2742 cryptsetup_parameters=cryptsetup_parameters,
2750 activate_key_template,
2758 part_uuid = get_partition_uuid(dev)
2759 dev = dmcrypt_map(dev, dmcrypt_key_dir)
2761 fstype = detect_fstype(dev=dev)
2762 except (subprocess.CalledProcessError,
2764 TooManyLinesError) as e:
2765 raise FilesystemTypeError(
2766 'device {dev}'.format(dev=dev),
2770 # TODO always using mount options from cluster=ceph for
2771 # now; see http://tracker.newdream.net/issues/3253
2772 mount_options = get_conf(
2774 variable='osd_mount_options_{fstype}'.format(
2779 if mount_options is None:
2780 mount_options = get_conf(
2782 variable='osd_fs_mount_options_{fstype}'.format(
2787 # remove whitespaces from mount_options
2788 if mount_options is not None:
2789 mount_options = "".join(mount_options.split())
2791 path = mount(dev=dev, fstype=fstype, options=mount_options)
2793 # check if the disk is deactive, change the journal owner, group
2794 # mode for correct user and group.
2795 if os.path.exists(os.path.join(path, 'deactive')):
2796 # logging to syslog will help us easy to know udev triggered failure
2799 # we need to unmap again because dmcrypt map will create again
2800 # on bootup stage (due to deactivate)
2801 if '/dev/mapper/' in dev:
2802 part_uuid = dev.replace('/dev/mapper/', '')
2803 dmcrypt_unmap(part_uuid)
2804 LOG.info('OSD deactivated! reactivate with: --reactivate')
2805 raise Error('OSD deactivated! reactivate with: --reactivate')
2806 # flag to activate a deactive osd.
2814 (osd_id, cluster) = activate(path, activate_key_template, init)
2816 # Now active successfully
2817 # If we got reactivate and deactive, remove the deactive file
2818 if deactive and reactivate:
2819 os.remove(os.path.join(path, 'deactive'))
2820 LOG.info('Remove `deactive` file.')
2822 # check if the disk is already active, or if something else is already
2826 src_dev = os.stat(path).st_dev
2828 dst_dev = os.stat((STATEDIR + '/osd/{cluster}-{osd_id}').format(
2830 osd_id=osd_id)).st_dev
2831 if src_dev == dst_dev:
2834 parent_dev = os.stat(STATEDIR + '/osd').st_dev
2835 if dst_dev != parent_dev:
2837 elif os.listdir(get_mount_point(cluster, osd_id)):
2838 LOG.info(get_mount_point(cluster, osd_id) +
2839 " is not empty, won't override")
2846 LOG.info('%s osd.%s already mounted in position; unmounting ours.'
2847 % (cluster, osd_id))
2850 raise Error('another %s osd.%s already mounted in position '
2851 '(old/different cluster instance?); unmounting ours.'
2852 % (cluster, osd_id))
2860 mount_options=mount_options,
2862 return (cluster, osd_id)
2865 LOG.error('Failed to activate')
2869 # remove our temp dir
2870 if os.path.exists(path):
2876 activate_key_template,
2880 if not os.path.exists(path):
2882 'directory %s does not exist' % path
2885 (osd_id, cluster) = activate(path, activate_key_template, init)
2887 if init not in (None, 'none'):
2888 canonical = (STATEDIR + '/osd/{cluster}-{osd_id}').format(
2891 if path != canonical:
2892 # symlink it from the proper location
2894 if os.path.lexists(canonical):
2895 old = os.readlink(canonical)
2897 LOG.debug('Removing old symlink %s -> %s', canonical, old)
2899 os.unlink(canonical)
2901 raise Error('unable to remove old symlink', canonical)
2905 LOG.debug('Creating symlink %s -> %s', canonical, path)
2907 os.symlink(path, canonical)
2909 raise Error('unable to create symlink %s -> %s'
2910 % (canonical, path))
2912 return (cluster, osd_id)
2915 def find_cluster_by_uuid(_uuid):
2917 Find a cluster name by searching /etc/ceph/*.conf for a conf file
2918 with the right uuid.
2920 _uuid = _uuid.lower()
2922 if not os.path.exists(SYSCONFDIR):
2924 for conf_file in os.listdir(SYSCONFDIR):
2925 if not conf_file.endswith('.conf'):
2927 cluster = conf_file[:-5]
2929 fsid = get_fsid(cluster)
2931 if e.message != 'getting cluster uuid from configuration failed':
2933 no_fsid.append(cluster)
2937 # be tolerant of /etc/ceph/ceph.conf without an fsid defined.
2938 if len(no_fsid) == 1 and no_fsid[0] == 'ceph':
2939 LOG.warning('No fsid defined in ' + SYSCONFDIR +
2940 '/ceph.conf; using anyway')
2947 activate_key_template,
2951 check_osd_magic(path)
2953 ceph_fsid = read_one_line(path, 'ceph_fsid')
2954 if ceph_fsid is None:
2955 raise Error('No cluster uuid assigned.')
2956 LOG.debug('Cluster uuid is %s', ceph_fsid)
2958 cluster = find_cluster_by_uuid(ceph_fsid)
2960 raise Error('No cluster conf found in ' + SYSCONFDIR +
2961 ' with fsid %s' % ceph_fsid)
2962 LOG.debug('Cluster name is %s', cluster)
2964 fsid = read_one_line(path, 'fsid')
2966 raise Error('No OSD uuid assigned.')
2967 LOG.debug('OSD uuid is %s', fsid)
2969 keyring = activate_key_template.format(cluster=cluster,
2972 osd_id = get_osd_id(path)
2974 osd_id = allocate_osd_id(
2979 write_one_line(path, 'whoami', osd_id)
2980 LOG.debug('OSD id is %s', osd_id)
2982 if not os.path.exists(os.path.join(path, 'ready')):
2983 LOG.debug('Initializing OSD...')
2984 # re-running mkfs is safe, so just run until it completes
2993 if init not in (None, 'none'):
2995 conf_val = get_conf(
2999 if conf_val is not None:
3004 LOG.debug('Marking with init system %s', init)
3005 with file(os.path.join(path, init), 'w'):
3008 # remove markers for others, just in case.
3009 for other in INIT_SYSTEMS:
3012 os.unlink(os.path.join(path, other))
3016 if not os.path.exists(os.path.join(path, 'active')):
3017 LOG.debug('Authorizing OSD key...')
3024 write_one_line(path, 'active', 'ok')
3025 LOG.debug('%s osd.%s data dir is ready at %s', cluster, osd_id, path)
3026 return (osd_id, cluster)
3029 def main_activate(args):
3033 if not os.path.exists(args.path):
3034 raise Error('%s does not exist' % args.path)
3036 if is_suppressed(args.path):
3037 LOG.info('suppressed activate request on %s', args.path)
3040 activate_lock.acquire() # noqa
3042 mode = os.stat(args.path).st_mode
3043 if stat.S_ISBLK(mode):
3044 if (is_partition(args.path) and
3045 (get_partition_type(args.path) ==
3046 PTYPE['mpath']['osd']['ready']) and
3047 not is_mpath(args.path)):
3048 raise Error('%s is not a multipath block device' %
3050 (cluster, osd_id) = mount_activate(
3052 activate_key_template=args.activate_key_template,
3053 init=args.mark_init,
3054 dmcrypt=args.dmcrypt,
3055 dmcrypt_key_dir=args.dmcrypt_key_dir,
3056 reactivate=args.reactivate,
3058 osd_data = get_mount_point(cluster, osd_id)
3060 elif stat.S_ISDIR(mode):
3061 (cluster, osd_id) = activate_dir(
3063 activate_key_template=args.activate_key_template,
3064 init=args.mark_init,
3066 osd_data = args.path
3069 raise Error('%s is not a directory or block device' % args.path)
3071 if (not args.no_start_daemon and args.mark_init == 'none'):
3075 '--cluster={cluster}'.format(cluster=cluster),
3076 '--id={osd_id}'.format(osd_id=osd_id),
3077 '--osd-data={path}'.format(path=osd_data),
3078 '--osd-journal={path}/journal'.format(path=osd_data),
3082 if (not args.no_start_daemon and
3083 args.mark_init not in (None, 'none')):
3091 activate_lock.release() # noqa
3094 ###########################
3096 def _mark_osd_out(cluster, osd_id):
3097 LOG.info('Prepare to mark osd.%d out...', osd_id)
3106 def _check_osd_status(cluster, osd_id):
3108 report the osd status:
3109 00(0) : means OSD OUT AND DOWN
3110 01(1) : means OSD OUT AND UP
3111 10(2) : means OSD IN AND DOWN
3112 11(3) : means OSD IN AND UP
3114 LOG.info("Checking osd id: %s ..." % osd_id)
3117 out, err, ret = command([
3121 '--cluster={cluster}'.format(
3127 out_json = json.loads(out)
3128 for item in out_json[u'osds']:
3129 if item.get(u'osd') == int(osd_id):
3131 if item.get(u'in') is 1:
3133 if item.get(u'up') is 1:
3136 raise Error('Could not osd.%s in osd tree!' % osd_id)
3140 def _remove_osd_directory_files(mounted_path, cluster):
3142 To remove the 'ready', 'active', INIT-specific files.
3144 if os.path.exists(os.path.join(mounted_path, 'ready')):
3145 os.remove(os.path.join(mounted_path, 'ready'))
3146 LOG.info('Remove `ready` file.')
3148 LOG.info('`ready` file is already removed.')
3150 if os.path.exists(os.path.join(mounted_path, 'active')):
3151 os.remove(os.path.join(mounted_path, 'active'))
3152 LOG.info('Remove `active` file.')
3154 LOG.info('`active` file is already removed.')
3156 # Just check `upstart` and `sysvinit` directly if filename is init-spec.
3157 conf_val = get_conf(
3161 if conf_val is not None:
3165 os.remove(os.path.join(mounted_path, init))
3166 LOG.info('Remove `%s` file.', init)
3170 def main_deactivate(args):
3171 activate_lock.acquire() # noqa
3173 main_deactivate_locked(args)
3175 activate_lock.release() # noqa
3178 def main_deactivate_locked(args):
3179 osd_id = args.deactivate_by_id
3183 devices = list_devices()
3185 # list all devices and found we need
3186 for device in devices:
3187 if 'partitions' in device:
3188 for dev_part in device.get('partitions'):
3190 'whoami' in dev_part and
3191 dev_part['whoami'] == osd_id):
3192 target_dev = dev_part
3194 'path' in dev_part and
3195 dev_part['path'] == path):
3196 target_dev = dev_part
3198 raise Error('Cannot find any match device!!')
3200 # set up all we need variable
3201 osd_id = target_dev['whoami']
3202 part_type = target_dev['ptype']
3203 mounted_path = target_dev['mount']
3204 if Ptype.is_dmcrypt(part_type, 'osd'):
3207 # Do not do anything if osd is already down.
3208 status_code = _check_osd_status(args.cluster, osd_id)
3209 if status_code == OSD_STATUS_IN_UP:
3210 if args.mark_out is True:
3211 _mark_osd_out(args.cluster, int(osd_id))
3212 stop_daemon(args.cluster, osd_id)
3213 elif status_code == OSD_STATUS_IN_DOWN:
3214 if args.mark_out is True:
3215 _mark_osd_out(args.cluster, int(osd_id))
3216 LOG.info("OSD already out/down. Do not do anything now.")
3218 elif status_code == OSD_STATUS_OUT_UP:
3219 stop_daemon(args.cluster, osd_id)
3220 elif status_code == OSD_STATUS_OUT_DOWN:
3221 LOG.info("OSD already out/down. Do not do anything now.")
3224 # remove 'ready', 'active', and INIT-specific files.
3225 _remove_osd_directory_files(mounted_path, args.cluster)
3227 # Write deactivate to osd directory!
3228 with open(os.path.join(mounted_path, 'deactive'), 'w'):
3229 path_set_context(os.path.join(mounted_path, 'deactive'))
3231 unmount(mounted_path)
3232 LOG.info("Umount `%s` successfully.", mounted_path)
3235 dmcrypt_unmap(target_dev['uuid'])
3236 for name in Space.NAMES:
3237 if name + '_uuid' in target_dev:
3238 dmcrypt_unmap(target_dev[name + '_uuid'])
3240 ###########################
3243 def _remove_from_crush_map(cluster, osd_id):
3244 LOG.info("Prepare to remove osd.%s from crush map..." % osd_id)
3254 def _delete_osd_auth_key(cluster, osd_id):
3255 LOG.info("Prepare to delete osd.%s cephx key..." % osd_id)
3264 def _deallocate_osd_id(cluster, osd_id):
3265 LOG.info("Prepare to deallocate the osd-id: %s..." % osd_id)
3274 def destroy_lookup_device(args, predicate, description):
3275 devices = list_devices()
3276 for device in devices:
3277 for partition in device.get('partitions', []):
3278 if partition['dmcrypt']:
3279 dmcrypt_path = dmcrypt_map(partition['path'],
3280 args.dmcrypt_key_dir)
3281 list_dev_osd(dmcrypt_path, {}, partition)
3282 dmcrypt_unmap(partition['uuid'])
3283 if predicate(partition):
3285 raise Error('found no device matching ', description)
3288 def main_destroy(args):
3289 osd_id = args.destroy_by_id
3295 if not is_partition(path):
3296 raise Error(path + " must be a partition device")
3297 path = os.path.realpath(path)
3300 target_dev = destroy_lookup_device(
3301 args, lambda x: x.get('path') == path,
3304 target_dev = destroy_lookup_device(
3305 args, lambda x: x.get('whoami') == osd_id,
3306 'osd id ' + str(osd_id))
3308 osd_id = target_dev['whoami']
3309 dev_path = target_dev['path']
3310 if target_dev['ptype'] == PTYPE['mpath']['osd']['ready']:
3311 base_dev = get_partition_base_mpath(dev_path)
3313 base_dev = get_partition_base(dev_path)
3315 # Before osd deactivate, we cannot destroy it
3316 status_code = _check_osd_status(args.cluster, osd_id)
3317 if status_code != OSD_STATUS_OUT_DOWN and \
3318 status_code != OSD_STATUS_IN_DOWN:
3319 raise Error("Could not destroy the active osd. (osd-id: %s)" %
3322 # Remove OSD from crush map
3323 _remove_from_crush_map(args.cluster, osd_id)
3325 # Remove OSD cephx key
3326 _delete_osd_auth_key(args.cluster, osd_id)
3329 _deallocate_osd_id(args.cluster, osd_id)
3331 # we remove the crypt map and device mapper (if dmcrypt is True)
3333 for name in Space.NAMES:
3334 if target_dev.get(name + '_uuid'):
3335 dmcrypt_unmap(target_dev[name + '_uuid'])
3337 # Check zap flag. If we found zap flag, we need to find device for
3338 # destroy this osd data.
3339 if args.zap is True:
3340 # erase the osd data
3341 LOG.info("Prepare to zap the device %s" % base_dev)
3345 def get_space_osd_uuid(name, path):
3346 if not os.path.exists(path):
3347 raise Error('%s does not exist' % path)
3349 mode = os.stat(path).st_mode
3350 if not stat.S_ISBLK(mode):
3351 raise Error('%s is not a block device' % path)
3353 if (is_partition(path) and
3354 get_partition_type(path) in (PTYPE['mpath']['journal']['ready'],
3355 PTYPE['mpath']['block']['ready']) and
3356 not is_mpath(path)):
3357 raise Error('%s is not a multipath block device' %
3361 out = _check_output(
3364 '--get-device-fsid',
3369 except subprocess.CalledProcessError as e:
3371 'failed to get osd uuid/fsid from %s' % name,
3374 value = str(out).split('\n', 1)[0]
3375 LOG.debug('%s %s has OSD UUID %s', name.capitalize(), path, value)
3379 def main_activate_space(name, args):
3380 if not os.path.exists(args.dev):
3381 raise Error('%s does not exist' % args.dev)
3387 activate_lock.acquire() # noqa
3390 dev = dmcrypt_map(args.dev, args.dmcrypt_key_dir)
3393 # FIXME: For an encrypted journal dev, does this return the
3394 # cyphertext or plaintext dev uuid!? Also, if the journal is
3395 # encrypted, is the data partition also always encrypted, or
3396 # are mixed pairs supported!?
3397 osd_uuid = get_space_osd_uuid(name, dev)
3398 path = os.path.join('/dev/disk/by-partuuid/', osd_uuid.lower())
3400 if is_suppressed(path):
3401 LOG.info('suppressed activate request on %s', path)
3404 (cluster, osd_id) = mount_activate(
3406 activate_key_template=args.activate_key_template,
3407 init=args.mark_init,
3408 dmcrypt=args.dmcrypt,
3409 dmcrypt_key_dir=args.dmcrypt_key_dir,
3410 reactivate=args.reactivate,
3419 activate_lock.release() # noqa
3422 ###########################
3425 def main_activate_all(args):
3426 dir = '/dev/disk/by-parttypeuuid'
3427 LOG.debug('Scanning %s', dir)
3428 if not os.path.exists(dir):
3431 for name in os.listdir(dir):
3432 if name.find('.') < 0:
3434 (tag, uuid) = name.split('.')
3436 if tag in Ptype.get_ready_by_name('osd'):
3438 if Ptype.is_dmcrypt(tag, 'osd'):
3439 path = os.path.join('/dev/mapper', uuid)
3441 path = os.path.join(dir, name)
3443 if is_suppressed(path):
3444 LOG.info('suppressed activate request on %s', path)
3447 LOG.info('Activating %s', path)
3448 activate_lock.acquire() # noqa
3450 # never map dmcrypt cyphertext devices
3451 (cluster, osd_id) = mount_activate(
3453 activate_key_template=args.activate_key_template,
3454 init=args.mark_init,
3463 except Exception as e:
3464 print >> sys.stderr, '{prog}: {msg}'.format(
3471 activate_lock.release() # noqa
3473 raise Error('One or more partitions failed to activate')
3476 ###########################
3479 dev = os.path.realpath(dev)
3480 with file('/proc/swaps', 'rb') as proc_swaps:
3481 for line in proc_swaps.readlines()[1:]:
3482 fields = line.split()
3485 swaps_dev = fields[0]
3486 if swaps_dev.startswith('/') and os.path.exists(swaps_dev):
3487 swaps_dev = os.path.realpath(swaps_dev)
3488 if swaps_dev == dev:
3493 def get_oneliner(base, name):
3494 path = os.path.join(base, name)
3495 if os.path.isfile(path):
3496 with open(path, 'r') as _file:
3497 return _file.readline().rstrip()
3501 def get_dev_fs(dev):
3502 fscheck, _, _ = command(
3510 if 'TYPE' in fscheck:
3511 fstype = fscheck.split()[1].split('"')[1]
3517 def split_dev_base_partnum(dev):
3519 partnum = partnum_mpath(dev)
3520 base = get_partition_base_mpath(dev)
3523 partnum = open(os.path.join(b, 'partition')).read().strip()
3524 base = get_partition_base(dev)
3525 return (base, partnum)
3528 def get_partition_type(part):
3529 return get_blkid_partition_info(part, 'ID_PART_ENTRY_TYPE')
3532 def get_partition_uuid(part):
3533 return get_blkid_partition_info(part, 'ID_PART_ENTRY_UUID')
3536 def get_blkid_partition_info(dev, what=None):
3537 out, _, _ = command(
3547 for line in out.splitlines():
3548 (key, value) = line.split('=')
3556 def more_osd_info(path, uuid_map, desc):
3557 desc['ceph_fsid'] = get_oneliner(path, 'ceph_fsid')
3558 if desc['ceph_fsid']:
3559 desc['cluster'] = find_cluster_by_uuid(desc['ceph_fsid'])
3560 desc['whoami'] = get_oneliner(path, 'whoami')
3561 for name in Space.NAMES:
3562 uuid = get_oneliner(path, name + '_uuid')
3564 desc[name + '_uuid'] = uuid.lower()
3565 if desc[name + '_uuid'] in uuid_map:
3566 desc[name + '_dev'] = uuid_map[desc[name + '_uuid']]
3569 def list_dev_osd(dev, uuid_map, desc):
3570 desc['mount'] = is_mounted(dev)
3571 desc['fs_type'] = get_dev_fs(dev)
3572 desc['state'] = 'unprepared'
3574 desc['state'] = 'active'
3575 more_osd_info(desc['mount'], uuid_map, desc)
3576 elif desc['fs_type']:
3578 tpath = mount(dev=dev, fstype=desc['fs_type'], options='')
3581 magic = get_oneliner(tpath, 'magic')
3582 if magic is not None:
3583 desc['magic'] = magic
3584 desc['state'] = 'prepared'
3585 more_osd_info(tpath, uuid_map, desc)
3592 def list_format_more_osd_info_plain(dev):
3594 if dev.get('ceph_fsid'):
3595 if dev.get('cluster'):
3596 desc.append('cluster ' + dev['cluster'])
3598 desc.append('unknown cluster ' + dev['ceph_fsid'])
3599 if dev.get('whoami'):
3600 desc.append('osd.%s' % dev['whoami'])
3601 for name in Space.NAMES:
3602 if dev.get(name + '_dev'):
3603 desc.append(name + ' %s' % dev[name + '_dev'])
3607 def list_format_dev_plain(dev, prefix=''):
3609 if dev['ptype'] == PTYPE['regular']['osd']['ready']:
3610 desc = (['ceph data', dev['state']] +
3611 list_format_more_osd_info_plain(dev))
3612 elif Ptype.is_dmcrypt(dev['ptype'], 'osd'):
3613 dmcrypt = dev['dmcrypt']
3614 if not dmcrypt['holders']:
3615 desc = ['ceph data (dmcrypt %s)' % dmcrypt['type'],
3616 'not currently mapped']
3617 elif len(dmcrypt['holders']) == 1:
3618 holder = get_dev_path(dmcrypt['holders'][0])
3619 desc = ['ceph data (dmcrypt %s %s)' %
3620 (dmcrypt['type'], holder)]
3621 desc += list_format_more_osd_info_plain(dev)
3623 desc = ['ceph data (dmcrypt %s)' % dmcrypt['type'],
3624 'holders: ' + ','.join(dmcrypt['holders'])]
3625 elif Ptype.is_regular_space(dev['ptype']):
3626 name = Ptype.space_ptype_to_name(dev['ptype'])
3627 desc.append('ceph ' + name)
3628 if dev.get(name + '_for'):
3629 desc.append('for %s' % dev[name + '_for'])
3630 elif Ptype.is_dmcrypt_space(dev['ptype']):
3631 name = Ptype.space_ptype_to_name(dev['ptype'])
3632 dmcrypt = dev['dmcrypt']
3633 if dmcrypt['holders'] and len(dmcrypt['holders']) == 1:
3634 holder = get_dev_path(dmcrypt['holders'][0])
3635 desc = ['ceph ' + name + ' (dmcrypt %s %s)' %
3636 (dmcrypt['type'], holder)]
3638 desc = ['ceph ' + name + ' (dmcrypt %s)' % dmcrypt['type']]
3639 if dev.get(name + '_for'):
3640 desc.append('for %s' % dev[name + '_for'])
3642 desc.append(dev['type'])
3643 if dev.get('fs_type'):
3644 desc.append(dev['fs_type'])
3645 elif dev.get('ptype'):
3646 desc.append(dev['ptype'])
3647 if dev.get('mount'):
3648 desc.append('mounted on %s' % dev['mount'])
3649 return '%s%s %s' % (prefix, dev['path'], ', '.join(desc))
3652 def list_format_plain(devices):
3654 for device in devices:
3655 if device.get('partitions'):
3656 lines.append('%s :' % device['path'])
3657 for p in sorted(device['partitions']):
3658 lines.append(list_format_dev_plain(dev=p,
3661 lines.append(list_format_dev_plain(dev=device,
3663 return "\n".join(lines)
3666 def list_dev(dev, uuid_map, space_map):
3672 info['is_partition'] = is_partition(dev)
3673 if info['is_partition']:
3674 ptype = get_partition_type(dev)
3675 info['uuid'] = get_partition_uuid(dev)
3678 info['ptype'] = ptype
3679 LOG.info("list_dev(dev = " + dev + ", ptype = " + str(ptype) + ")")
3680 if ptype in (PTYPE['regular']['osd']['ready'],
3681 PTYPE['mpath']['osd']['ready']):
3682 info['type'] = 'data'
3683 if ptype == PTYPE['mpath']['osd']['ready']:
3684 info['multipath'] = True
3685 list_dev_osd(dev, uuid_map, info)
3686 elif ptype == PTYPE['plain']['osd']['ready']:
3687 holders = is_held(dev)
3688 info['type'] = 'data'
3689 info['dmcrypt']['holders'] = holders
3690 info['dmcrypt']['type'] = 'plain'
3691 if len(holders) == 1:
3692 list_dev_osd(get_dev_path(holders[0]), uuid_map, info)
3693 elif ptype == PTYPE['luks']['osd']['ready']:
3694 holders = is_held(dev)
3695 info['type'] = 'data'
3696 info['dmcrypt']['holders'] = holders
3697 info['dmcrypt']['type'] = 'LUKS'
3698 if len(holders) == 1:
3699 list_dev_osd(get_dev_path(holders[0]), uuid_map, info)
3700 elif Ptype.is_regular_space(ptype) or Ptype.is_mpath_space(ptype):
3701 name = Ptype.space_ptype_to_name(ptype)
3703 if ptype == PTYPE['mpath'][name]['ready']:
3704 info['multipath'] = True
3705 if info.get('uuid') in space_map:
3706 info[name + '_for'] = space_map[info['uuid']]
3707 elif Ptype.is_plain_space(ptype):
3708 name = Ptype.space_ptype_to_name(ptype)
3709 holders = is_held(dev)
3711 info['dmcrypt']['type'] = 'plain'
3712 info['dmcrypt']['holders'] = holders
3713 if info.get('uuid') in space_map:
3714 info[name + '_for'] = space_map[info['uuid']]
3715 elif Ptype.is_luks_space(ptype):
3716 name = Ptype.space_ptype_to_name(ptype)
3717 holders = is_held(dev)
3719 info['dmcrypt']['type'] = 'LUKS'
3720 info['dmcrypt']['holders'] = holders
3721 if info.get('uuid') in space_map:
3722 info[name + '_for'] = space_map[info['uuid']]
3724 path = is_mounted(dev)
3725 fs_type = get_dev_fs(dev)
3727 info['type'] = 'swap'
3729 info['type'] = 'other'
3731 info['fs_type'] = fs_type
3733 info['mount'] = path
3739 partmap = list_all_partitions()
3743 for base, parts in sorted(partmap.iteritems()):
3745 dev = get_dev_path(p)
3746 part_uuid = get_partition_uuid(dev)
3748 uuid_map[part_uuid] = dev
3749 ptype = get_partition_type(dev)
3750 LOG.debug("main_list: " + dev +
3751 " ptype = " + str(ptype) +
3752 " uuid = " + str(part_uuid))
3753 if ptype in Ptype.get_ready_by_name('osd'):
3754 if Ptype.is_dmcrypt(ptype, 'osd'):
3755 holders = is_held(dev)
3756 if len(holders) != 1:
3758 dev_to_mount = get_dev_path(holders[0])
3762 fs_type = get_dev_fs(dev_to_mount)
3763 if fs_type is not None:
3765 tpath = mount(dev=dev_to_mount,
3766 fstype=fs_type, options='')
3768 for name in Space.NAMES:
3769 space_uuid = get_oneliner(tpath,
3772 space_map[space_uuid.lower()] = dev
3778 LOG.debug("main_list: " + str(partmap) + ", uuid_map = " +
3779 str(uuid_map) + ", space_map = " + str(space_map))
3782 for base, parts in sorted(partmap.iteritems()):
3784 disk = {'path': get_dev_path(base)}
3786 for p in sorted(parts):
3787 partitions.append(list_dev(get_dev_path(p),
3790 disk['partitions'] = partitions
3791 devices.append(disk)
3793 device = list_dev(get_dev_path(base), uuid_map, space_map)
3794 device['path'] = get_dev_path(base)
3795 devices.append(device)
3796 LOG.debug("list_devices: " + str(devices))
3800 def main_list(args):
3801 devices = list_devices()
3804 for path in args.path:
3805 if os.path.exists(path):
3806 paths.append(os.path.realpath(path))
3809 selected_devices = []
3810 for device in devices:
3812 if re.search(path + '$', device['path']):
3813 selected_devices.append(device)
3815 selected_devices = devices
3816 if args.format == 'json':
3817 print json.dumps(selected_devices)
3819 output = list_format_plain(selected_devices)
3824 ###########################
3826 # Mark devices that we want to suppress activates on with a
3829 # /var/lib/ceph/tmp/suppress-activate.sdb
3831 # where the last bit is the sanitized device name (/dev/X without the
3832 # /dev/ prefix) and the is_suppress() check matches a prefix. That
3833 # means suppressing sdb will stop activate on sdb1, sdb2, etc.
3836 def is_suppressed(path):
3837 disk = os.path.realpath(path)
3839 if (not disk.startswith('/dev/') or
3840 not stat.S_ISBLK(os.lstat(disk).st_mode)):
3842 base = get_dev_name(disk)
3844 if os.path.exists(SUPPRESS_PREFIX + base): # noqa
3851 def set_suppress(path):
3852 disk = os.path.realpath(path)
3853 if not os.path.exists(disk):
3854 raise Error('does not exist', path)
3855 if not stat.S_ISBLK(os.lstat(path).st_mode):
3856 raise Error('not a block device', path)
3857 base = get_dev_name(disk)
3859 with file(SUPPRESS_PREFIX + base, 'w') as f: # noqa
3861 LOG.info('set suppress flag on %s', base)
3864 def unset_suppress(path):
3865 disk = os.path.realpath(path)
3866 if not os.path.exists(disk):
3867 raise Error('does not exist', path)
3868 if not stat.S_ISBLK(os.lstat(path).st_mode):
3869 raise Error('not a block device', path)
3870 assert disk.startswith('/dev/')
3871 base = get_dev_name(disk)
3873 fn = SUPPRESS_PREFIX + base # noqa
3874 if not os.path.exists(fn):
3875 raise Error('not marked as suppressed', path)
3879 LOG.info('unset suppress flag on %s', base)
3880 except OSError as e:
3881 raise Error('failed to unsuppress', e)
3884 def main_suppress(args):
3885 set_suppress(args.path)
3888 def main_unsuppress(args):
3889 unset_suppress(args.path)
3893 for dev in args.dev:
3897 def main_trigger(args):
3898 LOG.debug("main_trigger: " + str(args))
3899 if is_systemd() and not args.sync:
3900 # http://www.freedesktop.org/software/systemd/man/systemd-escape.html
3901 escaped_dev = args.dev[1:].replace('-', '\\x2d')
3902 service = 'ceph-disk@{dev}.service'.format(dev=escaped_dev)
3903 LOG.info('systemd detected, triggering %s' % service)
3913 if is_upstart() and not args.sync:
3914 LOG.info('upstart detected, triggering ceph-disk task')
3920 'dev={dev}'.format(dev=args.dev),
3921 'pid={pid}'.format(pid=os.getpid()),
3926 parttype = get_partition_type(args.dev)
3927 partid = get_partition_uuid(args.dev)
3929 LOG.info('trigger {dev} parttype {parttype} uuid {partid}'.format(
3935 if parttype in (PTYPE['regular']['osd']['ready'],
3936 PTYPE['mpath']['osd']['ready']):
3944 elif parttype in (PTYPE['regular']['journal']['ready'],
3945 PTYPE['mpath']['journal']['ready']):
3954 # journals are easy: map, chown, activate-journal
3955 elif parttype == PTYPE['plain']['journal']['ready']:
3960 '/etc/ceph/dmcrypt-keys/{partid}'.format(partid=partid),
3968 newdev = '/dev/mapper/' + partid
3970 while not os.path.exists(newdev) and count <= 10:
3982 '/usr/sbin/ceph-disk',
3987 elif parttype == PTYPE['luks']['journal']['ready']:
3992 '/etc/ceph/dmcrypt-keys/{partid}.luks.key'.format(
3999 newdev = '/dev/mapper/' + partid
4001 while not os.path.exists(newdev) and count <= 10:
4013 '/usr/sbin/ceph-disk',
4019 elif parttype in (PTYPE['regular']['block']['ready'],
4020 PTYPE['mpath']['block']['ready']):
4029 # blocks are easy: map, chown, activate-block
4030 elif parttype == PTYPE['plain']['block']['ready']:
4035 '/etc/ceph/dmcrypt-keys/{partid}'.format(partid=partid),
4043 newdev = '/dev/mapper/' + partid
4045 while not os.path.exists(newdev) and count <= 10:
4057 '/usr/sbin/ceph-disk',
4062 elif parttype == PTYPE['luks']['block']['ready']:
4067 '/etc/ceph/dmcrypt-keys/{partid}.luks.key'.format(
4074 newdev = '/dev/mapper/' + partid
4076 while not os.path.exists(newdev) and count <= 10:
4088 '/usr/sbin/ceph-disk',
4094 # osd data: map, activate
4095 elif parttype == PTYPE['plain']['osd']['ready']:
4100 '/etc/ceph/dmcrypt-keys/{partid}'.format(partid=partid),
4108 newdev = '/dev/mapper/' + partid
4110 while not os.path.exists(newdev) and count <= 10:
4115 '/usr/sbin/ceph-disk',
4121 elif parttype == PTYPE['luks']['osd']['ready']:
4126 '/etc/ceph/dmcrypt-keys/{partid}.luks.key'.format(
4133 newdev = '/dev/mapper/' + partid
4135 while not os.path.exists(newdev) and count <= 10:
4140 '/usr/sbin/ceph-disk',
4147 raise Error('unrecognized partition type %s' % parttype)
4150 def setup_statedir(dir):
4151 # XXX The following use of globals makes linting
4152 # really hard. Global state in Python is iffy and
4153 # should be avoided.
4157 if not os.path.exists(STATEDIR):
4159 if not os.path.exists(STATEDIR + "/tmp"):
4160 os.mkdir(STATEDIR + "/tmp")
4163 prepare_lock = filelock(STATEDIR + '/tmp/ceph-disk.prepare.lock')
4165 global activate_lock
4166 activate_lock = filelock(STATEDIR + '/tmp/ceph-disk.activate.lock')
4168 global SUPPRESS_PREFIX
4169 SUPPRESS_PREFIX = STATEDIR + '/tmp/suppress-activate.'
4172 def setup_sysconfdir(dir):
4177 def parse_args(argv):
4178 parser = argparse.ArgumentParser(
4181 parser.add_argument(
4183 action='store_true', default=None,
4184 help='be more verbose',
4186 parser.add_argument(
4188 action='store_true', default=None,
4189 help='log to stdout',
4191 parser.add_argument(
4192 '--prepend-to-path',
4195 help=('prepend PATH to $PATH for backward compatibility '
4196 '(default /usr/bin)'),
4198 parser.add_argument(
4201 default='/var/lib/ceph',
4202 help=('directory in which ceph state is preserved '
4203 '(default /var/lib/ceph)'),
4205 parser.add_argument(
4208 default='/etc/ceph',
4209 help=('directory in which ceph configuration files are found '
4210 '(default /etc/ceph)'),
4212 parser.add_argument(
4216 help='use the given user for subprocesses, rather than ceph or root'
4218 parser.add_argument(
4222 help='use the given group for subprocesses, rather than ceph or root'
4224 parser.set_defaults(
4225 # we want to hold on to this, for later
4229 subparsers = parser.add_subparsers(
4230 title='subcommands',
4231 description='valid subcommands',
4232 help='sub-command help',
4235 Prepare.set_subparser(subparsers)
4236 make_activate_parser(subparsers)
4237 make_activate_block_parser(subparsers)
4238 make_activate_journal_parser(subparsers)
4239 make_activate_all_parser(subparsers)
4240 make_list_parser(subparsers)
4241 make_suppress_parser(subparsers)
4242 make_deactivate_parser(subparsers)
4243 make_destroy_parser(subparsers)
4244 make_zap_parser(subparsers)
4245 make_trigger_parser(subparsers)
4247 args = parser.parse_args(argv)
4251 def make_trigger_parser(subparsers):
4252 trigger_parser = subparsers.add_parser(
4254 help='Trigger an event (caled by udev)')
4255 trigger_parser.add_argument(
4259 trigger_parser.add_argument(
4261 action='store_true', default=None,
4262 help=('do operation synchronously; do not trigger systemd'),
4264 trigger_parser.set_defaults(
4267 return trigger_parser
4270 def make_activate_parser(subparsers):
4271 activate_parser = subparsers.add_parser(
4273 help='Activate a Ceph OSD')
4274 activate_parser.add_argument(
4276 action='store_true', default=None,
4277 help='mount a block device [deprecated, ignored]',
4279 activate_parser.add_argument(
4282 help='bootstrap-osd keyring path template (%(default)s)',
4283 dest='activate_key_template',
4285 activate_parser.add_argument(
4287 metavar='INITSYSTEM',
4288 help='init system to manage this dir',
4290 choices=INIT_SYSTEMS,
4292 activate_parser.add_argument(
4293 '--no-start-daemon',
4294 action='store_true', default=None,
4295 help='do not start the daemon',
4297 activate_parser.add_argument(
4300 help='path to block device or directory',
4302 activate_parser.add_argument(
4304 action='store_true', default=None,
4305 help='map DATA and/or JOURNAL devices with dm-crypt',
4307 activate_parser.add_argument(
4308 '--dmcrypt-key-dir',
4310 default='/etc/ceph/dmcrypt-keys',
4311 help='directory where dm-crypt keys are stored',
4313 activate_parser.add_argument(
4315 action='store_true', default=False,
4316 help='activate the deactived OSD',
4318 activate_parser.set_defaults(
4319 activate_key_template='{statedir}/bootstrap-osd/{cluster}.keyring',
4322 return activate_parser
4325 def make_activate_block_parser(subparsers):
4326 return make_activate_space_parser('block', subparsers)
4329 def make_activate_journal_parser(subparsers):
4330 return make_activate_space_parser('journal', subparsers)
4333 def make_activate_space_parser(name, subparsers):
4334 activate_space_parser = subparsers.add_parser(
4335 'activate-%s' % name,
4336 help='Activate an OSD via its %s device' % name)
4337 activate_space_parser.add_argument(
4340 help='path to %s block device' % name,
4342 activate_space_parser.add_argument(
4345 help='bootstrap-osd keyring path template (%(default)s)',
4346 dest='activate_key_template',
4348 activate_space_parser.add_argument(
4350 metavar='INITSYSTEM',
4351 help='init system to manage this dir',
4353 choices=INIT_SYSTEMS,
4355 activate_space_parser.add_argument(
4357 action='store_true', default=None,
4358 help=('map data and/or auxiliariy (journal, etc.) '
4359 'devices with dm-crypt'),
4361 activate_space_parser.add_argument(
4362 '--dmcrypt-key-dir',
4364 default='/etc/ceph/dmcrypt-keys',
4365 help='directory where dm-crypt keys are stored',
4367 activate_space_parser.add_argument(
4369 action='store_true', default=False,
4370 help='activate the deactived OSD',
4372 activate_space_parser.set_defaults(
4373 activate_key_template='{statedir}/bootstrap-osd/{cluster}.keyring',
4374 func=lambda args: main_activate_space(name, args),
4376 return activate_space_parser
4379 def make_activate_all_parser(subparsers):
4380 activate_all_parser = subparsers.add_parser(
4382 help='Activate all tagged OSD partitions')
4383 activate_all_parser.add_argument(
4386 help='bootstrap-osd keyring path template (%(default)s)',
4387 dest='activate_key_template',
4389 activate_all_parser.add_argument(
4391 metavar='INITSYSTEM',
4392 help='init system to manage this dir',
4394 choices=INIT_SYSTEMS,
4396 activate_all_parser.set_defaults(
4397 activate_key_template='{statedir}/bootstrap-osd/{cluster}.keyring',
4398 func=main_activate_all,
4400 return activate_all_parser
4403 def make_list_parser(subparsers):
4404 list_parser = subparsers.add_parser(
4406 help='List disks, partitions, and Ceph OSDs')
4407 list_parser.add_argument(
4409 help='output format',
4411 choices=['json', 'plain'],
4413 list_parser.add_argument(
4417 help='path to block devices, relative to /sys/block',
4419 list_parser.set_defaults(
4425 def make_suppress_parser(subparsers):
4426 suppress_parser = subparsers.add_parser(
4427 'suppress-activate',
4428 help='Suppress activate on a device (prefix)')
4429 suppress_parser.add_argument(
4432 help='path to block device or directory',
4434 suppress_parser.set_defaults(
4438 unsuppress_parser = subparsers.add_parser(
4439 'unsuppress-activate',
4440 help='Stop suppressing activate on a device (prefix)')
4441 unsuppress_parser.add_argument(
4444 help='path to block device or directory',
4446 unsuppress_parser.set_defaults(
4447 func=main_unsuppress,
4449 return suppress_parser
4452 def make_deactivate_parser(subparsers):
4453 deactivate_parser = subparsers.add_parser(
4455 help='Deactivate a Ceph OSD')
4456 deactivate_parser.add_argument(
4460 help='cluster name to assign this disk to',
4462 deactivate_parser.add_argument(
4466 help='path to block device or directory',
4468 deactivate_parser.add_argument(
4469 '--deactivate-by-id',
4471 help='ID of OSD to deactive'
4473 deactivate_parser.add_argument(
4475 action='store_true', default=False,
4476 help='option to mark the osd out',
4478 deactivate_parser.set_defaults(
4479 func=main_deactivate,
4483 def make_destroy_parser(subparsers):
4484 destroy_parser = subparsers.add_parser(
4486 help='Destroy a Ceph OSD')
4487 destroy_parser.add_argument(
4491 help='cluster name to assign this disk to',
4493 destroy_parser.add_argument(
4497 help='path to block device or directory',
4499 destroy_parser.add_argument(
4502 help='ID of OSD to destroy'
4504 destroy_parser.add_argument(
4505 '--dmcrypt-key-dir',
4507 default='/etc/ceph/dmcrypt-keys',
4508 help=('directory where dm-crypt keys are stored '
4509 '(If you don\'t know how it work, '
4510 'dont use it. we have default value)'),
4512 destroy_parser.add_argument(
4514 action='store_true', default=False,
4515 help='option to erase data and partition',
4517 destroy_parser.set_defaults(
4522 def make_zap_parser(subparsers):
4523 zap_parser = subparsers.add_parser(
4525 help='Zap/erase/destroy a device\'s partition table (and contents)')
4526 zap_parser.add_argument(
4530 help='path to block device',
4532 zap_parser.set_defaults(
4539 args = parse_args(argv)
4541 setup_logging(args.verbose, args.log_stdout)
4543 if args.prepend_to_path != '':
4544 path = os.environ.get('PATH', os.defpath)
4545 os.environ['PATH'] = args.prepend_to_path + ":" + path
4547 setup_statedir(args.statedir)
4548 setup_sysconfdir(args.sysconfdir)
4550 global CEPH_PREF_USER
4551 CEPH_PREF_USER = args.setuser
4552 global CEPH_PREF_GROUP
4553 CEPH_PREF_GROUP = args.setgroup
4558 main_catch(args.func, args)
4561 def setup_logging(verbose, log_stdout):
4562 loglevel = logging.WARNING
4564 loglevel = logging.DEBUG
4567 ch = logging.StreamHandler(stream=sys.stdout)
4568 ch.setLevel(loglevel)
4569 formatter = logging.Formatter('%(filename)s: %(message)s')
4570 ch.setFormatter(formatter)
4572 LOG.setLevel(loglevel)
4574 logging.basicConfig(
4579 def main_catch(func, args):
4586 '{prog}: {msg}'.format(
4592 except CephDiskException as error:
4593 exc_name = error.__class__.__name__
4595 '{prog} {exc_name}: {msg}'.format(
4606 if __name__ == '__main__':