2 API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention
3 that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides
4 set of utilities for interacting with LVM.
6 from ceph_volume import process
7 from ceph_volume.exceptions import MultipleLVsError, MultipleVGsError, MultiplePVsError
10 def _output_parser(output, fields):
12 Newer versions of LVM allow ``--reportformat=json``, but older versions,
13 like the one included in Xenial do not. LVM has the ability to filter and
14 format its output so we assume the output will be in a format this parser
15 can handle (using ',' as a delimiter)
17 :param fields: A string, possibly using ',' to group many items, as it
18 would be used on the CLI
19 :param output: The CLI output from the LVM call
21 field_items = fields.split(',')
24 # clear the leading/trailing whitespace
27 # remove the extra '"' in each field
28 line = line.replace('"', '')
30 # prevent moving forward with empty contents
34 # spliting on ';' because that is what the lvm call uses as
36 output_items = [i.strip() for i in line.split(';')]
37 # map the output to the fiels
39 dict(zip(field_items, output_items))
45 def parse_tags(lv_tags):
47 Return a dictionary mapping of all the tags associated with
48 a Volume from the comma-separated tags coming from the LVM API
52 "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0"
54 For the above example, the expected return value would be::
57 "ceph.osd_fsid": "aaa-fff-bbbb",
64 tags = lv_tags.split(',')
65 for tag_assignment in tags:
66 if not tag_assignment.startswith('ceph.'):
68 key, value = tag_assignment.split('=', 1)
69 tag_mapping[key] = value
76 Return the list of group volumes available in the system using flags to
77 include common metadata associated with them
79 Command and sample delimeted output, should look like::
81 $ vgs --noheadings --separator=';' \
82 -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free
83 ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m
84 osd_vg;3;1;0;wz--n-;29.21g;9.21g
87 fields = 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free'
88 stdout, stderr, returncode = process.call(
89 ['vgs', '--noheadings', '--separator=";"', '-o', fields]
91 return _output_parser(stdout, fields)
96 Return the list of logical volumes available in the system using flags to include common
97 metadata associated with them
99 Command and delimeted output, should look like::
101 $ lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name
102 ;/dev/ubuntubox-vg/root;root;ubuntubox-vg
103 ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg
106 fields = 'lv_tags,lv_path,lv_name,vg_name,lv_uuid'
107 stdout, stderr, returncode = process.call(
108 ['lvs', '--noheadings', '--separator=";"', '-o', fields]
110 return _output_parser(stdout, fields)
115 Return the list of physical volumes configured for lvm and available in the
116 system using flags to include common metadata associated with them like the uuid
118 Command and delimeted output, should look like::
120 $ pvs --noheadings --separator=';' -o pv_name,pv_tags,pv_uuid
122 /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D
125 fields = 'pv_name,pv_tags,pv_uuid'
127 # note the use of `pvs -a` which will return every physical volume including
128 # ones that have not been initialized as "pv" by LVM
129 stdout, stderr, returncode = process.call(
130 ['pvs', '-a', '--no-heading', '--separator=";"', '-o', fields]
133 return _output_parser(stdout, fields)
136 def get_lv_from_argument(argument):
138 Helper proxy function that consumes a possible logical volume passed in from the CLI
139 in the form of `vg/lv`, but with some validation so that an argument that is a full
140 path to a device can be ignored
142 if argument.startswith('/'):
143 lv = get_lv(lv_path=argument)
146 vg_name, lv_name = argument.split('/')
147 except (ValueError, AttributeError):
149 return get_lv(lv_name=lv_name, vg_name=vg_name)
152 def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
154 Return a matching lv for the current system, requiring ``lv_name``,
155 ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv
158 It is useful to use ``tags`` when trying to find a specific logical volume,
159 but it can also lead to multiple lvs being found, since a lot of metadata
160 is shared between lvs of a distinct OSD.
162 if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
166 lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_uuid=lv_uuid,
171 def get_pv(pv_name=None, pv_uuid=None, pv_tags=None):
173 Return a matching pv (physical volume) for the current system, requiring
174 ``pv_name``, ``pv_uuid``, or ``pv_tags``. Raises an error if more than one
177 if not any([pv_name, pv_uuid, pv_tags]):
180 return pvs.get(pv_name=pv_name, pv_uuid=pv_uuid, pv_tags=pv_tags)
183 def create_pv(device):
185 Create a physical volume from a device, useful when devices need to be later mapped
192 '--yes', # answer yes to any prompts
197 def create_vg(name, *devices):
199 Create a Volume Group. Command looks like::
201 vgcreate --force --yes group_name device
203 Once created the volume group is returned as a ``VolumeGroup`` object
209 name] + list(devices)
212 vg = get_vg(vg_name=name)
218 Removes a logical volume given it's absolute path.
220 Will return True if the lv is successfully removed or
221 raises a RuntimeError if the removal fails.
223 stdout, stderr, returncode = process.call(
231 terminal_verbose=True,
234 raise RuntimeError("Unable to remove %s".format(path))
238 def create_lv(name, group, size=None, tags=None):
240 Create a Logical Volume in a Volume Group. Command looks like::
242 lvcreate -L 50G -n gfslv vg0
244 ``name``, ``group``, are required. If ``size`` is provided it must follow
245 lvm's size notation (like 1G, or 20M). Tags are an optional dictionary and is expected to
246 conform to the convention of prefixing them with "ceph." like::
248 {"ceph.block_device": "/dev/ceph/osd-1"}
250 # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations
252 'journal': 'ceph.journal_device',
253 'data': 'ceph.data_device',
254 'block': 'ceph.block_device',
255 'wal': 'ceph.wal_device',
256 'db': 'ceph.db_device',
257 'lockbox': 'ceph.lockbox_device', # XXX might not ever need this lockbox sorcery
267 # create the lv with all the space available, this is needed because the
268 # system call is different for LVM
278 lv = get_lv(lv_name=name, vg_name=group)
281 # when creating a distinct type, the caller doesn't know what the path will
282 # be so this function will set it after creation using the mapping
283 path_tag = type_path_tag.get(tags.get('ceph.type'))
286 {path_tag: lv.lv_path}
291 def get_vg(vg_name=None, vg_tags=None):
293 Return a matching vg for the current system, requires ``vg_name`` or
294 ``tags``. Raises an error if more than one vg is found.
296 It is useful to use ``tags`` when trying to find a specific volume group,
297 but it can also lead to multiple vgs being found.
299 if not any([vg_name, vg_tags]):
302 return vgs.get(vg_name=vg_name, vg_tags=vg_tags)
305 class VolumeGroups(list):
307 A list of all known volume groups for the current system, with the ability
308 to filter them via keyword arguments.
315 # get all the vgs in the current system
316 for vg_item in get_api_vgs():
317 self.append(VolumeGroup(**vg_item))
321 Deplete all the items in the list, used internally only so that we can
322 dynamically allocate the items when filtering without the concern of
323 messing up the contents
327 def _filter(self, vg_name=None, vg_tags=None):
329 The actual method that filters using a new list. Useful so that other
330 methods that do not want to alter the contents of the list (e.g.
331 ``self.find``) can operate safely.
333 .. note:: ``vg_tags`` is not yet implemented
335 filtered = [i for i in self]
337 filtered = [i for i in filtered if i.vg_name == vg_name]
339 # at this point, `filtered` has either all the volumes in self or is an
340 # actual filtered list if any filters were applied
343 for volume in filtered:
344 matches = all(volume.tags.get(k) == str(v) for k, v in vg_tags.items())
346 tag_filtered.append(volume)
351 def filter(self, vg_name=None, vg_tags=None):
353 Filter out groups on top level attributes like ``vg_name`` or by
354 ``vg_tags`` where a dict is required. For example, to find a Ceph group
355 with dmcache as the type, the filter would look like::
357 vg_tags={'ceph.type': 'dmcache'}
359 .. warning:: These tags are not documented because they are currently
360 unused, but are here to maintain API consistency
362 if not any([vg_name, vg_tags]):
363 raise TypeError('.filter() requires vg_name or vg_tags (none given)')
364 # first find the filtered volumes with the values in self
365 filtered_groups = self._filter(
369 # then purge everything
371 # and add the filtered items
372 self.extend(filtered_groups)
374 def get(self, vg_name=None, vg_tags=None):
376 This is a bit expensive, since it will try to filter out all the
377 matching items in the list, filter them out applying anything that was
378 added and return the matching item.
380 This method does *not* alter the list, and it will raise an error if
381 multiple VGs are matched
383 It is useful to use ``tags`` when trying to find a specific volume group,
384 but it can also lead to multiple vgs being found (although unlikely)
386 if not any([vg_name, vg_tags]):
395 # this is probably never going to happen, but it is here to keep
396 # the API code consistent
397 raise MultipleVGsError(vg_name)
403 A list of all known (logical) volumes for the current system, with the ability
404 to filter them via keyword arguments.
411 # get all the lvs in the current system
412 for lv_item in get_api_lvs():
413 self.append(Volume(**lv_item))
417 Deplete all the items in the list, used internally only so that we can
418 dynamically allocate the items when filtering without the concern of
419 messing up the contents
423 def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
425 The actual method that filters using a new list. Useful so that other
426 methods that do not want to alter the contents of the list (e.g.
427 ``self.find``) can operate safely.
429 filtered = [i for i in self]
431 filtered = [i for i in filtered if i.lv_name == lv_name]
434 filtered = [i for i in filtered if i.vg_name == vg_name]
437 filtered = [i for i in filtered if i.lv_uuid == lv_uuid]
440 filtered = [i for i in filtered if i.lv_path == lv_path]
442 # at this point, `filtered` has either all the volumes in self or is an
443 # actual filtered list if any filters were applied
446 for volume in filtered:
447 # all the tags we got need to match on the volume
448 matches = all(volume.tags.get(k) == str(v) for k, v in lv_tags.items())
450 tag_filtered.append(volume)
455 def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
457 Filter out volumes on top level attributes like ``lv_name`` or by
458 ``lv_tags`` where a dict is required. For example, to find a volume
459 that has an OSD ID of 0, the filter would look like::
461 lv_tags={'ceph.osd_id': '0'}
464 if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
465 raise TypeError('.filter() requires lv_name, vg_name, lv_path, lv_uuid, or tags (none given)')
466 # first find the filtered volumes with the values in self
467 filtered_volumes = self._filter(
474 # then purge everything
476 # and add the filtered items
477 self.extend(filtered_volumes)
479 def get(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
481 This is a bit expensive, since it will try to filter out all the
482 matching items in the list, filter them out applying anything that was
483 added and return the matching item.
485 This method does *not* alter the list, and it will raise an error if
486 multiple LVs are matched
488 It is useful to use ``tags`` when trying to find a specific logical volume,
489 but it can also lead to multiple lvs being found, since a lot of metadata
490 is shared between lvs of a distinct OSD.
492 if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
504 raise MultipleLVsError(lv_name, lv_path)
508 class PVolumes(list):
510 A list of all known (physical) volumes for the current system, with the ability
511 to filter them via keyword arguments.
518 # get all the pvs in the current system
519 for pv_item in get_api_pvs():
520 self.append(PVolume(**pv_item))
524 Deplete all the items in the list, used internally only so that we can
525 dynamically allocate the items when filtering without the concern of
526 messing up the contents
530 def _filter(self, pv_name=None, pv_uuid=None, pv_tags=None):
532 The actual method that filters using a new list. Useful so that other
533 methods that do not want to alter the contents of the list (e.g.
534 ``self.find``) can operate safely.
536 filtered = [i for i in self]
538 filtered = [i for i in filtered if i.pv_name == pv_name]
541 filtered = [i for i in filtered if i.pv_uuid == pv_uuid]
543 # at this point, `filtered` has either all the physical volumes in self
544 # or is an actual filtered list if any filters were applied
547 for pvolume in filtered:
548 matches = all(pvolume.tags.get(k) == str(v) for k, v in pv_tags.items())
550 tag_filtered.append(pvolume)
551 # return the tag_filtered pvolumes here, the `filtered` list is no
557 def filter(self, pv_name=None, pv_uuid=None, pv_tags=None):
559 Filter out volumes on top level attributes like ``pv_name`` or by
560 ``pv_tags`` where a dict is required. For example, to find a physical volume
561 that has an OSD ID of 0, the filter would look like::
563 pv_tags={'ceph.osd_id': '0'}
566 if not any([pv_name, pv_uuid, pv_tags]):
567 raise TypeError('.filter() requires pv_name, pv_uuid, or pv_tags (none given)')
568 # first find the filtered volumes with the values in self
569 filtered_volumes = self._filter(
574 # then purge everything
576 # and add the filtered items
577 self.extend(filtered_volumes)
579 def get(self, pv_name=None, pv_uuid=None, pv_tags=None):
581 This is a bit expensive, since it will try to filter out all the
582 matching items in the list, filter them out applying anything that was
583 added and return the matching item.
585 This method does *not* alter the list, and it will raise an error if
586 multiple pvs are matched
588 It is useful to use ``tags`` when trying to find a specific logical volume,
589 but it can also lead to multiple pvs being found, since a lot of metadata
590 is shared between pvs of a distinct OSD.
592 if not any([pv_name, pv_uuid, pv_tags]):
602 raise MultiplePVsError(pv_name)
606 class VolumeGroup(object):
608 Represents an LVM group, with some top-level attributes like ``vg_name``
611 def __init__(self, **kw):
612 for k, v in kw.items():
614 self.name = kw['vg_name']
615 self.tags = parse_tags(kw.get('vg_tags', ''))
618 return '<%s>' % self.name
621 return self.__str__()
624 class Volume(object):
626 Represents a Logical Volume from LVM, with some top-level attributes like
627 ``lv_name`` and parsed tags as a dictionary of key/value pairs.
630 def __init__(self, **kw):
631 for k, v in kw.items():
634 self.name = kw['lv_name']
635 self.tags = parse_tags(kw['lv_tags'])
638 return '<%s>' % self.lv_api['lv_path']
641 return self.__str__()
645 obj.update(self.lv_api)
646 obj['tags'] = self.tags
647 obj['name'] = self.name
648 obj['type'] = self.tags['ceph.type']
649 obj['path'] = self.lv_path
652 def clear_tags(self):
654 Removes all tags from the Logical Volume.
656 for k, v in self.tags.items():
657 tag = "%s=%s" % (k, v)
658 process.run(['lvchange', '--deltag', tag, self.lv_path])
660 def set_tags(self, tags):
662 :param tags: A dictionary of tag names and values, like::
665 "ceph.osd_fsid": "aaa-fff-bbbb",
669 At the end of all modifications, the tags are refreshed to reflect
670 LVM's most current view.
672 for k, v in tags.items():
674 # after setting all the tags, refresh them for the current object, use the
675 # lv_* identifiers to filter because those shouldn't change
676 lv_object = get_lv(lv_name=self.lv_name, lv_path=self.lv_path)
677 self.tags = lv_object.tags
679 def set_tag(self, key, value):
681 Set the key/value pair as an LVM tag. Does not "refresh" the values of
682 the current object for its tags. Meant to be a "fire and forget" type
685 # remove it first if it exists
686 if self.tags.get(key):
687 current_value = self.tags[key]
688 tag = "%s=%s" % (key, current_value)
689 process.call(['lvchange', '--deltag', tag, self.lv_api['lv_path']])
694 '--addtag', '%s=%s' % (key, value), self.lv_path
699 class PVolume(object):
701 Represents a Physical Volume from LVM, with some top-level attributes like
702 ``pv_name`` and parsed tags as a dictionary of key/value pairs.
705 def __init__(self, **kw):
706 for k, v in kw.items():
709 self.name = kw['pv_name']
710 self.tags = parse_tags(kw['pv_tags'])
713 return '<%s>' % self.pv_api['pv_name']
716 return self.__str__()
718 def set_tags(self, tags):
720 :param tags: A dictionary of tag names and values, like::
723 "ceph.osd_fsid": "aaa-fff-bbbb",
727 At the end of all modifications, the tags are refreshed to reflect
728 LVM's most current view.
730 for k, v in tags.items():
732 # after setting all the tags, refresh them for the current object, use the
733 # pv_* identifiers to filter because those shouldn't change
734 pv_object = get_pv(pv_name=self.pv_name, pv_uuid=self.pv_uuid)
735 self.tags = pv_object.tags
737 def set_tag(self, key, value):
739 Set the key/value pair as an LVM tag. Does not "refresh" the values of
740 the current object for its tags. Meant to be a "fire and forget" type
743 **warning**: Altering tags on a PV has to be done ensuring that the
744 device is actually the one intended. ``pv_name`` is *not* a persistent
745 value, only ``pv_uuid`` is. Using ``pv_uuid`` is the best way to make
746 sure the device getting changed is the one needed.
748 # remove it first if it exists
749 if self.tags.get(key):
750 current_value = self.tags[key]
751 tag = "%s=%s" % (key, current_value)
752 process.call(['pvchange', '--deltag', tag, self.pv_name])
757 '--addtag', '%s=%s' % (key, value), self.pv_name