--- /dev/null
+"""
+API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention
+that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides
+set of utilities for interacting with LVM.
+"""
+from ceph_volume import process
+from ceph_volume.exceptions import MultipleLVsError, MultipleVGsError, MultiplePVsError
+
+
+def _output_parser(output, fields):
+ """
+ Newer versions of LVM allow ``--reportformat=json``, but older versions,
+ like the one included in Xenial do not. LVM has the ability to filter and
+ format its output so we assume the output will be in a format this parser
+ can handle (using ',' as a delimiter)
+
+ :param fields: A string, possibly using ',' to group many items, as it
+ would be used on the CLI
+ :param output: The CLI output from the LVM call
+ """
+ field_items = fields.split(',')
+ report = []
+ for line in output:
+ # clear the leading/trailing whitespace
+ line = line.strip()
+
+ # remove the extra '"' in each field
+ line = line.replace('"', '')
+
+ # prevent moving forward with empty contents
+ if not line:
+ continue
+
+ # spliting on ';' because that is what the lvm call uses as
+ # '--separator'
+ output_items = [i.strip() for i in line.split(';')]
+ # map the output to the fiels
+ report.append(
+ dict(zip(field_items, output_items))
+ )
+
+ return report
+
+
+def parse_tags(lv_tags):
+ """
+ Return a dictionary mapping of all the tags associated with
+ a Volume from the comma-separated tags coming from the LVM API
+
+ Input look like::
+
+ "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0"
+
+ For the above example, the expected return value would be::
+
+ {
+ "ceph.osd_fsid": "aaa-fff-bbbb",
+ "ceph.osd_id": "0"
+ }
+ """
+ if not lv_tags:
+ return {}
+ tag_mapping = {}
+ tags = lv_tags.split(',')
+ for tag_assignment in tags:
+ if not tag_assignment.startswith('ceph.'):
+ continue
+ key, value = tag_assignment.split('=', 1)
+ tag_mapping[key] = value
+
+ return tag_mapping
+
+
+def get_api_vgs():
+ """
+ Return the list of group volumes available in the system using flags to
+ include common metadata associated with them
+
+ Command and sample delimeted output, should look like::
+
+ $ vgs --noheadings --separator=';' \
+ -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free
+ ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m
+ osd_vg;3;1;0;wz--n-;29.21g;9.21g
+
+ """
+ fields = 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free'
+ stdout, stderr, returncode = process.call(
+ ['vgs', '--noheadings', '--separator=";"', '-o', fields]
+ )
+ return _output_parser(stdout, fields)
+
+
+def get_api_lvs():
+ """
+ Return the list of logical volumes available in the system using flags to include common
+ metadata associated with them
+
+ Command and delimeted output, should look like::
+
+ $ lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name
+ ;/dev/ubuntubox-vg/root;root;ubuntubox-vg
+ ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg
+
+ """
+ fields = 'lv_tags,lv_path,lv_name,vg_name,lv_uuid'
+ stdout, stderr, returncode = process.call(
+ ['lvs', '--noheadings', '--separator=";"', '-o', fields]
+ )
+ return _output_parser(stdout, fields)
+
+
+def get_api_pvs():
+ """
+ Return the list of physical volumes configured for lvm and available in the
+ system using flags to include common metadata associated with them like the uuid
+
+ Command and delimeted output, should look like::
+
+ $ pvs --noheadings --separator=';' -o pv_name,pv_tags,pv_uuid
+ /dev/sda1;;
+ /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D
+
+ """
+ fields = 'pv_name,pv_tags,pv_uuid'
+
+ # note the use of `pvs -a` which will return every physical volume including
+ # ones that have not been initialized as "pv" by LVM
+ stdout, stderr, returncode = process.call(
+ ['pvs', '-a', '--no-heading', '--separator=";"', '-o', fields]
+ )
+
+ return _output_parser(stdout, fields)
+
+
+def get_lv_from_argument(argument):
+ """
+ Helper proxy function that consumes a possible logical volume passed in from the CLI
+ in the form of `vg/lv`, but with some validation so that an argument that is a full
+ path to a device can be ignored
+ """
+ if argument.startswith('/'):
+ lv = get_lv(lv_path=argument)
+ return lv
+ try:
+ vg_name, lv_name = argument.split('/')
+ except (ValueError, AttributeError):
+ return None
+ return get_lv(lv_name=lv_name, vg_name=vg_name)
+
+
+def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
+ """
+ Return a matching lv for the current system, requiring ``lv_name``,
+ ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv
+ is found.
+
+ It is useful to use ``tags`` when trying to find a specific logical volume,
+ but it can also lead to multiple lvs being found, since a lot of metadata
+ is shared between lvs of a distinct OSD.
+ """
+ if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
+ return None
+ lvs = Volumes()
+ return lvs.get(
+ lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_uuid=lv_uuid,
+ lv_tags=lv_tags
+ )
+
+
+def get_pv(pv_name=None, pv_uuid=None, pv_tags=None):
+ """
+ Return a matching pv (physical volume) for the current system, requiring
+ ``pv_name``, ``pv_uuid``, or ``pv_tags``. Raises an error if more than one
+ pv is found.
+ """
+ if not any([pv_name, pv_uuid, pv_tags]):
+ return None
+ pvs = PVolumes()
+ return pvs.get(pv_name=pv_name, pv_uuid=pv_uuid, pv_tags=pv_tags)
+
+
+def create_pv(device):
+ """
+ Create a physical volume from a device, useful when devices need to be later mapped
+ to journals.
+ """
+ process.run([
+ 'pvcreate',
+ '-v', # verbose
+ '-f', # force it
+ '--yes', # answer yes to any prompts
+ device
+ ])
+
+
+def create_vg(name, *devices):
+ """
+ Create a Volume Group. Command looks like::
+
+ vgcreate --force --yes group_name device
+
+ Once created the volume group is returned as a ``VolumeGroup`` object
+ """
+ process.run([
+ 'vgcreate',
+ '--force',
+ '--yes',
+ name] + list(devices)
+ )
+
+ vg = get_vg(vg_name=name)
+ return vg
+
+
+def remove_lv(path):
+ """
+ Removes a logical volume given it's absolute path.
+
+ Will return True if the lv is successfully removed or
+ raises a RuntimeError if the removal fails.
+ """
+ stdout, stderr, returncode = process.call(
+ [
+ 'lvremove',
+ '-v', # verbose
+ '-f', # force it
+ path
+ ],
+ show_command=True,
+ terminal_verbose=True,
+ )
+ if returncode != 0:
+ raise RuntimeError("Unable to remove %s".format(path))
+ return True
+
+
+def create_lv(name, group, size=None, tags=None):
+ """
+ Create a Logical Volume in a Volume Group. Command looks like::
+
+ lvcreate -L 50G -n gfslv vg0
+
+ ``name``, ``group``, are required. If ``size`` is provided it must follow
+ lvm's size notation (like 1G, or 20M). Tags are an optional dictionary and is expected to
+ conform to the convention of prefixing them with "ceph." like::
+
+ {"ceph.block_device": "/dev/ceph/osd-1"}
+ """
+ # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations
+ type_path_tag = {
+ 'journal': 'ceph.journal_device',
+ 'data': 'ceph.data_device',
+ 'block': 'ceph.block_device',
+ 'wal': 'ceph.wal_device',
+ 'db': 'ceph.db_device',
+ 'lockbox': 'ceph.lockbox_device', # XXX might not ever need this lockbox sorcery
+ }
+ if size:
+ process.run([
+ 'lvcreate',
+ '--yes',
+ '-L',
+ '%s' % size,
+ '-n', name, group
+ ])
+ # create the lv with all the space available, this is needed because the
+ # system call is different for LVM
+ else:
+ process.run([
+ 'lvcreate',
+ '--yes',
+ '-l',
+ '100%FREE',
+ '-n', name, group
+ ])
+
+ lv = get_lv(lv_name=name, vg_name=group)
+ lv.set_tags(tags)
+
+ # when creating a distinct type, the caller doesn't know what the path will
+ # be so this function will set it after creation using the mapping
+ path_tag = type_path_tag.get(tags.get('ceph.type'))
+ if path_tag:
+ lv.set_tags(
+ {path_tag: lv.lv_path}
+ )
+ return lv
+
+
+def get_vg(vg_name=None, vg_tags=None):
+ """
+ Return a matching vg for the current system, requires ``vg_name`` or
+ ``tags``. Raises an error if more than one vg is found.
+
+ It is useful to use ``tags`` when trying to find a specific volume group,
+ but it can also lead to multiple vgs being found.
+ """
+ if not any([vg_name, vg_tags]):
+ return None
+ vgs = VolumeGroups()
+ return vgs.get(vg_name=vg_name, vg_tags=vg_tags)
+
+
+class VolumeGroups(list):
+ """
+ A list of all known volume groups for the current system, with the ability
+ to filter them via keyword arguments.
+ """
+
+ def __init__(self):
+ self._populate()
+
+ def _populate(self):
+ # get all the vgs in the current system
+ for vg_item in get_api_vgs():
+ self.append(VolumeGroup(**vg_item))
+
+ def _purge(self):
+ """
+ Deplete all the items in the list, used internally only so that we can
+ dynamically allocate the items when filtering without the concern of
+ messing up the contents
+ """
+ self[:] = []
+
+ def _filter(self, vg_name=None, vg_tags=None):
+ """
+ The actual method that filters using a new list. Useful so that other
+ methods that do not want to alter the contents of the list (e.g.
+ ``self.find``) can operate safely.
+
+ .. note:: ``vg_tags`` is not yet implemented
+ """
+ filtered = [i for i in self]
+ if vg_name:
+ filtered = [i for i in filtered if i.vg_name == vg_name]
+
+ # at this point, `filtered` has either all the volumes in self or is an
+ # actual filtered list if any filters were applied
+ if vg_tags:
+ tag_filtered = []
+ for volume in filtered:
+ matches = all(volume.tags.get(k) == str(v) for k, v in vg_tags.items())
+ if matches:
+ tag_filtered.append(volume)
+ return tag_filtered
+
+ return filtered
+
+ def filter(self, vg_name=None, vg_tags=None):
+ """
+ Filter out groups on top level attributes like ``vg_name`` or by
+ ``vg_tags`` where a dict is required. For example, to find a Ceph group
+ with dmcache as the type, the filter would look like::
+
+ vg_tags={'ceph.type': 'dmcache'}
+
+ .. warning:: These tags are not documented because they are currently
+ unused, but are here to maintain API consistency
+ """
+ if not any([vg_name, vg_tags]):
+ raise TypeError('.filter() requires vg_name or vg_tags (none given)')
+ # first find the filtered volumes with the values in self
+ filtered_groups = self._filter(
+ vg_name=vg_name,
+ vg_tags=vg_tags
+ )
+ # then purge everything
+ self._purge()
+ # and add the filtered items
+ self.extend(filtered_groups)
+
+ def get(self, vg_name=None, vg_tags=None):
+ """
+ This is a bit expensive, since it will try to filter out all the
+ matching items in the list, filter them out applying anything that was
+ added and return the matching item.
+
+ This method does *not* alter the list, and it will raise an error if
+ multiple VGs are matched
+
+ It is useful to use ``tags`` when trying to find a specific volume group,
+ but it can also lead to multiple vgs being found (although unlikely)
+ """
+ if not any([vg_name, vg_tags]):
+ return None
+ vgs = self._filter(
+ vg_name=vg_name,
+ vg_tags=vg_tags
+ )
+ if not vgs:
+ return None
+ if len(vgs) > 1:
+ # this is probably never going to happen, but it is here to keep
+ # the API code consistent
+ raise MultipleVGsError(vg_name)
+ return vgs[0]
+
+
+class Volumes(list):
+ """
+ A list of all known (logical) volumes for the current system, with the ability
+ to filter them via keyword arguments.
+ """
+
+ def __init__(self):
+ self._populate()
+
+ def _populate(self):
+ # get all the lvs in the current system
+ for lv_item in get_api_lvs():
+ self.append(Volume(**lv_item))
+
+ def _purge(self):
+ """
+ Deplete all the items in the list, used internally only so that we can
+ dynamically allocate the items when filtering without the concern of
+ messing up the contents
+ """
+ self[:] = []
+
+ def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
+ """
+ The actual method that filters using a new list. Useful so that other
+ methods that do not want to alter the contents of the list (e.g.
+ ``self.find``) can operate safely.
+ """
+ filtered = [i for i in self]
+ if lv_name:
+ filtered = [i for i in filtered if i.lv_name == lv_name]
+
+ if vg_name:
+ filtered = [i for i in filtered if i.vg_name == vg_name]
+
+ if lv_uuid:
+ filtered = [i for i in filtered if i.lv_uuid == lv_uuid]
+
+ if lv_path:
+ filtered = [i for i in filtered if i.lv_path == lv_path]
+
+ # at this point, `filtered` has either all the volumes in self or is an
+ # actual filtered list if any filters were applied
+ if lv_tags:
+ tag_filtered = []
+ for volume in filtered:
+ # all the tags we got need to match on the volume
+ matches = all(volume.tags.get(k) == str(v) for k, v in lv_tags.items())
+ if matches:
+ tag_filtered.append(volume)
+ return tag_filtered
+
+ return filtered
+
+ def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
+ """
+ Filter out volumes on top level attributes like ``lv_name`` or by
+ ``lv_tags`` where a dict is required. For example, to find a volume
+ that has an OSD ID of 0, the filter would look like::
+
+ lv_tags={'ceph.osd_id': '0'}
+
+ """
+ if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
+ raise TypeError('.filter() requires lv_name, vg_name, lv_path, lv_uuid, or tags (none given)')
+ # first find the filtered volumes with the values in self
+ filtered_volumes = self._filter(
+ lv_name=lv_name,
+ vg_name=vg_name,
+ lv_path=lv_path,
+ lv_uuid=lv_uuid,
+ lv_tags=lv_tags
+ )
+ # then purge everything
+ self._purge()
+ # and add the filtered items
+ self.extend(filtered_volumes)
+
+ def get(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
+ """
+ This is a bit expensive, since it will try to filter out all the
+ matching items in the list, filter them out applying anything that was
+ added and return the matching item.
+
+ This method does *not* alter the list, and it will raise an error if
+ multiple LVs are matched
+
+ It is useful to use ``tags`` when trying to find a specific logical volume,
+ but it can also lead to multiple lvs being found, since a lot of metadata
+ is shared between lvs of a distinct OSD.
+ """
+ if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
+ return None
+ lvs = self._filter(
+ lv_name=lv_name,
+ vg_name=vg_name,
+ lv_path=lv_path,
+ lv_uuid=lv_uuid,
+ lv_tags=lv_tags
+ )
+ if not lvs:
+ return None
+ if len(lvs) > 1:
+ raise MultipleLVsError(lv_name, lv_path)
+ return lvs[0]
+
+
+class PVolumes(list):
+ """
+ A list of all known (physical) volumes for the current system, with the ability
+ to filter them via keyword arguments.
+ """
+
+ def __init__(self):
+ self._populate()
+
+ def _populate(self):
+ # get all the pvs in the current system
+ for pv_item in get_api_pvs():
+ self.append(PVolume(**pv_item))
+
+ def _purge(self):
+ """
+ Deplete all the items in the list, used internally only so that we can
+ dynamically allocate the items when filtering without the concern of
+ messing up the contents
+ """
+ self[:] = []
+
+ def _filter(self, pv_name=None, pv_uuid=None, pv_tags=None):
+ """
+ The actual method that filters using a new list. Useful so that other
+ methods that do not want to alter the contents of the list (e.g.
+ ``self.find``) can operate safely.
+ """
+ filtered = [i for i in self]
+ if pv_name:
+ filtered = [i for i in filtered if i.pv_name == pv_name]
+
+ if pv_uuid:
+ filtered = [i for i in filtered if i.pv_uuid == pv_uuid]
+
+ # at this point, `filtered` has either all the physical volumes in self
+ # or is an actual filtered list if any filters were applied
+ if pv_tags:
+ tag_filtered = []
+ for pvolume in filtered:
+ matches = all(pvolume.tags.get(k) == str(v) for k, v in pv_tags.items())
+ if matches:
+ tag_filtered.append(pvolume)
+ # return the tag_filtered pvolumes here, the `filtered` list is no
+ # longer useable
+ return tag_filtered
+
+ return filtered
+
+ def filter(self, pv_name=None, pv_uuid=None, pv_tags=None):
+ """
+ Filter out volumes on top level attributes like ``pv_name`` or by
+ ``pv_tags`` where a dict is required. For example, to find a physical volume
+ that has an OSD ID of 0, the filter would look like::
+
+ pv_tags={'ceph.osd_id': '0'}
+
+ """
+ if not any([pv_name, pv_uuid, pv_tags]):
+ raise TypeError('.filter() requires pv_name, pv_uuid, or pv_tags (none given)')
+ # first find the filtered volumes with the values in self
+ filtered_volumes = self._filter(
+ pv_name=pv_name,
+ pv_uuid=pv_uuid,
+ pv_tags=pv_tags
+ )
+ # then purge everything
+ self._purge()
+ # and add the filtered items
+ self.extend(filtered_volumes)
+
+ def get(self, pv_name=None, pv_uuid=None, pv_tags=None):
+ """
+ This is a bit expensive, since it will try to filter out all the
+ matching items in the list, filter them out applying anything that was
+ added and return the matching item.
+
+ This method does *not* alter the list, and it will raise an error if
+ multiple pvs are matched
+
+ It is useful to use ``tags`` when trying to find a specific logical volume,
+ but it can also lead to multiple pvs being found, since a lot of metadata
+ is shared between pvs of a distinct OSD.
+ """
+ if not any([pv_name, pv_uuid, pv_tags]):
+ return None
+ pvs = self._filter(
+ pv_name=pv_name,
+ pv_uuid=pv_uuid,
+ pv_tags=pv_tags
+ )
+ if not pvs:
+ return None
+ if len(pvs) > 1:
+ raise MultiplePVsError(pv_name)
+ return pvs[0]
+
+
+class VolumeGroup(object):
+ """
+ Represents an LVM group, with some top-level attributes like ``vg_name``
+ """
+
+ def __init__(self, **kw):
+ for k, v in kw.items():
+ setattr(self, k, v)
+ self.name = kw['vg_name']
+ self.tags = parse_tags(kw.get('vg_tags', ''))
+
+ def __str__(self):
+ return '<%s>' % self.name
+
+ def __repr__(self):
+ return self.__str__()
+
+
+class Volume(object):
+ """
+ Represents a Logical Volume from LVM, with some top-level attributes like
+ ``lv_name`` and parsed tags as a dictionary of key/value pairs.
+ """
+
+ def __init__(self, **kw):
+ for k, v in kw.items():
+ setattr(self, k, v)
+ self.lv_api = kw
+ self.name = kw['lv_name']
+ self.tags = parse_tags(kw['lv_tags'])
+
+ def __str__(self):
+ return '<%s>' % self.lv_api['lv_path']
+
+ def __repr__(self):
+ return self.__str__()
+
+ def as_dict(self):
+ obj = {}
+ obj.update(self.lv_api)
+ obj['tags'] = self.tags
+ obj['name'] = self.name
+ obj['type'] = self.tags['ceph.type']
+ obj['path'] = self.lv_path
+ return obj
+
+ def clear_tags(self):
+ """
+ Removes all tags from the Logical Volume.
+ """
+ for k, v in self.tags.items():
+ tag = "%s=%s" % (k, v)
+ process.run(['lvchange', '--deltag', tag, self.lv_path])
+
+ def set_tags(self, tags):
+ """
+ :param tags: A dictionary of tag names and values, like::
+
+ {
+ "ceph.osd_fsid": "aaa-fff-bbbb",
+ "ceph.osd_id": "0"
+ }
+
+ At the end of all modifications, the tags are refreshed to reflect
+ LVM's most current view.
+ """
+ for k, v in tags.items():
+ self.set_tag(k, v)
+ # after setting all the tags, refresh them for the current object, use the
+ # lv_* identifiers to filter because those shouldn't change
+ lv_object = get_lv(lv_name=self.lv_name, lv_path=self.lv_path)
+ self.tags = lv_object.tags
+
+ def set_tag(self, key, value):
+ """
+ Set the key/value pair as an LVM tag. Does not "refresh" the values of
+ the current object for its tags. Meant to be a "fire and forget" type
+ of modification.
+ """
+ # remove it first if it exists
+ if self.tags.get(key):
+ current_value = self.tags[key]
+ tag = "%s=%s" % (key, current_value)
+ process.call(['lvchange', '--deltag', tag, self.lv_api['lv_path']])
+
+ process.call(
+ [
+ 'lvchange',
+ '--addtag', '%s=%s' % (key, value), self.lv_path
+ ]
+ )
+
+
+class PVolume(object):
+ """
+ Represents a Physical Volume from LVM, with some top-level attributes like
+ ``pv_name`` and parsed tags as a dictionary of key/value pairs.
+ """
+
+ def __init__(self, **kw):
+ for k, v in kw.items():
+ setattr(self, k, v)
+ self.pv_api = kw
+ self.name = kw['pv_name']
+ self.tags = parse_tags(kw['pv_tags'])
+
+ def __str__(self):
+ return '<%s>' % self.pv_api['pv_name']
+
+ def __repr__(self):
+ return self.__str__()
+
+ def set_tags(self, tags):
+ """
+ :param tags: A dictionary of tag names and values, like::
+
+ {
+ "ceph.osd_fsid": "aaa-fff-bbbb",
+ "ceph.osd_id": "0"
+ }
+
+ At the end of all modifications, the tags are refreshed to reflect
+ LVM's most current view.
+ """
+ for k, v in tags.items():
+ self.set_tag(k, v)
+ # after setting all the tags, refresh them for the current object, use the
+ # pv_* identifiers to filter because those shouldn't change
+ pv_object = get_pv(pv_name=self.pv_name, pv_uuid=self.pv_uuid)
+ self.tags = pv_object.tags
+
+ def set_tag(self, key, value):
+ """
+ Set the key/value pair as an LVM tag. Does not "refresh" the values of
+ the current object for its tags. Meant to be a "fire and forget" type
+ of modification.
+
+ **warning**: Altering tags on a PV has to be done ensuring that the
+ device is actually the one intended. ``pv_name`` is *not* a persistent
+ value, only ``pv_uuid`` is. Using ``pv_uuid`` is the best way to make
+ sure the device getting changed is the one needed.
+ """
+ # remove it first if it exists
+ if self.tags.get(key):
+ current_value = self.tags[key]
+ tag = "%s=%s" % (key, current_value)
+ process.call(['pvchange', '--deltag', tag, self.pv_name])
+
+ process.call(
+ [
+ 'pvchange',
+ '--addtag', '%s=%s' % (key, value), self.pv_name
+ ]
+ )
Code Review / stor4nfv.git / blobdiff