X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=src%2Fceph%2Fsrc%2Fceph-volume%2Fceph_volume%2Fapi%2Flvm.py;fp=src%2Fceph%2Fsrc%2Fceph-volume%2Fceph_volume%2Fapi%2Flvm.py;h=0000000000000000000000000000000000000000;hb=7da45d65be36d36b880cc55c5036e96c24b53f00;hp=df1ec48cd1ab2dfbd24bf21b885cda047569d222;hpb=691462d09d0987b47e112d6ee8740375df3c51b2;p=stor4nfv.git diff --git a/src/ceph/src/ceph-volume/ceph_volume/api/lvm.py b/src/ceph/src/ceph-volume/ceph_volume/api/lvm.py deleted file mode 100644 index df1ec48..0000000 --- a/src/ceph/src/ceph-volume/ceph_volume/api/lvm.py +++ /dev/null @@ -1,759 +0,0 @@ -""" -API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention -that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides -set of utilities for interacting with LVM. -""" -from ceph_volume import process -from ceph_volume.exceptions import MultipleLVsError, MultipleVGsError, MultiplePVsError - - -def _output_parser(output, fields): - """ - Newer versions of LVM allow ``--reportformat=json``, but older versions, - like the one included in Xenial do not. LVM has the ability to filter and - format its output so we assume the output will be in a format this parser - can handle (using ',' as a delimiter) - - :param fields: A string, possibly using ',' to group many items, as it - would be used on the CLI - :param output: The CLI output from the LVM call - """ - field_items = fields.split(',') - report = [] - for line in output: - # clear the leading/trailing whitespace - line = line.strip() - - # remove the extra '"' in each field - line = line.replace('"', '') - - # prevent moving forward with empty contents - if not line: - continue - - # spliting on ';' because that is what the lvm call uses as - # '--separator' - output_items = [i.strip() for i in line.split(';')] - # map the output to the fiels - report.append( - dict(zip(field_items, output_items)) - ) - - return report - - -def parse_tags(lv_tags): - """ - Return a dictionary mapping of all the tags associated with - a Volume from the comma-separated tags coming from the LVM API - - Input look like:: - - "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0" - - For the above example, the expected return value would be:: - - { - "ceph.osd_fsid": "aaa-fff-bbbb", - "ceph.osd_id": "0" - } - """ - if not lv_tags: - return {} - tag_mapping = {} - tags = lv_tags.split(',') - for tag_assignment in tags: - if not tag_assignment.startswith('ceph.'): - continue - key, value = tag_assignment.split('=', 1) - tag_mapping[key] = value - - return tag_mapping - - -def get_api_vgs(): - """ - Return the list of group volumes available in the system using flags to - include common metadata associated with them - - Command and sample delimeted output, should look like:: - - $ vgs --noheadings --separator=';' \ - -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free - ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m - osd_vg;3;1;0;wz--n-;29.21g;9.21g - - """ - fields = 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free' - stdout, stderr, returncode = process.call( - ['vgs', '--noheadings', '--separator=";"', '-o', fields] - ) - return _output_parser(stdout, fields) - - -def get_api_lvs(): - """ - Return the list of logical volumes available in the system using flags to include common - metadata associated with them - - Command and delimeted output, should look like:: - - $ lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name - ;/dev/ubuntubox-vg/root;root;ubuntubox-vg - ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg - - """ - fields = 'lv_tags,lv_path,lv_name,vg_name,lv_uuid' - stdout, stderr, returncode = process.call( - ['lvs', '--noheadings', '--separator=";"', '-o', fields] - ) - return _output_parser(stdout, fields) - - -def get_api_pvs(): - """ - Return the list of physical volumes configured for lvm and available in the - system using flags to include common metadata associated with them like the uuid - - Command and delimeted output, should look like:: - - $ pvs --noheadings --separator=';' -o pv_name,pv_tags,pv_uuid - /dev/sda1;; - /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D - - """ - fields = 'pv_name,pv_tags,pv_uuid' - - # note the use of `pvs -a` which will return every physical volume including - # ones that have not been initialized as "pv" by LVM - stdout, stderr, returncode = process.call( - ['pvs', '-a', '--no-heading', '--separator=";"', '-o', fields] - ) - - return _output_parser(stdout, fields) - - -def get_lv_from_argument(argument): - """ - Helper proxy function that consumes a possible logical volume passed in from the CLI - in the form of `vg/lv`, but with some validation so that an argument that is a full - path to a device can be ignored - """ - if argument.startswith('/'): - lv = get_lv(lv_path=argument) - return lv - try: - vg_name, lv_name = argument.split('/') - except (ValueError, AttributeError): - return None - return get_lv(lv_name=lv_name, vg_name=vg_name) - - -def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): - """ - Return a matching lv for the current system, requiring ``lv_name``, - ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv - is found. - - It is useful to use ``tags`` when trying to find a specific logical volume, - but it can also lead to multiple lvs being found, since a lot of metadata - is shared between lvs of a distinct OSD. - """ - if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): - return None - lvs = Volumes() - return lvs.get( - lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_uuid=lv_uuid, - lv_tags=lv_tags - ) - - -def get_pv(pv_name=None, pv_uuid=None, pv_tags=None): - """ - Return a matching pv (physical volume) for the current system, requiring - ``pv_name``, ``pv_uuid``, or ``pv_tags``. Raises an error if more than one - pv is found. - """ - if not any([pv_name, pv_uuid, pv_tags]): - return None - pvs = PVolumes() - return pvs.get(pv_name=pv_name, pv_uuid=pv_uuid, pv_tags=pv_tags) - - -def create_pv(device): - """ - Create a physical volume from a device, useful when devices need to be later mapped - to journals. - """ - process.run([ - 'pvcreate', - '-v', # verbose - '-f', # force it - '--yes', # answer yes to any prompts - device - ]) - - -def create_vg(name, *devices): - """ - Create a Volume Group. Command looks like:: - - vgcreate --force --yes group_name device - - Once created the volume group is returned as a ``VolumeGroup`` object - """ - process.run([ - 'vgcreate', - '--force', - '--yes', - name] + list(devices) - ) - - vg = get_vg(vg_name=name) - return vg - - -def remove_lv(path): - """ - Removes a logical volume given it's absolute path. - - Will return True if the lv is successfully removed or - raises a RuntimeError if the removal fails. - """ - stdout, stderr, returncode = process.call( - [ - 'lvremove', - '-v', # verbose - '-f', # force it - path - ], - show_command=True, - terminal_verbose=True, - ) - if returncode != 0: - raise RuntimeError("Unable to remove %s".format(path)) - return True - - -def create_lv(name, group, size=None, tags=None): - """ - Create a Logical Volume in a Volume Group. Command looks like:: - - lvcreate -L 50G -n gfslv vg0 - - ``name``, ``group``, are required. If ``size`` is provided it must follow - lvm's size notation (like 1G, or 20M). Tags are an optional dictionary and is expected to - conform to the convention of prefixing them with "ceph." like:: - - {"ceph.block_device": "/dev/ceph/osd-1"} - """ - # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations - type_path_tag = { - 'journal': 'ceph.journal_device', - 'data': 'ceph.data_device', - 'block': 'ceph.block_device', - 'wal': 'ceph.wal_device', - 'db': 'ceph.db_device', - 'lockbox': 'ceph.lockbox_device', # XXX might not ever need this lockbox sorcery - } - if size: - process.run([ - 'lvcreate', - '--yes', - '-L', - '%s' % size, - '-n', name, group - ]) - # create the lv with all the space available, this is needed because the - # system call is different for LVM - else: - process.run([ - 'lvcreate', - '--yes', - '-l', - '100%FREE', - '-n', name, group - ]) - - lv = get_lv(lv_name=name, vg_name=group) - lv.set_tags(tags) - - # when creating a distinct type, the caller doesn't know what the path will - # be so this function will set it after creation using the mapping - path_tag = type_path_tag.get(tags.get('ceph.type')) - if path_tag: - lv.set_tags( - {path_tag: lv.lv_path} - ) - return lv - - -def get_vg(vg_name=None, vg_tags=None): - """ - Return a matching vg for the current system, requires ``vg_name`` or - ``tags``. Raises an error if more than one vg is found. - - It is useful to use ``tags`` when trying to find a specific volume group, - but it can also lead to multiple vgs being found. - """ - if not any([vg_name, vg_tags]): - return None - vgs = VolumeGroups() - return vgs.get(vg_name=vg_name, vg_tags=vg_tags) - - -class VolumeGroups(list): - """ - A list of all known volume groups for the current system, with the ability - to filter them via keyword arguments. - """ - - def __init__(self): - self._populate() - - def _populate(self): - # get all the vgs in the current system - for vg_item in get_api_vgs(): - self.append(VolumeGroup(**vg_item)) - - def _purge(self): - """ - Deplete all the items in the list, used internally only so that we can - dynamically allocate the items when filtering without the concern of - messing up the contents - """ - self[:] = [] - - def _filter(self, vg_name=None, vg_tags=None): - """ - The actual method that filters using a new list. Useful so that other - methods that do not want to alter the contents of the list (e.g. - ``self.find``) can operate safely. - - .. note:: ``vg_tags`` is not yet implemented - """ - filtered = [i for i in self] - if vg_name: - filtered = [i for i in filtered if i.vg_name == vg_name] - - # at this point, `filtered` has either all the volumes in self or is an - # actual filtered list if any filters were applied - if vg_tags: - tag_filtered = [] - for volume in filtered: - matches = all(volume.tags.get(k) == str(v) for k, v in vg_tags.items()) - if matches: - tag_filtered.append(volume) - return tag_filtered - - return filtered - - def filter(self, vg_name=None, vg_tags=None): - """ - Filter out groups on top level attributes like ``vg_name`` or by - ``vg_tags`` where a dict is required. For example, to find a Ceph group - with dmcache as the type, the filter would look like:: - - vg_tags={'ceph.type': 'dmcache'} - - .. warning:: These tags are not documented because they are currently - unused, but are here to maintain API consistency - """ - if not any([vg_name, vg_tags]): - raise TypeError('.filter() requires vg_name or vg_tags (none given)') - # first find the filtered volumes with the values in self - filtered_groups = self._filter( - vg_name=vg_name, - vg_tags=vg_tags - ) - # then purge everything - self._purge() - # and add the filtered items - self.extend(filtered_groups) - - def get(self, vg_name=None, vg_tags=None): - """ - This is a bit expensive, since it will try to filter out all the - matching items in the list, filter them out applying anything that was - added and return the matching item. - - This method does *not* alter the list, and it will raise an error if - multiple VGs are matched - - It is useful to use ``tags`` when trying to find a specific volume group, - but it can also lead to multiple vgs being found (although unlikely) - """ - if not any([vg_name, vg_tags]): - return None - vgs = self._filter( - vg_name=vg_name, - vg_tags=vg_tags - ) - if not vgs: - return None - if len(vgs) > 1: - # this is probably never going to happen, but it is here to keep - # the API code consistent - raise MultipleVGsError(vg_name) - return vgs[0] - - -class Volumes(list): - """ - A list of all known (logical) volumes for the current system, with the ability - to filter them via keyword arguments. - """ - - def __init__(self): - self._populate() - - def _populate(self): - # get all the lvs in the current system - for lv_item in get_api_lvs(): - self.append(Volume(**lv_item)) - - def _purge(self): - """ - Deplete all the items in the list, used internally only so that we can - dynamically allocate the items when filtering without the concern of - messing up the contents - """ - self[:] = [] - - def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): - """ - The actual method that filters using a new list. Useful so that other - methods that do not want to alter the contents of the list (e.g. - ``self.find``) can operate safely. - """ - filtered = [i for i in self] - if lv_name: - filtered = [i for i in filtered if i.lv_name == lv_name] - - if vg_name: - filtered = [i for i in filtered if i.vg_name == vg_name] - - if lv_uuid: - filtered = [i for i in filtered if i.lv_uuid == lv_uuid] - - if lv_path: - filtered = [i for i in filtered if i.lv_path == lv_path] - - # at this point, `filtered` has either all the volumes in self or is an - # actual filtered list if any filters were applied - if lv_tags: - tag_filtered = [] - for volume in filtered: - # all the tags we got need to match on the volume - matches = all(volume.tags.get(k) == str(v) for k, v in lv_tags.items()) - if matches: - tag_filtered.append(volume) - return tag_filtered - - return filtered - - def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): - """ - Filter out volumes on top level attributes like ``lv_name`` or by - ``lv_tags`` where a dict is required. For example, to find a volume - that has an OSD ID of 0, the filter would look like:: - - lv_tags={'ceph.osd_id': '0'} - - """ - if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): - raise TypeError('.filter() requires lv_name, vg_name, lv_path, lv_uuid, or tags (none given)') - # first find the filtered volumes with the values in self - filtered_volumes = self._filter( - lv_name=lv_name, - vg_name=vg_name, - lv_path=lv_path, - lv_uuid=lv_uuid, - lv_tags=lv_tags - ) - # then purge everything - self._purge() - # and add the filtered items - self.extend(filtered_volumes) - - def get(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): - """ - This is a bit expensive, since it will try to filter out all the - matching items in the list, filter them out applying anything that was - added and return the matching item. - - This method does *not* alter the list, and it will raise an error if - multiple LVs are matched - - It is useful to use ``tags`` when trying to find a specific logical volume, - but it can also lead to multiple lvs being found, since a lot of metadata - is shared between lvs of a distinct OSD. - """ - if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): - return None - lvs = self._filter( - lv_name=lv_name, - vg_name=vg_name, - lv_path=lv_path, - lv_uuid=lv_uuid, - lv_tags=lv_tags - ) - if not lvs: - return None - if len(lvs) > 1: - raise MultipleLVsError(lv_name, lv_path) - return lvs[0] - - -class PVolumes(list): - """ - A list of all known (physical) volumes for the current system, with the ability - to filter them via keyword arguments. - """ - - def __init__(self): - self._populate() - - def _populate(self): - # get all the pvs in the current system - for pv_item in get_api_pvs(): - self.append(PVolume(**pv_item)) - - def _purge(self): - """ - Deplete all the items in the list, used internally only so that we can - dynamically allocate the items when filtering without the concern of - messing up the contents - """ - self[:] = [] - - def _filter(self, pv_name=None, pv_uuid=None, pv_tags=None): - """ - The actual method that filters using a new list. Useful so that other - methods that do not want to alter the contents of the list (e.g. - ``self.find``) can operate safely. - """ - filtered = [i for i in self] - if pv_name: - filtered = [i for i in filtered if i.pv_name == pv_name] - - if pv_uuid: - filtered = [i for i in filtered if i.pv_uuid == pv_uuid] - - # at this point, `filtered` has either all the physical volumes in self - # or is an actual filtered list if any filters were applied - if pv_tags: - tag_filtered = [] - for pvolume in filtered: - matches = all(pvolume.tags.get(k) == str(v) for k, v in pv_tags.items()) - if matches: - tag_filtered.append(pvolume) - # return the tag_filtered pvolumes here, the `filtered` list is no - # longer useable - return tag_filtered - - return filtered - - def filter(self, pv_name=None, pv_uuid=None, pv_tags=None): - """ - Filter out volumes on top level attributes like ``pv_name`` or by - ``pv_tags`` where a dict is required. For example, to find a physical volume - that has an OSD ID of 0, the filter would look like:: - - pv_tags={'ceph.osd_id': '0'} - - """ - if not any([pv_name, pv_uuid, pv_tags]): - raise TypeError('.filter() requires pv_name, pv_uuid, or pv_tags (none given)') - # first find the filtered volumes with the values in self - filtered_volumes = self._filter( - pv_name=pv_name, - pv_uuid=pv_uuid, - pv_tags=pv_tags - ) - # then purge everything - self._purge() - # and add the filtered items - self.extend(filtered_volumes) - - def get(self, pv_name=None, pv_uuid=None, pv_tags=None): - """ - This is a bit expensive, since it will try to filter out all the - matching items in the list, filter them out applying anything that was - added and return the matching item. - - This method does *not* alter the list, and it will raise an error if - multiple pvs are matched - - It is useful to use ``tags`` when trying to find a specific logical volume, - but it can also lead to multiple pvs being found, since a lot of metadata - is shared between pvs of a distinct OSD. - """ - if not any([pv_name, pv_uuid, pv_tags]): - return None - pvs = self._filter( - pv_name=pv_name, - pv_uuid=pv_uuid, - pv_tags=pv_tags - ) - if not pvs: - return None - if len(pvs) > 1: - raise MultiplePVsError(pv_name) - return pvs[0] - - -class VolumeGroup(object): - """ - Represents an LVM group, with some top-level attributes like ``vg_name`` - """ - - def __init__(self, **kw): - for k, v in kw.items(): - setattr(self, k, v) - self.name = kw['vg_name'] - self.tags = parse_tags(kw.get('vg_tags', '')) - - def __str__(self): - return '<%s>' % self.name - - def __repr__(self): - return self.__str__() - - -class Volume(object): - """ - Represents a Logical Volume from LVM, with some top-level attributes like - ``lv_name`` and parsed tags as a dictionary of key/value pairs. - """ - - def __init__(self, **kw): - for k, v in kw.items(): - setattr(self, k, v) - self.lv_api = kw - self.name = kw['lv_name'] - self.tags = parse_tags(kw['lv_tags']) - - def __str__(self): - return '<%s>' % self.lv_api['lv_path'] - - def __repr__(self): - return self.__str__() - - def as_dict(self): - obj = {} - obj.update(self.lv_api) - obj['tags'] = self.tags - obj['name'] = self.name - obj['type'] = self.tags['ceph.type'] - obj['path'] = self.lv_path - return obj - - def clear_tags(self): - """ - Removes all tags from the Logical Volume. - """ - for k, v in self.tags.items(): - tag = "%s=%s" % (k, v) - process.run(['lvchange', '--deltag', tag, self.lv_path]) - - def set_tags(self, tags): - """ - :param tags: A dictionary of tag names and values, like:: - - { - "ceph.osd_fsid": "aaa-fff-bbbb", - "ceph.osd_id": "0" - } - - At the end of all modifications, the tags are refreshed to reflect - LVM's most current view. - """ - for k, v in tags.items(): - self.set_tag(k, v) - # after setting all the tags, refresh them for the current object, use the - # lv_* identifiers to filter because those shouldn't change - lv_object = get_lv(lv_name=self.lv_name, lv_path=self.lv_path) - self.tags = lv_object.tags - - def set_tag(self, key, value): - """ - Set the key/value pair as an LVM tag. Does not "refresh" the values of - the current object for its tags. Meant to be a "fire and forget" type - of modification. - """ - # remove it first if it exists - if self.tags.get(key): - current_value = self.tags[key] - tag = "%s=%s" % (key, current_value) - process.call(['lvchange', '--deltag', tag, self.lv_api['lv_path']]) - - process.call( - [ - 'lvchange', - '--addtag', '%s=%s' % (key, value), self.lv_path - ] - ) - - -class PVolume(object): - """ - Represents a Physical Volume from LVM, with some top-level attributes like - ``pv_name`` and parsed tags as a dictionary of key/value pairs. - """ - - def __init__(self, **kw): - for k, v in kw.items(): - setattr(self, k, v) - self.pv_api = kw - self.name = kw['pv_name'] - self.tags = parse_tags(kw['pv_tags']) - - def __str__(self): - return '<%s>' % self.pv_api['pv_name'] - - def __repr__(self): - return self.__str__() - - def set_tags(self, tags): - """ - :param tags: A dictionary of tag names and values, like:: - - { - "ceph.osd_fsid": "aaa-fff-bbbb", - "ceph.osd_id": "0" - } - - At the end of all modifications, the tags are refreshed to reflect - LVM's most current view. - """ - for k, v in tags.items(): - self.set_tag(k, v) - # after setting all the tags, refresh them for the current object, use the - # pv_* identifiers to filter because those shouldn't change - pv_object = get_pv(pv_name=self.pv_name, pv_uuid=self.pv_uuid) - self.tags = pv_object.tags - - def set_tag(self, key, value): - """ - Set the key/value pair as an LVM tag. Does not "refresh" the values of - the current object for its tags. Meant to be a "fire and forget" type - of modification. - - **warning**: Altering tags on a PV has to be done ensuring that the - device is actually the one intended. ``pv_name`` is *not* a persistent - value, only ``pv_uuid`` is. Using ``pv_uuid`` is the best way to make - sure the device getting changed is the one needed. - """ - # remove it first if it exists - if self.tags.get(key): - current_value = self.tags[key] - tag = "%s=%s" % (key, current_value) - process.call(['pvchange', '--deltag', tag, self.pv_name]) - - process.call( - [ - 'pvchange', - '--addtag', '%s=%s' % (key, value), self.pv_name - ] - )