2 This module is a thin wrapper around rgw_file.
6 from cpython cimport PyObject, ref, exc, array
7 from libc.stdint cimport *
8 from libcpp cimport bool
9 from libc.stdlib cimport malloc, realloc, free
13 from collections import namedtuple
14 from datetime import datetime
18 cdef extern from "Python.h":
19 # These are in cpython/string.pxd, but use "object" types instead of
20 # PyObject*, which invokes assumptions in cpython that we need to
21 # legitimately break to implement zero-copy string buffers in Image.read().
22 # This is valid use of the Python API and documented as a special case.
23 PyObject *PyBytes_FromStringAndSize(char *v, Py_ssize_t len) except NULL
24 char* PyBytes_AsString(PyObject *string) except NULL
25 int _PyBytes_Resize(PyObject **string, Py_ssize_t newsize) except -1
26 void PyEval_InitThreads()
29 cdef extern from "time.h":
30 ctypedef long int time_t
33 cdef extern from "sys/stat.h":
37 unsigned long st_nlink
51 cdef extern from "rados/librgw.h" nogil:
52 ctypedef void* librgw_t
54 int librgw_create(librgw_t *rgw, int argc, char **argv)
55 void librgw_shutdown(librgw_t rgw)
58 cdef extern from "rados/rgw_file.h" nogil:
63 RGW_LOOKUP_FLAG_CREATE
74 RGW_READDIR_FLAG_DOTDOT
77 RGW_OPEN_FLAG_V3 # ops have v3 semantics
78 RGW_OPEN_FLAG_STATELESS # alias it
83 ctypedef void *rgw_fh_hk
84 cdef struct rgw_file_handle:
92 # mount info hypothetical--emulate Unix, support at least UUID-length fsid
93 cdef struct rgw_statvfs:
94 uint64_t f_bsize # file system block size
95 uint64_t f_frsize # fragment size
96 uint64_t f_blocks # size of fs in f_frsize units
97 uint64_t f_bfree # free blocks
98 uint64_t f_bavail # free blocks for unprivileged users
99 uint64_t f_files # inodes
100 uint64_t f_ffree # free inodes
101 uint64_t f_favail # free inodes for unprivileged users
102 uint64_t f_fsid[2] # /* file system ID
103 uint64_t f_flag # mount flags
104 uint64_t f_namemax # maximum filename length
106 void rgwfile_version(int *major, int *minor, int *extra)
108 int rgw_lookup(rgw_fs *fs,
109 rgw_file_handle *parent_fh, const char *path,
110 rgw_file_handle **fh, uint32_t flags)
112 int rgw_lookup_handle(rgw_fs *fs, rgw_fh_hk *fh_hk,
113 rgw_file_handle **fh, uint32_t flags)
115 int rgw_fh_rele(rgw_fs *fs, rgw_file_handle *fh,
118 int rgw_mount(librgw_t rgw, const char *uid, const char *key,
119 const char *secret, rgw_fs **fs, uint32_t flags)
121 int rgw_umount(rgw_fs *fs, uint32_t flags)
123 int rgw_statfs(rgw_fs *fs, rgw_file_handle *parent_fh,
124 rgw_statvfs *vfs_st, uint32_t flags)
126 int rgw_create(rgw_fs *fs, rgw_file_handle *parent_fh,
127 const char *name, stat *st, uint32_t mask,
128 rgw_file_handle **fh, uint32_t posix_flags,
131 int rgw_mkdir(rgw_fs *fs,
132 rgw_file_handle *parent_fh,
133 const char *name, stat *st, uint32_t mask,
134 rgw_file_handle **fh, uint32_t flags)
136 int rgw_rename(rgw_fs *fs,
137 rgw_file_handle *olddir, const char* old_name,
138 rgw_file_handle *newdir, const char* new_name,
141 int rgw_unlink(rgw_fs *fs,
142 rgw_file_handle *parent_fh, const char* path,
145 int rgw_readdir(rgw_fs *fs,
146 rgw_file_handle *parent_fh, uint64_t *offset,
147 bool (*cb)(const char *name, void *arg, uint64_t offset) nogil except? -9000,
148 void *cb_arg, bool *eof, uint32_t flags) except? -9000
150 int rgw_getattr(rgw_fs *fs,
151 rgw_file_handle *fh, stat *st,
154 int rgw_setattr(rgw_fs *fs, rgw_file_handle *fh, stat *st,
155 uint32_t mask, uint32_t flags)
157 int rgw_truncate(rgw_fs *fs, rgw_file_handle *fh, uint64_t size, uint32_t flags)
159 int rgw_open(rgw_fs *fs, rgw_file_handle *parent_fh,
160 uint32_t posix_flags, uint32_t flags)
162 int rgw_close(rgw_fs *fs, rgw_file_handle *fh,
165 int rgw_read(rgw_fs *fs,
166 rgw_file_handle *fh, uint64_t offset,
167 size_t length, size_t *bytes_read, void *buffer,
170 int rgw_write(rgw_fs *fs,
171 rgw_file_handle *fh, uint64_t offset,
172 size_t length, size_t *bytes_written, void *buffer,
175 int rgw_fsync(rgw_fs *fs, rgw_file_handle *fh,
178 int rgw_commit(rgw_fs *fs, rgw_file_handle *fh,
179 uint64_t offset, uint64_t length, uint32_t flags)
182 class Error(Exception):
186 class OSError(Error):
187 """ `OSError` class, derived from `Error` """
188 def __init__(self, errno, strerror):
190 self.strerror = strerror
193 return '[Errno {0}] {1}'.format(self.errno, self.strerror)
196 class PermissionError(OSError):
200 class ObjectNotFound(OSError):
208 class ObjectExists(Error):
212 class IOError(OSError):
216 class NoSpace(Error):
220 class InvalidValue(Error):
224 class OperationNotSupported(Error):
228 class IncompleteWriteError(Error):
232 class LibCephFSStateError(Error):
235 class WouldBlock(Error):
238 class OutOfRange(Error):
241 IF UNAME_SYSNAME == "FreeBSD":
242 cdef errno_to_exception = {
243 errno.EPERM : PermissionError,
244 errno.ENOENT : ObjectNotFound,
246 errno.ENOSPC : NoSpace,
247 errno.EEXIST : ObjectExists,
248 errno.ENOATTR : NoData,
249 errno.EINVAL : InvalidValue,
250 errno.EOPNOTSUPP : OperationNotSupported,
251 errno.ERANGE : OutOfRange,
252 errno.EWOULDBLOCK: WouldBlock,
255 cdef errno_to_exception = {
256 errno.EPERM : PermissionError,
257 errno.ENOENT : ObjectNotFound,
259 errno.ENOSPC : NoSpace,
260 errno.EEXIST : ObjectExists,
261 errno.ENODATA : NoData,
262 errno.EINVAL : InvalidValue,
263 errno.EOPNOTSUPP : OperationNotSupported,
264 errno.ERANGE : OutOfRange,
265 errno.EWOULDBLOCK: WouldBlock,
269 cdef class FileHandle(object):
270 cdef rgw_file_handle *handler
273 StatResult = namedtuple('StatResult',
274 ["st_dev", "st_ino", "st_mode", "st_nlink", "st_uid",
275 "st_gid", "st_rdev", "st_size", "st_blksize",
276 "st_blocks", "st_atime", "st_mtime", "st_ctime"])
279 def cstr(val, name, encoding="utf-8", opt=False):
281 Create a byte string from a Python string
283 :param basestring val: Python string
284 :param str name: Name of the string parameter, for exceptions
285 :param str encoding: Encoding to use
286 :param bool opt: If True, None is allowed
288 :raises: :class:`InvalidArgument`
290 if opt and val is None:
292 if isinstance(val, bytes):
294 elif isinstance(val, unicode):
295 return val.encode(encoding)
297 raise TypeError('%s must be a string' % name)
300 cdef make_ex(ret, msg):
302 Translate a librados return code into an exception.
304 :param ret: the return code
306 :param msg: the error message to use
308 :returns: a subclass of :class:`Error`
311 if ret in errno_to_exception:
312 return errno_to_exception[ret](ret, msg)
314 return Error(msg + (": error code %d" % ret))
317 cdef bool readdir_cb(const char *name, void *arg, uint64_t offset) \
318 except? -9000 with gil:
319 if exc.PyErr_Occurred():
321 (<object>arg)(name, offset)
325 class LibCephFSStateError(Error):
329 cdef class LibRGWFS(object):
330 """librgwfs python wrapper"""
332 cdef public object state
333 cdef public object uid
334 cdef public object key
335 cdef public object secret
336 cdef librgw_t cluster
339 def require_state(self, *args):
340 if self.state in args:
342 raise LibCephFSStateError("You cannot perform that operation on a "
343 "RGWFS object in state %s." % (self.state))
345 def __cinit__(self, uid, key, secret):
347 self.state = "umounted"
348 ret = librgw_create(&self.cluster, 0, NULL)
350 raise make_ex(ret, "error calling librgw_create")
351 self.uid = cstr(uid, "uid")
352 self.key = cstr(key, "key")
353 self.secret = cstr(secret, "secret")
357 Unmount and destroy the ceph mount handle.
359 if self.state in ["mounted"]:
361 ret = rgw_umount(self.fs, 0);
363 raise make_ex(ret, "error calling rgw_unmount")
364 self.state = "shutdown"
370 def __exit__(self, type_, value, traceback):
373 def __dealloc__(self):
378 Get the version number of the ``librgwfile`` C library.
380 :returns: a tuple of ``(major, minor, extra)`` components of the
388 rgwfile_version(&major, &minor, &extra)
389 return (major, minor, extra)
392 self.require_state("umounted")
394 char *_uid = self.uid
395 char *_key = self.key
396 char *_secret = self.secret
398 ret = rgw_mount(self.cluster, <const char*>_uid, <const char*>_key,
399 <const char*>_secret, &self.fs, 0)
401 raise make_ex(ret, "error calling rgw_mount")
402 self.state = "mounted"
403 dir_handler = FileHandle()
404 dir_handler.handler = <rgw_file_handle*>self.fs.root_fh
408 self.require_state("mounted")
410 ret = rgw_umount(self.fs, 0)
412 raise make_ex(ret, "error calling rgw_umount")
413 self.state = "umounted"
416 self.require_state("mounted")
421 ret = rgw_statfs(self.fs, <rgw_file_handle*>self.fs.root_fh, &statbuf, 0)
423 raise make_ex(ret, "statfs failed")
424 cdef uint64_t[:] fsid = statbuf.f_fsid
425 return {'f_bsize': statbuf.f_bsize,
426 'f_frsize': statbuf.f_frsize,
427 'f_blocks': statbuf.f_blocks,
428 'f_bfree': statbuf.f_bfree,
429 'f_bavail': statbuf.f_bavail,
430 'f_files': statbuf.f_files,
431 'f_ffree': statbuf.f_ffree,
432 'f_favail': statbuf.f_favail,
434 'f_flag': statbuf.f_flag,
435 'f_namemax': statbuf.f_namemax}
438 def create(self, FileHandle dir_handler, filename, flags = 0):
439 self.require_state("mounted")
441 if not isinstance(flags, int):
442 raise TypeError("flags must be an integer")
444 filename = cstr(filename, 'filename')
447 rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
448 rgw_file_handle *_file_handler
450 char* _filename = filename
454 ret = rgw_create(self.fs, _dir_handler, _filename, &statbuf, 0,
455 &_file_handler, 0, _flags)
457 raise make_ex(ret, "error in create '%s'" % filename)
459 ret = rgw_open(self.fs, _file_handler, 0, _flags)
461 raise make_ex(ret, "error in open '%s'" % filename)
463 file_handler = FileHandle()
464 file_handler.handler = _file_handler
467 def mkdir(self, FileHandle dir_handler, dirname, flags = 0):
468 self.require_state("mounted")
469 dirname = cstr(dirname, 'dirname')
470 new_dir_handler = FileHandle()
472 rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
473 rgw_file_handle *_new_dir_handler
474 char* _dirname = dirname
478 ret = rgw_mkdir(self.fs, _dir_handler, _dirname, &statbuf,
479 0, &_new_dir_handler, _flags)
481 raise make_ex(ret, "error in mkdir '%s'" % dirname)
482 new_dir_handler.handler = _new_dir_handler
483 return new_dir_handler
485 def rename(self, FileHandle src_handler, src_name, FileHandle dst_handler, dst_name, flags = 0):
486 self.require_state("mounted")
488 src_name = cstr(src_name, 'src_name')
489 dst_name = cstr(dst_name, 'dst_name')
492 rgw_file_handle *_src_dir_handler = <rgw_file_handle*>src_handler.handler
493 rgw_file_handle *_dst_dir_handler = <rgw_file_handle*>dst_handler.handler
494 char* _src_name = src_name
495 char* _dst_name = dst_name
499 ret = rgw_rename(self.fs, _src_dir_handler, _src_name,
500 _dst_dir_handler, _dst_name, _flags)
502 raise make_ex(ret, "error in rename '%s' to '%s'" % (src_name,
506 def unlink(self, FileHandle handler, name, flags = 0):
507 self.require_state("mounted")
508 name = cstr(name, 'name')
510 rgw_file_handle *_handler = <rgw_file_handle*>handler.handler
514 ret = rgw_unlink(self.fs, _handler, _name, _flags)
516 raise make_ex(ret, "error in unlink")
519 def readdir(self, FileHandle dir_handler, iterate_cb, offset, flags = 0):
520 self.require_state("mounted")
523 rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
524 uint64_t _offset = offset
526 uint32_t _flags = flags
528 ret = rgw_readdir(self.fs, _dir_handler, &_offset, &readdir_cb,
529 <void *>iterate_cb, &_eof, _flags)
531 raise make_ex(ret, "error in readdir")
533 return (_offset, _eof)
535 def fstat(self, FileHandle file_handler):
536 self.require_state("mounted")
539 rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
543 ret = rgw_getattr(self.fs, _file_handler, &statbuf, 0)
545 raise make_ex(ret, "error in getattr")
546 return StatResult(st_dev=statbuf.st_dev, st_ino=statbuf.st_ino,
547 st_mode=statbuf.st_mode, st_nlink=statbuf.st_nlink,
548 st_uid=statbuf.st_uid, st_gid=statbuf.st_gid,
549 st_rdev=statbuf.st_rdev, st_size=statbuf.st_size,
550 st_blksize=statbuf.st_blksize,
551 st_blocks=statbuf.st_blocks,
552 st_atime=datetime.fromtimestamp(statbuf.st_atime),
553 st_mtime=datetime.fromtimestamp(statbuf.st_mtime),
554 st_ctime=datetime.fromtimestamp(statbuf.st_ctime))
556 def opendir(self, FileHandle dir_handler, dirname, flags = 0):
557 self.require_state("mounted")
559 if not isinstance(flags, int):
560 raise TypeError("flags must be an integer")
562 dirname = cstr(dirname, 'dirname')
565 rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
566 rgw_file_handle *_file_handler
568 char* _dirname = dirname
571 ret = rgw_lookup(self.fs, _dir_handler, _dirname,
572 &_file_handler, _flags)
574 raise make_ex(ret, "error in open '%s'" % dirname)
576 file_handler = FileHandle()
577 file_handler.handler = _file_handler
580 def open(self, FileHandle dir_handler, filename, flags = 0):
581 self.require_state("mounted")
583 if not isinstance(flags, int):
584 raise TypeError("flags must be an integer")
586 filename = cstr(filename, 'filename')
589 rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
590 rgw_file_handle *_file_handler
592 char* _filename = filename
595 ret = rgw_lookup(self.fs, _dir_handler, _filename,
596 &_file_handler, _flags)
598 raise make_ex(ret, "error in open '%s'" % filename)
600 ret = rgw_open(self.fs, _file_handler, 0, _flags)
602 raise make_ex(ret, "error in open '%s'" % filename)
604 file_handler = FileHandle()
605 file_handler.handler = _file_handler
608 def close(self, FileHandle file_handler, flags = 0):
609 self.require_state("mounted")
611 rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
614 ret = rgw_close(self.fs, _file_handler, _flags)
616 raise make_ex(ret, "error in close")
618 def read(self, FileHandle file_handler, offset, l, flags = 0):
619 self.require_state("mounted")
620 if not isinstance(offset, int):
621 raise TypeError('offset must be an int')
622 if not isinstance(l, int):
623 raise TypeError('l must be an int')
625 rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
626 int64_t _offset = offset
632 PyObject* ret_s = NULL
634 ret_s = PyBytes_FromStringAndSize(NULL, _length)
636 ret_buf = PyBytes_AsString(ret_s)
638 ret = rgw_read(self.fs, _file_handler, _offset, _length,
639 &_got, ret_buf, _flags)
641 raise make_ex(ret, "error in read")
644 _PyBytes_Resize(&ret_s, _got)
648 # We DECREF unconditionally: the cast to object above will have
649 # INCREFed if necessary. This also takes care of exceptions,
650 # including if _PyString_Resize fails (that will free the string
651 # itself and set ret_s to NULL, hence XDECREF).
652 ref.Py_XDECREF(ret_s)
655 def write(self, FileHandle file_handler, offset, buf, flags = 0):
656 self.require_state("mounted")
657 if not isinstance(buf, bytes):
658 raise TypeError('buf must be a bytes')
659 if not isinstance(offset, int):
660 raise TypeError('offset must be an int')
663 rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
665 int64_t _offset = offset
667 size_t length = len(buf)
672 ret = rgw_write(self.fs, _file_handler, _offset, length, &_written,
675 raise make_ex(ret, "error in write")
679 def fsync(self, FileHandle handler, flags = 0):
680 self.require_state("mounted")
683 rgw_file_handle *_file_handler = <rgw_file_handle*>handler.handler
686 ret = rgw_fsync(self.fs, _file_handler, _flags)
689 raise make_ex(ret, "fsync failed")