2 * GlusterFS backend for QEMU
4 * Copyright (C) 2012 Bharata B Rao <bharata@linux.vnet.ibm.com>
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
10 #include "qemu/osdep.h"
11 #include <glusterfs/api/glfs.h>
12 #include "block/block_int.h"
13 #include "qapi/error.h"
16 typedef struct GlusterAIOCB {
21 AioContext *aio_context;
24 typedef struct BDRVGlusterState {
29 typedef struct GlusterConf {
37 static void qemu_gluster_gconf_free(GlusterConf *gconf)
40 g_free(gconf->server);
41 g_free(gconf->volname);
43 g_free(gconf->transport);
48 static int parse_volume_options(GlusterConf *gconf, char *path)
57 p = q = path + strspn(path, "/");
62 gconf->volname = g_strndup(q, p - q);
69 gconf->image = g_strdup(p);
74 * file=gluster[+transport]://[server[:port]]/volname/image[?socket=...]
76 * 'gluster' is the protocol.
78 * 'transport' specifies the transport type used to connect to gluster
79 * management daemon (glusterd). Valid transport types are
80 * tcp, unix and rdma. If a transport type isn't specified, then tcp
83 * 'server' specifies the server where the volume file specification for
84 * the given volume resides. This can be either hostname, ipv4 address
85 * or ipv6 address. ipv6 address needs to be within square brackets [ ].
86 * If transport type is 'unix', then 'server' field should not be specified.
87 * The 'socket' field needs to be populated with the path to unix domain
90 * 'port' is the port number on which glusterd is listening. This is optional
91 * and if not specified, QEMU will send 0 which will make gluster to use the
92 * default port. If the transport type is unix, then 'port' should not be
95 * 'volname' is the name of the gluster volume which contains the VM image.
97 * 'image' is the path to the actual VM image that resides on gluster volume.
101 * file=gluster://1.2.3.4/testvol/a.img
102 * file=gluster+tcp://1.2.3.4/testvol/a.img
103 * file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img
104 * file=gluster+tcp://[1:2:3:4:5:6:7:8]/testvol/dir/a.img
105 * file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img
106 * file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img
107 * file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
108 * file=gluster+rdma://1.2.3.4:24007/testvol/a.img
110 static int qemu_gluster_parseuri(GlusterConf *gconf, const char *filename)
113 QueryParams *qp = NULL;
114 bool is_unix = false;
117 uri = uri_parse(filename);
123 if (!uri->scheme || !strcmp(uri->scheme, "gluster")) {
124 gconf->transport = g_strdup("tcp");
125 } else if (!strcmp(uri->scheme, "gluster+tcp")) {
126 gconf->transport = g_strdup("tcp");
127 } else if (!strcmp(uri->scheme, "gluster+unix")) {
128 gconf->transport = g_strdup("unix");
130 } else if (!strcmp(uri->scheme, "gluster+rdma")) {
131 gconf->transport = g_strdup("rdma");
137 ret = parse_volume_options(gconf, uri->path);
142 qp = query_params_parse(uri->query);
143 if (qp->n > 1 || (is_unix && !qp->n) || (!is_unix && qp->n)) {
149 if (uri->server || uri->port) {
153 if (strcmp(qp->p[0].name, "socket")) {
157 gconf->server = g_strdup(qp->p[0].value);
159 gconf->server = g_strdup(uri->server ? uri->server : "localhost");
160 gconf->port = uri->port;
165 query_params_free(qp);
171 static struct glfs *qemu_gluster_init(GlusterConf *gconf, const char *filename,
174 struct glfs *glfs = NULL;
178 ret = qemu_gluster_parseuri(gconf, filename);
180 error_setg(errp, "Usage: file=gluster[+transport]://[server[:port]]/"
181 "volname/image[?socket=...]");
186 glfs = glfs_new(gconf->volname);
191 ret = glfs_set_volfile_server(glfs, gconf->transport, gconf->server,
198 * TODO: Use GF_LOG_ERROR instead of hard code value of 4 here when
199 * GlusterFS makes GF_LOG_* macros available to libgfapi users.
201 ret = glfs_set_logging(glfs, "-", 4);
206 ret = glfs_init(glfs);
208 error_setg_errno(errp, errno,
209 "Gluster connection failed for server=%s port=%d "
210 "volume=%s image=%s transport=%s", gconf->server,
211 gconf->port, gconf->volname, gconf->image,
214 /* glfs_init sometimes doesn't set errno although docs suggest that */
231 static void qemu_gluster_complete_aio(void *opaque)
233 GlusterAIOCB *acb = (GlusterAIOCB *)opaque;
235 qemu_bh_delete(acb->bh);
237 qemu_coroutine_enter(acb->coroutine, NULL);
241 * AIO callback routine called from GlusterFS thread.
243 static void gluster_finish_aiocb(struct glfs_fd *fd, ssize_t ret, void *arg)
245 GlusterAIOCB *acb = (GlusterAIOCB *)arg;
247 if (!ret || ret == acb->size) {
248 acb->ret = 0; /* Success */
249 } else if (ret < 0) {
250 acb->ret = -errno; /* Read/Write failed */
252 acb->ret = -EIO; /* Partial read/write - fail it */
255 acb->bh = aio_bh_new(acb->aio_context, qemu_gluster_complete_aio, acb);
256 qemu_bh_schedule(acb->bh);
259 /* TODO Convert to fine grained options */
260 static QemuOptsList runtime_opts = {
262 .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
266 .type = QEMU_OPT_STRING,
267 .help = "URL to the gluster image",
269 { /* end of list */ }
273 static void qemu_gluster_parse_flags(int bdrv_flags, int *open_flags)
275 assert(open_flags != NULL);
277 *open_flags |= O_BINARY;
279 if (bdrv_flags & BDRV_O_RDWR) {
280 *open_flags |= O_RDWR;
282 *open_flags |= O_RDONLY;
285 if ((bdrv_flags & BDRV_O_NOCACHE)) {
286 *open_flags |= O_DIRECT;
290 static int qemu_gluster_open(BlockDriverState *bs, QDict *options,
291 int bdrv_flags, Error **errp)
293 BDRVGlusterState *s = bs->opaque;
296 GlusterConf *gconf = g_new0(GlusterConf, 1);
298 Error *local_err = NULL;
299 const char *filename;
301 opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
302 qemu_opts_absorb_qdict(opts, options, &local_err);
304 error_propagate(errp, local_err);
309 filename = qemu_opt_get(opts, "filename");
311 s->glfs = qemu_gluster_init(gconf, filename, errp);
317 #ifdef CONFIG_GLUSTERFS_XLATOR_OPT
318 /* Without this, if fsync fails for a recoverable reason (for instance,
319 * ENOSPC), gluster will dump its cache, preventing retries. This means
320 * almost certain data loss. Not all gluster versions support the
321 * 'resync-failed-syncs-after-fsync' key value, but there is no way to
322 * discover during runtime if it is supported (this api returns success for
323 * unknown key/value pairs) */
324 ret = glfs_set_xlator_option(s->glfs, "*-write-behind",
325 "resync-failed-syncs-after-fsync",
328 error_setg_errno(errp, errno, "Unable to set xlator key/value pair");
334 qemu_gluster_parse_flags(bdrv_flags, &open_flags);
336 s->fd = glfs_open(s->glfs, gconf->image, open_flags);
343 qemu_gluster_gconf_free(gconf);
356 typedef struct BDRVGlusterReopenState {
359 } BDRVGlusterReopenState;
362 static int qemu_gluster_reopen_prepare(BDRVReopenState *state,
363 BlockReopenQueue *queue, Error **errp)
366 BDRVGlusterReopenState *reop_s;
367 GlusterConf *gconf = NULL;
370 assert(state != NULL);
371 assert(state->bs != NULL);
373 state->opaque = g_new0(BDRVGlusterReopenState, 1);
374 reop_s = state->opaque;
376 qemu_gluster_parse_flags(state->flags, &open_flags);
378 gconf = g_new0(GlusterConf, 1);
380 reop_s->glfs = qemu_gluster_init(gconf, state->bs->filename, errp);
381 if (reop_s->glfs == NULL) {
386 #ifdef CONFIG_GLUSTERFS_XLATOR_OPT
387 ret = glfs_set_xlator_option(reop_s->glfs, "*-write-behind",
388 "resync-failed-syncs-after-fsync", "on");
390 error_setg_errno(errp, errno, "Unable to set xlator key/value pair");
396 reop_s->fd = glfs_open(reop_s->glfs, gconf->image, open_flags);
397 if (reop_s->fd == NULL) {
398 /* reops->glfs will be cleaned up in _abort */
404 /* state->opaque will be freed in either the _abort or _commit */
405 qemu_gluster_gconf_free(gconf);
409 static void qemu_gluster_reopen_commit(BDRVReopenState *state)
411 BDRVGlusterReopenState *reop_s = state->opaque;
412 BDRVGlusterState *s = state->bs->opaque;
423 /* use the newly opened image / connection */
425 s->glfs = reop_s->glfs;
427 g_free(state->opaque);
428 state->opaque = NULL;
434 static void qemu_gluster_reopen_abort(BDRVReopenState *state)
436 BDRVGlusterReopenState *reop_s = state->opaque;
438 if (reop_s == NULL) {
443 glfs_close(reop_s->fd);
447 glfs_fini(reop_s->glfs);
450 g_free(state->opaque);
451 state->opaque = NULL;
456 #ifdef CONFIG_GLUSTERFS_ZEROFILL
457 static coroutine_fn int qemu_gluster_co_write_zeroes(BlockDriverState *bs,
458 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
462 BDRVGlusterState *s = bs->opaque;
463 off_t size = nb_sectors * BDRV_SECTOR_SIZE;
464 off_t offset = sector_num * BDRV_SECTOR_SIZE;
468 acb.coroutine = qemu_coroutine_self();
469 acb.aio_context = bdrv_get_aio_context(bs);
471 ret = glfs_zerofill_async(s->fd, offset, size, gluster_finish_aiocb, &acb);
476 qemu_coroutine_yield();
480 static inline bool gluster_supports_zerofill(void)
485 static inline int qemu_gluster_zerofill(struct glfs_fd *fd, int64_t offset,
488 return glfs_zerofill(fd, offset, size);
492 static inline bool gluster_supports_zerofill(void)
497 static inline int qemu_gluster_zerofill(struct glfs_fd *fd, int64_t offset,
504 static int qemu_gluster_create(const char *filename,
505 QemuOpts *opts, Error **errp)
511 int64_t total_size = 0;
513 GlusterConf *gconf = g_new0(GlusterConf, 1);
515 glfs = qemu_gluster_init(gconf, filename, errp);
521 total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
524 tmp = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
525 if (!tmp || !strcmp(tmp, "off")) {
527 } else if (!strcmp(tmp, "full") &&
528 gluster_supports_zerofill()) {
531 error_setg(errp, "Invalid preallocation mode: '%s'"
532 " or GlusterFS doesn't support zerofill API",
538 fd = glfs_creat(glfs, gconf->image,
539 O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRUSR | S_IWUSR);
543 if (!glfs_ftruncate(fd, total_size)) {
544 if (prealloc && qemu_gluster_zerofill(fd, 0, total_size)) {
551 if (glfs_close(fd) != 0) {
557 qemu_gluster_gconf_free(gconf);
564 static coroutine_fn int qemu_gluster_co_rw(BlockDriverState *bs,
565 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, int write)
569 BDRVGlusterState *s = bs->opaque;
570 size_t size = nb_sectors * BDRV_SECTOR_SIZE;
571 off_t offset = sector_num * BDRV_SECTOR_SIZE;
575 acb.coroutine = qemu_coroutine_self();
576 acb.aio_context = bdrv_get_aio_context(bs);
579 ret = glfs_pwritev_async(s->fd, qiov->iov, qiov->niov, offset, 0,
580 gluster_finish_aiocb, &acb);
582 ret = glfs_preadv_async(s->fd, qiov->iov, qiov->niov, offset, 0,
583 gluster_finish_aiocb, &acb);
590 qemu_coroutine_yield();
594 static int qemu_gluster_truncate(BlockDriverState *bs, int64_t offset)
597 BDRVGlusterState *s = bs->opaque;
599 ret = glfs_ftruncate(s->fd, offset);
607 static coroutine_fn int qemu_gluster_co_readv(BlockDriverState *bs,
608 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
610 return qemu_gluster_co_rw(bs, sector_num, nb_sectors, qiov, 0);
613 static coroutine_fn int qemu_gluster_co_writev(BlockDriverState *bs,
614 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
616 return qemu_gluster_co_rw(bs, sector_num, nb_sectors, qiov, 1);
619 static void qemu_gluster_close(BlockDriverState *bs)
621 BDRVGlusterState *s = bs->opaque;
630 static coroutine_fn int qemu_gluster_co_flush_to_disk(BlockDriverState *bs)
634 BDRVGlusterState *s = bs->opaque;
638 acb.coroutine = qemu_coroutine_self();
639 acb.aio_context = bdrv_get_aio_context(bs);
641 ret = glfs_fsync_async(s->fd, gluster_finish_aiocb, &acb);
647 qemu_coroutine_yield();
656 /* Some versions of Gluster (3.5.6 -> 3.5.8?) will not retain its cache
657 * after a fsync failure, so we have no way of allowing the guest to safely
658 * continue. Gluster versions prior to 3.5.6 don't retain the cache
659 * either, but will invalidate the fd on error, so this is again our only
662 * The 'resync-failed-syncs-after-fsync' xlator option for the
663 * write-behind cache will cause later gluster versions to retain its
664 * cache after error, so long as the fd remains open. However, we
665 * currently have no way of knowing if this option is supported.
667 * TODO: Once gluster provides a way for us to determine if the option
668 * is supported, bypass the closure and setting drv to NULL. */
669 qemu_gluster_close(bs);
674 #ifdef CONFIG_GLUSTERFS_DISCARD
675 static coroutine_fn int qemu_gluster_co_discard(BlockDriverState *bs,
676 int64_t sector_num, int nb_sectors)
680 BDRVGlusterState *s = bs->opaque;
681 size_t size = nb_sectors * BDRV_SECTOR_SIZE;
682 off_t offset = sector_num * BDRV_SECTOR_SIZE;
686 acb.coroutine = qemu_coroutine_self();
687 acb.aio_context = bdrv_get_aio_context(bs);
689 ret = glfs_discard_async(s->fd, offset, size, gluster_finish_aiocb, &acb);
694 qemu_coroutine_yield();
699 static int64_t qemu_gluster_getlength(BlockDriverState *bs)
701 BDRVGlusterState *s = bs->opaque;
704 ret = glfs_lseek(s->fd, 0, SEEK_END);
712 static int64_t qemu_gluster_allocated_file_size(BlockDriverState *bs)
714 BDRVGlusterState *s = bs->opaque;
718 ret = glfs_fstat(s->fd, &st);
722 return st.st_blocks * 512;
726 static int qemu_gluster_has_zero_init(BlockDriverState *bs)
728 /* GlusterFS volume could be backed by a block device */
732 static QemuOptsList qemu_gluster_create_opts = {
733 .name = "qemu-gluster-create-opts",
734 .head = QTAILQ_HEAD_INITIALIZER(qemu_gluster_create_opts.head),
737 .name = BLOCK_OPT_SIZE,
738 .type = QEMU_OPT_SIZE,
739 .help = "Virtual disk size"
742 .name = BLOCK_OPT_PREALLOC,
743 .type = QEMU_OPT_STRING,
744 .help = "Preallocation mode (allowed values: off, full)"
746 { /* end of list */ }
750 static BlockDriver bdrv_gluster = {
751 .format_name = "gluster",
752 .protocol_name = "gluster",
753 .instance_size = sizeof(BDRVGlusterState),
754 .bdrv_needs_filename = true,
755 .bdrv_file_open = qemu_gluster_open,
756 .bdrv_reopen_prepare = qemu_gluster_reopen_prepare,
757 .bdrv_reopen_commit = qemu_gluster_reopen_commit,
758 .bdrv_reopen_abort = qemu_gluster_reopen_abort,
759 .bdrv_close = qemu_gluster_close,
760 .bdrv_create = qemu_gluster_create,
761 .bdrv_getlength = qemu_gluster_getlength,
762 .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
763 .bdrv_truncate = qemu_gluster_truncate,
764 .bdrv_co_readv = qemu_gluster_co_readv,
765 .bdrv_co_writev = qemu_gluster_co_writev,
766 .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
767 .bdrv_has_zero_init = qemu_gluster_has_zero_init,
768 #ifdef CONFIG_GLUSTERFS_DISCARD
769 .bdrv_co_discard = qemu_gluster_co_discard,
771 #ifdef CONFIG_GLUSTERFS_ZEROFILL
772 .bdrv_co_write_zeroes = qemu_gluster_co_write_zeroes,
774 .create_opts = &qemu_gluster_create_opts,
777 static BlockDriver bdrv_gluster_tcp = {
778 .format_name = "gluster",
779 .protocol_name = "gluster+tcp",
780 .instance_size = sizeof(BDRVGlusterState),
781 .bdrv_needs_filename = true,
782 .bdrv_file_open = qemu_gluster_open,
783 .bdrv_reopen_prepare = qemu_gluster_reopen_prepare,
784 .bdrv_reopen_commit = qemu_gluster_reopen_commit,
785 .bdrv_reopen_abort = qemu_gluster_reopen_abort,
786 .bdrv_close = qemu_gluster_close,
787 .bdrv_create = qemu_gluster_create,
788 .bdrv_getlength = qemu_gluster_getlength,
789 .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
790 .bdrv_truncate = qemu_gluster_truncate,
791 .bdrv_co_readv = qemu_gluster_co_readv,
792 .bdrv_co_writev = qemu_gluster_co_writev,
793 .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
794 .bdrv_has_zero_init = qemu_gluster_has_zero_init,
795 #ifdef CONFIG_GLUSTERFS_DISCARD
796 .bdrv_co_discard = qemu_gluster_co_discard,
798 #ifdef CONFIG_GLUSTERFS_ZEROFILL
799 .bdrv_co_write_zeroes = qemu_gluster_co_write_zeroes,
801 .create_opts = &qemu_gluster_create_opts,
804 static BlockDriver bdrv_gluster_unix = {
805 .format_name = "gluster",
806 .protocol_name = "gluster+unix",
807 .instance_size = sizeof(BDRVGlusterState),
808 .bdrv_needs_filename = true,
809 .bdrv_file_open = qemu_gluster_open,
810 .bdrv_reopen_prepare = qemu_gluster_reopen_prepare,
811 .bdrv_reopen_commit = qemu_gluster_reopen_commit,
812 .bdrv_reopen_abort = qemu_gluster_reopen_abort,
813 .bdrv_close = qemu_gluster_close,
814 .bdrv_create = qemu_gluster_create,
815 .bdrv_getlength = qemu_gluster_getlength,
816 .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
817 .bdrv_truncate = qemu_gluster_truncate,
818 .bdrv_co_readv = qemu_gluster_co_readv,
819 .bdrv_co_writev = qemu_gluster_co_writev,
820 .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
821 .bdrv_has_zero_init = qemu_gluster_has_zero_init,
822 #ifdef CONFIG_GLUSTERFS_DISCARD
823 .bdrv_co_discard = qemu_gluster_co_discard,
825 #ifdef CONFIG_GLUSTERFS_ZEROFILL
826 .bdrv_co_write_zeroes = qemu_gluster_co_write_zeroes,
828 .create_opts = &qemu_gluster_create_opts,
831 static BlockDriver bdrv_gluster_rdma = {
832 .format_name = "gluster",
833 .protocol_name = "gluster+rdma",
834 .instance_size = sizeof(BDRVGlusterState),
835 .bdrv_needs_filename = true,
836 .bdrv_file_open = qemu_gluster_open,
837 .bdrv_reopen_prepare = qemu_gluster_reopen_prepare,
838 .bdrv_reopen_commit = qemu_gluster_reopen_commit,
839 .bdrv_reopen_abort = qemu_gluster_reopen_abort,
840 .bdrv_close = qemu_gluster_close,
841 .bdrv_create = qemu_gluster_create,
842 .bdrv_getlength = qemu_gluster_getlength,
843 .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
844 .bdrv_truncate = qemu_gluster_truncate,
845 .bdrv_co_readv = qemu_gluster_co_readv,
846 .bdrv_co_writev = qemu_gluster_co_writev,
847 .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
848 .bdrv_has_zero_init = qemu_gluster_has_zero_init,
849 #ifdef CONFIG_GLUSTERFS_DISCARD
850 .bdrv_co_discard = qemu_gluster_co_discard,
852 #ifdef CONFIG_GLUSTERFS_ZEROFILL
853 .bdrv_co_write_zeroes = qemu_gluster_co_write_zeroes,
855 .create_opts = &qemu_gluster_create_opts,
858 static void bdrv_gluster_init(void)
860 bdrv_register(&bdrv_gluster_rdma);
861 bdrv_register(&bdrv_gluster_unix);
862 bdrv_register(&bdrv_gluster_tcp);
863 bdrv_register(&bdrv_gluster);
866 block_init(bdrv_gluster_init);