These changes are the raw update to linux-4.4.6-rt14. Kernel sources

[kvmfornfv.git] / kernel / fs / direct-io.c

diff --git a/kernel/fs/direct-io.c b/kernel/fs/direct-io.c
index 745d234..01171d8 100644 (file)
--- a/kernel/fs/direct-io.c
+++ b/kernel/fs/direct-io.c
@@ -109,6 +109,8 @@ struct dio_submit {
 struct dio {
        int flags;                      /* doesn't change */
        int rw;
+       blk_qc_t bio_cookie;
+       struct block_device *bio_bdev;
        struct inode *inode;
        loff_t i_size;                  /* i_size when submitted */
        dio_iodone_t *end_io;           /* IO completion function */
@@ -120,6 +122,7 @@ struct dio {
        int page_errors;                /* errno from get_user_pages() */
        int is_async;                   /* is IO async ? */
        bool defer_completion;          /* defer AIO completion to workqueue? */
+       bool should_dirty;              /* if pages should be dirtied */
        int io_error;                   /* IO error in completion path */
        unsigned long refcount;         /* direct_io_worker() and bios */
        struct bio *bio_list;           /* singly linked via bi_private */
@@ -285,7 +288,7 @@ static int dio_bio_complete(struct dio *dio, struct bio *bio);
 /*
  * Asynchronous IO callback. 
  */
-static void dio_bio_end_aio(struct bio *bio, int error)
+static void dio_bio_end_aio(struct bio *bio)
 {
        struct dio *dio = bio->bi_private;
        unsigned long remaining;
@@ -318,7 +321,7 @@ static void dio_bio_end_aio(struct bio *bio, int error)
  * During I/O bi_private points at the dio.  After I/O, bi_private is used to
  * implement a singly-linked list of completed BIOs, at dio->bio_list.
  */
-static void dio_bio_end_io(struct bio *bio, int error)
+static void dio_bio_end_io(struct bio *bio)
 {
        struct dio *dio = bio->bi_private;
        unsigned long flags;
@@ -345,9 +348,9 @@ void dio_end_io(struct bio *bio, int error)
        struct dio *dio = bio->bi_private;
 
        if (dio->is_async)
-               dio_bio_end_aio(bio, error);
+               dio_bio_end_aio(bio);
        else
-               dio_bio_end_io(bio, error);
+               dio_bio_end_io(bio);
 }
 EXPORT_SYMBOL_GPL(dio_end_io);
 
@@ -360,7 +363,7 @@ dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,
 
        /*
         * bio_alloc() is guaranteed to return a bio when called with
-        * __GFP_WAIT and we request a valid number of vectors.
+        * __GFP_RECLAIM and we request a valid number of vectors.
         */
        bio = bio_alloc(GFP_KERNEL, nr_vecs);
 
@@ -393,14 +396,17 @@ static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)
        dio->refcount++;
        spin_unlock_irqrestore(&dio->bio_lock, flags);
 
-       if (dio->is_async && dio->rw == READ)
+       if (dio->is_async && dio->rw == READ && dio->should_dirty)
                bio_set_pages_dirty(bio);
 
-       if (sdio->submit_io)
+       dio->bio_bdev = bio->bi_bdev;
+
+       if (sdio->submit_io) {
                sdio->submit_io(dio->rw, bio, dio->inode,
                               sdio->logical_offset_in_bio);
-       else
-               submit_bio(dio->rw, bio);
+               dio->bio_cookie = BLK_QC_T_NONE;
+       } else
+               dio->bio_cookie = submit_bio(dio->rw, bio);
 
        sdio->bio = NULL;
        sdio->boundary = 0;
@@ -439,7 +445,8 @@ static struct bio *dio_await_one(struct dio *dio)
                __set_current_state(TASK_UNINTERRUPTIBLE);
                dio->waiter = current;
                spin_unlock_irqrestore(&dio->bio_lock, flags);
-               io_schedule();
+               if (!blk_poll(bdev_get_queue(dio->bio_bdev), dio->bio_cookie))
+                       io_schedule();
                /* wake up sets us TASK_RUNNING */
                spin_lock_irqsave(&dio->bio_lock, flags);
                dio->waiter = NULL;
@@ -457,26 +464,29 @@ static struct bio *dio_await_one(struct dio *dio)
  */
 static int dio_bio_complete(struct dio *dio, struct bio *bio)
 {
-       const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
        struct bio_vec *bvec;
        unsigned i;
+       int err;
 
-       if (!uptodate)
+       if (bio->bi_error)
                dio->io_error = -EIO;
 
-       if (dio->is_async && dio->rw == READ) {
+       if (dio->is_async && dio->rw == READ && dio->should_dirty) {
+               err = bio->bi_error;
                bio_check_pages_dirty(bio);     /* transfers ownership */
        } else {
                bio_for_each_segment_all(bvec, bio, i) {
                        struct page *page = bvec->bv_page;
 
-                       if (dio->rw == READ && !PageCompound(page))
+                       if (dio->rw == READ && !PageCompound(page) &&
+                                       dio->should_dirty)
                                set_page_dirty_lock(page);
                        page_cache_release(page);
                }
+               err = bio->bi_error;
                bio_put(bio);
        }
-       return uptodate ? 0 : -EIO;
+       return err;
 }
 
 /*
@@ -653,7 +663,7 @@ static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio,
        if (ret)
                goto out;
        sector = start_sector << (sdio->blkbits - 9);
-       nr_pages = min(sdio->pages_in_io, bio_get_nr_vecs(map_bh->b_bdev));
+       nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES);
        BUG_ON(nr_pages <= 0);
        dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages);
        sdio->boundary = 0;
@@ -1159,6 +1169,16 @@ do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
                }
        }
 
+       /* Once we sampled i_size check for reads beyond EOF */
+       dio->i_size = i_size_read(inode);
+       if (iov_iter_rw(iter) == READ && offset >= dio->i_size) {
+               if (dio->flags & DIO_LOCKING)
+                       mutex_unlock(&inode->i_mutex);
+               kmem_cache_free(dio_cache, dio);
+               retval = 0;
+               goto out;
+       }
+
        /*
         * For file extending writes updating i_size before data writeouts
         * complete can expose uninitialized blocks in dumb filesystems.
@@ -1212,11 +1232,11 @@ do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
        sdio.next_block_for_io = -1;
 
        dio->iocb = iocb;
-       dio->i_size = i_size_read(inode);
 
        spin_lock_init(&dio->bio_lock);
        dio->refcount = 1;
 
+       dio->should_dirty = (iter->type == ITER_IOVEC);
        sdio.iter = iter;
        sdio.final_block_in_request =
                (offset + iov_iter_count(iter)) >> blkbits;