4 * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org>
8 #include <linux/file.h>
9 #include <linux/poll.h>
10 #include <linux/init.h>
12 #include <linux/sched.h>
13 #include <linux/kernel.h>
14 #include <linux/slab.h>
15 #include <linux/list.h>
16 #include <linux/spinlock.h>
17 #include <linux/anon_inodes.h>
18 #include <linux/syscalls.h>
19 #include <linux/export.h>
20 #include <linux/kref.h>
21 #include <linux/eventfd.h>
22 #include <linux/proc_fs.h>
23 #include <linux/seq_file.h>
27 wait_queue_head_t wqh;
29 * Every time that a write(2) is performed on an eventfd, the
30 * value of the __u64 being written is added to "count" and a
31 * wakeup is performed on "wqh". A read(2) will return the "count"
32 * value to userspace, and will reset "count" to zero. The kernel
33 * side eventfd_signal() also, adds to the "count" counter and
41 * eventfd_signal - Adds @n to the eventfd counter.
42 * @ctx: [in] Pointer to the eventfd context.
43 * @n: [in] Value of the counter to be added to the eventfd internal counter.
44 * The value cannot be negative.
46 * This function is supposed to be called by the kernel in paths that do not
47 * allow sleeping. In this function we allow the counter to reach the ULLONG_MAX
48 * value, and we signal this as overflow condition by returining a POLLERR
51 * Returns the amount by which the counter was incrememnted. This will be less
52 * than @n if the counter has overflowed.
54 __u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n)
58 spin_lock_irqsave(&ctx->wqh.lock, flags);
59 if (ULLONG_MAX - ctx->count < n)
60 n = ULLONG_MAX - ctx->count;
62 if (waitqueue_active(&ctx->wqh))
63 wake_up_locked_poll(&ctx->wqh, POLLIN);
64 spin_unlock_irqrestore(&ctx->wqh.lock, flags);
68 EXPORT_SYMBOL_GPL(eventfd_signal);
70 static void eventfd_free_ctx(struct eventfd_ctx *ctx)
75 static void eventfd_free(struct kref *kref)
77 struct eventfd_ctx *ctx = container_of(kref, struct eventfd_ctx, kref);
79 eventfd_free_ctx(ctx);
83 * eventfd_ctx_get - Acquires a reference to the internal eventfd context.
84 * @ctx: [in] Pointer to the eventfd context.
86 * Returns: In case of success, returns a pointer to the eventfd context.
88 struct eventfd_ctx *eventfd_ctx_get(struct eventfd_ctx *ctx)
93 EXPORT_SYMBOL_GPL(eventfd_ctx_get);
96 * eventfd_ctx_put - Releases a reference to the internal eventfd context.
97 * @ctx: [in] Pointer to eventfd context.
99 * The eventfd context reference must have been previously acquired either
100 * with eventfd_ctx_get() or eventfd_ctx_fdget().
102 void eventfd_ctx_put(struct eventfd_ctx *ctx)
104 kref_put(&ctx->kref, eventfd_free);
106 EXPORT_SYMBOL_GPL(eventfd_ctx_put);
108 static int eventfd_release(struct inode *inode, struct file *file)
110 struct eventfd_ctx *ctx = file->private_data;
112 wake_up_poll(&ctx->wqh, POLLHUP);
113 eventfd_ctx_put(ctx);
117 static unsigned int eventfd_poll(struct file *file, poll_table *wait)
119 struct eventfd_ctx *ctx = file->private_data;
120 unsigned int events = 0;
123 poll_wait(file, &ctx->wqh, wait);
129 if (count == ULLONG_MAX)
131 if (ULLONG_MAX - 1 > count)
137 static void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt)
139 *cnt = (ctx->flags & EFD_SEMAPHORE) ? 1 : ctx->count;
144 * eventfd_ctx_remove_wait_queue - Read the current counter and removes wait queue.
145 * @ctx: [in] Pointer to eventfd context.
146 * @wait: [in] Wait queue to be removed.
147 * @cnt: [out] Pointer to the 64-bit counter value.
149 * Returns %0 if successful, or the following error codes:
151 * -EAGAIN : The operation would have blocked.
153 * This is used to atomically remove a wait queue entry from the eventfd wait
154 * queue head, and read/reset the counter value.
156 int eventfd_ctx_remove_wait_queue(struct eventfd_ctx *ctx, wait_queue_t *wait,
161 spin_lock_irqsave(&ctx->wqh.lock, flags);
162 eventfd_ctx_do_read(ctx, cnt);
163 __remove_wait_queue(&ctx->wqh, wait);
164 if (*cnt != 0 && waitqueue_active(&ctx->wqh))
165 wake_up_locked_poll(&ctx->wqh, POLLOUT);
166 spin_unlock_irqrestore(&ctx->wqh.lock, flags);
168 return *cnt != 0 ? 0 : -EAGAIN;
170 EXPORT_SYMBOL_GPL(eventfd_ctx_remove_wait_queue);
173 * eventfd_ctx_read - Reads the eventfd counter or wait if it is zero.
174 * @ctx: [in] Pointer to eventfd context.
175 * @no_wait: [in] Different from zero if the operation should not block.
176 * @cnt: [out] Pointer to the 64-bit counter value.
178 * Returns %0 if successful, or the following error codes:
180 * -EAGAIN : The operation would have blocked but @no_wait was non-zero.
181 * -ERESTARTSYS : A signal interrupted the wait operation.
183 * If @no_wait is zero, the function might sleep until the eventfd internal
184 * counter becomes greater than zero.
186 ssize_t eventfd_ctx_read(struct eventfd_ctx *ctx, int no_wait, __u64 *cnt)
189 DECLARE_WAITQUEUE(wait, current);
191 spin_lock_irq(&ctx->wqh.lock);
197 __add_wait_queue(&ctx->wqh, &wait);
199 set_current_state(TASK_INTERRUPTIBLE);
200 if (ctx->count > 0) {
204 if (signal_pending(current)) {
208 spin_unlock_irq(&ctx->wqh.lock);
210 spin_lock_irq(&ctx->wqh.lock);
212 __remove_wait_queue(&ctx->wqh, &wait);
213 __set_current_state(TASK_RUNNING);
215 if (likely(res == 0)) {
216 eventfd_ctx_do_read(ctx, cnt);
217 if (waitqueue_active(&ctx->wqh))
218 wake_up_locked_poll(&ctx->wqh, POLLOUT);
220 spin_unlock_irq(&ctx->wqh.lock);
224 EXPORT_SYMBOL_GPL(eventfd_ctx_read);
226 static ssize_t eventfd_read(struct file *file, char __user *buf, size_t count,
229 struct eventfd_ctx *ctx = file->private_data;
233 if (count < sizeof(cnt))
235 res = eventfd_ctx_read(ctx, file->f_flags & O_NONBLOCK, &cnt);
239 return put_user(cnt, (__u64 __user *) buf) ? -EFAULT : sizeof(cnt);
242 static ssize_t eventfd_write(struct file *file, const char __user *buf, size_t count,
245 struct eventfd_ctx *ctx = file->private_data;
248 DECLARE_WAITQUEUE(wait, current);
250 if (count < sizeof(ucnt))
252 if (copy_from_user(&ucnt, buf, sizeof(ucnt)))
254 if (ucnt == ULLONG_MAX)
256 spin_lock_irq(&ctx->wqh.lock);
258 if (ULLONG_MAX - ctx->count > ucnt)
260 else if (!(file->f_flags & O_NONBLOCK)) {
261 __add_wait_queue(&ctx->wqh, &wait);
263 set_current_state(TASK_INTERRUPTIBLE);
264 if (ULLONG_MAX - ctx->count > ucnt) {
268 if (signal_pending(current)) {
272 spin_unlock_irq(&ctx->wqh.lock);
274 spin_lock_irq(&ctx->wqh.lock);
276 __remove_wait_queue(&ctx->wqh, &wait);
277 __set_current_state(TASK_RUNNING);
279 if (likely(res > 0)) {
281 if (waitqueue_active(&ctx->wqh))
282 wake_up_locked_poll(&ctx->wqh, POLLIN);
284 spin_unlock_irq(&ctx->wqh.lock);
289 #ifdef CONFIG_PROC_FS
290 static void eventfd_show_fdinfo(struct seq_file *m, struct file *f)
292 struct eventfd_ctx *ctx = f->private_data;
294 spin_lock_irq(&ctx->wqh.lock);
295 seq_printf(m, "eventfd-count: %16llx\n",
296 (unsigned long long)ctx->count);
297 spin_unlock_irq(&ctx->wqh.lock);
301 static const struct file_operations eventfd_fops = {
302 #ifdef CONFIG_PROC_FS
303 .show_fdinfo = eventfd_show_fdinfo,
305 .release = eventfd_release,
306 .poll = eventfd_poll,
307 .read = eventfd_read,
308 .write = eventfd_write,
309 .llseek = noop_llseek,
313 * eventfd_fget - Acquire a reference of an eventfd file descriptor.
314 * @fd: [in] Eventfd file descriptor.
316 * Returns a pointer to the eventfd file structure in case of success, or the
317 * following error pointer:
319 * -EBADF : Invalid @fd file descriptor.
320 * -EINVAL : The @fd file descriptor is not an eventfd file.
322 struct file *eventfd_fget(int fd)
328 return ERR_PTR(-EBADF);
329 if (file->f_op != &eventfd_fops) {
331 return ERR_PTR(-EINVAL);
336 EXPORT_SYMBOL_GPL(eventfd_fget);
339 * eventfd_ctx_fdget - Acquires a reference to the internal eventfd context.
340 * @fd: [in] Eventfd file descriptor.
342 * Returns a pointer to the internal eventfd context, otherwise the error
343 * pointers returned by the following functions:
347 struct eventfd_ctx *eventfd_ctx_fdget(int fd)
349 struct eventfd_ctx *ctx;
350 struct fd f = fdget(fd);
352 return ERR_PTR(-EBADF);
353 ctx = eventfd_ctx_fileget(f.file);
357 EXPORT_SYMBOL_GPL(eventfd_ctx_fdget);
360 * eventfd_ctx_fileget - Acquires a reference to the internal eventfd context.
361 * @file: [in] Eventfd file pointer.
363 * Returns a pointer to the internal eventfd context, otherwise the error
366 * -EINVAL : The @fd file descriptor is not an eventfd file.
368 struct eventfd_ctx *eventfd_ctx_fileget(struct file *file)
370 if (file->f_op != &eventfd_fops)
371 return ERR_PTR(-EINVAL);
373 return eventfd_ctx_get(file->private_data);
375 EXPORT_SYMBOL_GPL(eventfd_ctx_fileget);
378 * eventfd_file_create - Creates an eventfd file pointer.
379 * @count: Initial eventfd counter value.
380 * @flags: Flags for the eventfd file.
382 * This function creates an eventfd file pointer, w/out installing it into
383 * the fd table. This is useful when the eventfd file is used during the
384 * initialization of data structures that require extra setup after the eventfd
385 * creation. So the eventfd creation is split into the file pointer creation
386 * phase, and the file descriptor installation phase.
387 * In this way races with userspace closing the newly installed file descriptor
389 * Returns an eventfd file pointer, or a proper error pointer.
391 struct file *eventfd_file_create(unsigned int count, int flags)
394 struct eventfd_ctx *ctx;
396 /* Check the EFD_* constants for consistency. */
397 BUILD_BUG_ON(EFD_CLOEXEC != O_CLOEXEC);
398 BUILD_BUG_ON(EFD_NONBLOCK != O_NONBLOCK);
400 if (flags & ~EFD_FLAGS_SET)
401 return ERR_PTR(-EINVAL);
403 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
405 return ERR_PTR(-ENOMEM);
407 kref_init(&ctx->kref);
408 init_waitqueue_head(&ctx->wqh);
412 file = anon_inode_getfile("[eventfd]", &eventfd_fops, ctx,
413 O_RDWR | (flags & EFD_SHARED_FCNTL_FLAGS));
415 eventfd_free_ctx(ctx);
420 SYSCALL_DEFINE2(eventfd2, unsigned int, count, int, flags)
425 error = get_unused_fd_flags(flags & EFD_SHARED_FCNTL_FLAGS);
430 file = eventfd_file_create(count, flags);
432 error = PTR_ERR(file);
433 goto err_put_unused_fd;
435 fd_install(fd, file);
445 SYSCALL_DEFINE1(eventfd, unsigned int, count)
447 return sys_eventfd2(count, 0);