diff options
Diffstat (limited to 'drivers/gpio/gpiolib-cdev.c')
| -rw-r--r-- | drivers/gpio/gpiolib-cdev.c | 1427 |
1 files changed, 933 insertions, 494 deletions
diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index c7b5446d01fd..3735c9fe1502 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -5,6 +5,7 @@ #include <linux/bitmap.h> #include <linux/build_bug.h> #include <linux/cdev.h> +#include <linux/cleanup.h> #include <linux/compat.h> #include <linux/compiler.h> #include <linux/device.h> @@ -12,18 +13,22 @@ #include <linux/file.h> #include <linux/gpio.h> #include <linux/gpio/driver.h> +#include <linux/hte.h> #include <linux/interrupt.h> #include <linux/irqreturn.h> -#include <linux/kernel.h> #include <linux/kfifo.h> #include <linux/module.h> #include <linux/mutex.h> +#include <linux/overflow.h> #include <linux/pinctrl/consumer.h> #include <linux/poll.h> +#include <linux/seq_file.h> #include <linux/spinlock.h> +#include <linux/string.h> #include <linux/timekeeping.h> #include <linux/uaccess.h> #include <linux/workqueue.h> + #include <uapi/linux/gpio.h> #include "gpiolib.h" @@ -83,6 +88,10 @@ struct linehandle_state { GPIOHANDLE_REQUEST_OPEN_DRAIN | \ GPIOHANDLE_REQUEST_OPEN_SOURCE) +#define GPIOHANDLE_REQUEST_DIRECTION_FLAGS \ + (GPIOHANDLE_REQUEST_INPUT | \ + GPIOHANDLE_REQUEST_OUTPUT) + static int linehandle_validate_flags(u32 flags) { /* Return an error if an unknown flag is set */ @@ -133,18 +142,22 @@ static int linehandle_validate_flags(u32 flags) static void linehandle_flags_to_desc_flags(u32 lflags, unsigned long *flagsp) { - assign_bit(FLAG_ACTIVE_LOW, flagsp, + unsigned long flags = READ_ONCE(*flagsp); + + assign_bit(GPIOD_FLAG_ACTIVE_LOW, &flags, lflags & GPIOHANDLE_REQUEST_ACTIVE_LOW); - assign_bit(FLAG_OPEN_DRAIN, flagsp, + assign_bit(GPIOD_FLAG_OPEN_DRAIN, &flags, lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN); - assign_bit(FLAG_OPEN_SOURCE, flagsp, + assign_bit(GPIOD_FLAG_OPEN_SOURCE, &flags, lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE); - assign_bit(FLAG_PULL_UP, flagsp, + assign_bit(GPIOD_FLAG_PULL_UP, &flags, lflags & GPIOHANDLE_REQUEST_BIAS_PULL_UP); - assign_bit(FLAG_PULL_DOWN, flagsp, + assign_bit(GPIOD_FLAG_PULL_DOWN, &flags, lflags & GPIOHANDLE_REQUEST_BIAS_PULL_DOWN); - assign_bit(FLAG_BIAS_DISABLE, flagsp, + assign_bit(GPIOD_FLAG_BIAS_DISABLE, &flags, lflags & GPIOHANDLE_REQUEST_BIAS_DISABLE); + + WRITE_ONCE(*flagsp, flags); } static long linehandle_set_config(struct linehandle_state *lh, @@ -163,29 +176,27 @@ static long linehandle_set_config(struct linehandle_state *lh, if (ret) return ret; + /* Lines must be reconfigured explicitly as input or output. */ + if (!(lflags & GPIOHANDLE_REQUEST_DIRECTION_FLAGS)) + return -EINVAL; + for (i = 0; i < lh->num_descs; i++) { desc = lh->descs[i]; - linehandle_flags_to_desc_flags(gcnf.flags, &desc->flags); + linehandle_flags_to_desc_flags(lflags, &desc->flags); - /* - * Lines have to be requested explicitly for input - * or output, else the line will be treated "as is". - */ if (lflags & GPIOHANDLE_REQUEST_OUTPUT) { int val = !!gcnf.default_values[i]; - ret = gpiod_direction_output(desc, val); + ret = gpiod_direction_output_nonotify(desc, val); if (ret) return ret; - } else if (lflags & GPIOHANDLE_REQUEST_INPUT) { - ret = gpiod_direction_input(desc); + } else { + ret = gpiod_direction_input_nonotify(desc); if (ret) return ret; } - blocking_notifier_call_chain(&desc->gdev->notifier, - GPIO_V2_LINE_CHANGED_CONFIG, - desc); + gpiod_line_state_notify(desc, GPIO_V2_LINE_CHANGED_CONFIG); } return 0; } @@ -197,16 +208,20 @@ static long linehandle_ioctl(struct file *file, unsigned int cmd, void __user *ip = (void __user *)arg; struct gpiohandle_data ghd; DECLARE_BITMAP(vals, GPIOHANDLES_MAX); - int i; + unsigned int i; + int ret; - if (cmd == GPIOHANDLE_GET_LINE_VALUES_IOCTL) { - /* NOTE: It's ok to read values of output lines. */ - int ret = gpiod_get_array_value_complex(false, - true, - lh->num_descs, - lh->descs, - NULL, - vals); + guard(srcu)(&lh->gdev->srcu); + + if (!rcu_access_pointer(lh->gdev->chip)) + return -ENODEV; + + switch (cmd) { + case GPIOHANDLE_GET_LINE_VALUES_IOCTL: + /* NOTE: It's okay to read values of output lines */ + ret = gpiod_get_array_value_complex(false, true, + lh->num_descs, lh->descs, + NULL, vals); if (ret) return ret; @@ -218,12 +233,12 @@ static long linehandle_ioctl(struct file *file, unsigned int cmd, return -EFAULT; return 0; - } else if (cmd == GPIOHANDLE_SET_LINE_VALUES_IOCTL) { + case GPIOHANDLE_SET_LINE_VALUES_IOCTL: /* * All line descriptors were created at once with the same * flags so just check if the first one is really output. */ - if (!test_bit(FLAG_IS_OUT, &lh->descs[0]->flags)) + if (!test_bit(GPIOD_FLAG_IS_OUT, &lh->descs[0]->flags)) return -EPERM; if (copy_from_user(&ghd, ip, sizeof(ghd))) @@ -240,10 +255,11 @@ static long linehandle_ioctl(struct file *file, unsigned int cmd, lh->descs, NULL, vals); - } else if (cmd == GPIOHANDLE_SET_CONFIG_IOCTL) { + case GPIOHANDLE_SET_CONFIG_IOCTL: return linehandle_set_config(lh, ip); + default: + return -EINVAL; } - return -EINVAL; } #ifdef CONFIG_COMPAT @@ -262,7 +278,7 @@ static void linehandle_free(struct linehandle_state *lh) if (lh->descs[i]) gpiod_free(lh->descs[i]); kfree(lh->label); - put_device(&lh->gdev->dev); + gpio_device_put(lh->gdev); kfree(lh); } @@ -282,12 +298,13 @@ static const struct file_operations linehandle_fileops = { #endif }; +DEFINE_FREE(linehandle_free, struct linehandle_state *, if (!IS_ERR_OR_NULL(_T)) linehandle_free(_T)) + static int linehandle_create(struct gpio_device *gdev, void __user *ip) { struct gpiohandle_request handlereq; - struct linehandle_state *lh; - struct file *file; - int fd, i, ret; + struct linehandle_state *lh __free(linehandle_free) = NULL; + int i, ret; u32 lflags; if (copy_from_user(&handlereq, ip, sizeof(handlereq))) @@ -304,18 +321,15 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) lh = kzalloc(sizeof(*lh), GFP_KERNEL); if (!lh) return -ENOMEM; - lh->gdev = gdev; - get_device(&gdev->dev); + lh->gdev = gpio_device_get(gdev); if (handlereq.consumer_label[0] != '\0') { /* label is only initialized if consumer_label is set */ lh->label = kstrndup(handlereq.consumer_label, sizeof(handlereq.consumer_label) - 1, GFP_KERNEL); - if (!lh->label) { - ret = -ENOMEM; - goto out_free_lh; - } + if (!lh->label) + return -ENOMEM; } lh->num_descs = handlereq.lines; @@ -323,22 +337,20 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) /* Request each GPIO */ for (i = 0; i < handlereq.lines; i++) { u32 offset = handlereq.lineoffsets[i]; - struct gpio_desc *desc = gpiochip_get_desc(gdev->chip, offset); + struct gpio_desc *desc = gpio_device_get_desc(gdev, offset); - if (IS_ERR(desc)) { - ret = PTR_ERR(desc); - goto out_free_lh; - } + if (IS_ERR(desc)) + return PTR_ERR(desc); - ret = gpiod_request(desc, lh->label); + ret = gpiod_request_user(desc, lh->label); if (ret) - goto out_free_lh; + return ret; lh->descs[i] = desc; linehandle_flags_to_desc_flags(handlereq.flags, &desc->flags); ret = gpiod_set_transitory(desc, false); if (ret < 0) - goto out_free_lh; + return ret; /* * Lines have to be requested explicitly for input @@ -347,60 +359,38 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) if (lflags & GPIOHANDLE_REQUEST_OUTPUT) { int val = !!handlereq.default_values[i]; - ret = gpiod_direction_output(desc, val); + ret = gpiod_direction_output_nonotify(desc, val); if (ret) - goto out_free_lh; + return ret; } else if (lflags & GPIOHANDLE_REQUEST_INPUT) { - ret = gpiod_direction_input(desc); + ret = gpiod_direction_input_nonotify(desc); if (ret) - goto out_free_lh; + return ret; } - blocking_notifier_call_chain(&desc->gdev->notifier, - GPIO_V2_LINE_CHANGED_REQUESTED, desc); + gpiod_line_state_notify(desc, GPIO_V2_LINE_CHANGED_REQUESTED); dev_dbg(&gdev->dev, "registered chardev handle for line %d\n", offset); } - fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC); - if (fd < 0) { - ret = fd; - goto out_free_lh; - } - - file = anon_inode_getfile("gpio-linehandle", - &linehandle_fileops, - lh, - O_RDONLY | O_CLOEXEC); - if (IS_ERR(file)) { - ret = PTR_ERR(file); - goto out_put_unused_fd; - } + FD_PREPARE(fdf, O_RDONLY | O_CLOEXEC, + anon_inode_getfile("gpio-linehandle", &linehandle_fileops, + lh, O_RDONLY | O_CLOEXEC)); + if (fdf.err) + return fdf.err; + retain_and_null_ptr(lh); - handlereq.fd = fd; - if (copy_to_user(ip, &handlereq, sizeof(handlereq))) { - /* - * fput() will trigger the release() callback, so do not go onto - * the regular error cleanup path here. - */ - fput(file); - put_unused_fd(fd); + handlereq.fd = fd_prepare_fd(fdf); + if (copy_to_user(ip, &handlereq, sizeof(handlereq))) return -EFAULT; - } - fd_install(fd, file); + fd_publish(fdf); dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n", lh->num_descs); return 0; - -out_put_unused_fd: - put_unused_fd(fd); -out_free_lh: - linehandle_free(lh); - return ret; } #endif /* CONFIG_GPIO_CDEV_V1 */ @@ -409,7 +399,7 @@ out_free_lh: * @desc: the GPIO descriptor for this line. * @req: the corresponding line request * @irq: the interrupt triggered in response to events on this GPIO - * @eflags: the edge flags, GPIO_V2_LINE_FLAG_EDGE_RISING and/or + * @edflags: the edge flags, GPIO_V2_LINE_FLAG_EDGE_RISING and/or * GPIO_V2_LINE_FLAG_EDGE_FALLING, indicating the edge detection applied * @timestamp_ns: cache for the timestamp storing it between hardirq and * IRQ thread, used to bring the timestamp close to the actual event @@ -420,6 +410,10 @@ out_free_lh: * @work: the worker that implements software debouncing * @sw_debounced: flag indicating if the software debouncer is active * @level: the current debounced physical level of the line + * @hdesc: the Hardware Timestamp Engine (HTE) descriptor + * @raw_level: the line level at the time of event + * @total_discard_seq: the running counter of the discarded events + * @last_seqno: the last sequence number before debounce period expires */ struct line { struct gpio_desc *desc; @@ -429,12 +423,15 @@ struct line { struct linereq *req; unsigned int irq; /* - * eflags is set by edge_detector_setup(), edge_detector_stop() and - * edge_detector_update(), which are themselves mutually exclusive, - * and is accessed by edge_irq_thread() and debounce_work_func(), - * which can both live with a slightly stale value. + * The flags for the active edge detector configuration. + * + * edflags is set by linereq_create(), linereq_free(), and + * linereq_set_config(), which are themselves mutually + * exclusive, and is accessed by edge_irq_thread(), + * process_hw_ts_thread() and debounce_work_func(), + * which can all live with a slightly stale value. */ - u64 eflags; + u64 edflags; /* * timestamp_ns and req_seqno are accessed only by * edge_irq_handler() and edge_irq_thread(), which are themselves @@ -464,6 +461,24 @@ struct line { * stale value. */ unsigned int level; +#ifdef CONFIG_HTE + struct hte_ts_desc hdesc; + /* + * HTE provider sets line level at the time of event. The valid + * value is 0 or 1 and negative value for an error. + */ + int raw_level; + /* + * when sw_debounce is set on HTE enabled line, this is running + * counter of the discarded events. + */ + u32 total_discard_seq; + /* + * when sw_debounce is set on HTE enabled line, this variable records + * last sequence number before debounce period expires. + */ + u32 last_seqno; +#endif /* CONFIG_HTE */ }; /** @@ -472,6 +487,7 @@ struct line { * @label: consumer label used to tag GPIO descriptors * @num_lines: the number of lines in the lines array * @wait: wait queue that handles blocking reads of events + * @device_unregistered_nb: notifier block for receiving gdev unregister events * @event_buffer_size: the number of elements allocated in @events * @events: KFIFO for the GPIO events * @seqno: the sequence number for edge events generated on all lines in @@ -486,11 +502,12 @@ struct linereq { const char *label; u32 num_lines; wait_queue_head_t wait; + struct notifier_block device_unregistered_nb; u32 event_buffer_size; DECLARE_KFIFO_PTR(events, struct gpio_v2_line_event); atomic_t seqno; struct mutex config_mutex; - struct line lines[]; + struct line lines[] __counted_by(num_lines); }; #define GPIO_V2_LINE_BIAS_FLAGS \ @@ -518,20 +535,38 @@ struct linereq { GPIO_V2_LINE_DRIVE_FLAGS | \ GPIO_V2_LINE_EDGE_FLAGS | \ GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME | \ + GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE | \ GPIO_V2_LINE_BIAS_FLAGS) +/* subset of flags relevant for edge detector configuration */ +#define GPIO_V2_LINE_EDGE_DETECTOR_FLAGS \ + (GPIO_V2_LINE_FLAG_ACTIVE_LOW | \ + GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE | \ + GPIO_V2_LINE_EDGE_FLAGS) + +static int linereq_unregistered_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct linereq *lr = container_of(nb, struct linereq, + device_unregistered_nb); + + wake_up_poll(&lr->wait, EPOLLIN | EPOLLERR); + + return NOTIFY_OK; +} + static void linereq_put_event(struct linereq *lr, struct gpio_v2_line_event *le) { bool overflow = false; - spin_lock(&lr->wait.lock); - if (kfifo_is_full(&lr->events)) { - overflow = true; - kfifo_skip(&lr->events); + scoped_guard(spinlock, &lr->wait.lock) { + if (kfifo_is_full(&lr->events)) { + overflow = true; + kfifo_skip(&lr->events); + } + kfifo_in(&lr->events, le, 1); } - kfifo_in(&lr->events, le, 1); - spin_unlock(&lr->wait.lock); if (!overflow) wake_up_poll(&lr->wait, EPOLLIN); else @@ -540,18 +575,166 @@ static void linereq_put_event(struct linereq *lr, static u64 line_event_timestamp(struct line *line) { - if (test_bit(FLAG_EVENT_CLOCK_REALTIME, &line->desc->flags)) + if (test_bit(GPIOD_FLAG_EVENT_CLOCK_REALTIME, &line->desc->flags)) return ktime_get_real_ns(); + else if (IS_ENABLED(CONFIG_HTE) && + test_bit(GPIOD_FLAG_EVENT_CLOCK_HTE, &line->desc->flags)) + return line->timestamp_ns; return ktime_get_ns(); } +static u32 line_event_id(int level) +{ + return level ? GPIO_V2_LINE_EVENT_RISING_EDGE : + GPIO_V2_LINE_EVENT_FALLING_EDGE; +} + +static inline char *make_irq_label(const char *orig) +{ + char *new; + + if (!orig) + return NULL; + + new = kstrdup_and_replace(orig, '/', ':', GFP_KERNEL); + if (!new) + return ERR_PTR(-ENOMEM); + + return new; +} + +static inline void free_irq_label(const char *label) +{ + kfree(label); +} + +#ifdef CONFIG_HTE + +static enum hte_return process_hw_ts_thread(void *p) +{ + struct line *line; + struct linereq *lr; + struct gpio_v2_line_event le; + u64 edflags; + int level; + + if (!p) + return HTE_CB_HANDLED; + + line = p; + lr = line->req; + + memset(&le, 0, sizeof(le)); + + le.timestamp_ns = line->timestamp_ns; + edflags = READ_ONCE(line->edflags); + + switch (edflags & GPIO_V2_LINE_EDGE_FLAGS) { + case GPIO_V2_LINE_FLAG_EDGE_BOTH: + level = (line->raw_level >= 0) ? + line->raw_level : + gpiod_get_raw_value_cansleep(line->desc); + + if (edflags & GPIO_V2_LINE_FLAG_ACTIVE_LOW) + level = !level; + + le.id = line_event_id(level); + break; + case GPIO_V2_LINE_FLAG_EDGE_RISING: + le.id = GPIO_V2_LINE_EVENT_RISING_EDGE; + break; + case GPIO_V2_LINE_FLAG_EDGE_FALLING: + le.id = GPIO_V2_LINE_EVENT_FALLING_EDGE; + break; + default: + return HTE_CB_HANDLED; + } + le.line_seqno = line->line_seqno; + le.seqno = (lr->num_lines == 1) ? le.line_seqno : line->req_seqno; + le.offset = gpiod_hwgpio(line->desc); + + linereq_put_event(lr, &le); + + return HTE_CB_HANDLED; +} + +static enum hte_return process_hw_ts(struct hte_ts_data *ts, void *p) +{ + struct line *line; + struct linereq *lr; + int diff_seqno = 0; + + if (!ts || !p) + return HTE_CB_HANDLED; + + line = p; + line->timestamp_ns = ts->tsc; + line->raw_level = ts->raw_level; + lr = line->req; + + if (READ_ONCE(line->sw_debounced)) { + line->total_discard_seq++; + line->last_seqno = ts->seq; + mod_delayed_work(system_percpu_wq, &line->work, + usecs_to_jiffies(READ_ONCE(line->desc->debounce_period_us))); + } else { + if (unlikely(ts->seq < line->line_seqno)) + return HTE_CB_HANDLED; + + diff_seqno = ts->seq - line->line_seqno; + line->line_seqno = ts->seq; + if (lr->num_lines != 1) + line->req_seqno = atomic_add_return(diff_seqno, + &lr->seqno); + + return HTE_RUN_SECOND_CB; + } + + return HTE_CB_HANDLED; +} + +static int hte_edge_setup(struct line *line, u64 eflags) +{ + int ret; + unsigned long flags = 0; + struct hte_ts_desc *hdesc = &line->hdesc; + + if (eflags & GPIO_V2_LINE_FLAG_EDGE_RISING) + flags |= test_bit(GPIOD_FLAG_ACTIVE_LOW, &line->desc->flags) ? + HTE_FALLING_EDGE_TS : + HTE_RISING_EDGE_TS; + if (eflags & GPIO_V2_LINE_FLAG_EDGE_FALLING) + flags |= test_bit(GPIOD_FLAG_ACTIVE_LOW, &line->desc->flags) ? + HTE_RISING_EDGE_TS : + HTE_FALLING_EDGE_TS; + + line->total_discard_seq = 0; + + hte_init_line_attr(hdesc, desc_to_gpio(line->desc), flags, NULL, + line->desc); + + ret = hte_ts_get(NULL, hdesc, 0); + if (ret) + return ret; + + return hte_request_ts_ns(hdesc, process_hw_ts, process_hw_ts_thread, + line); +} + +#else + +static int hte_edge_setup(struct line *line, u64 eflags) +{ + return 0; +} +#endif /* CONFIG_HTE */ + static irqreturn_t edge_irq_thread(int irq, void *p) { struct line *line = p; struct linereq *lr = line->req; struct gpio_v2_line_event le; - u64 eflags; /* Do not leak kernel stack to userspace */ memset(&le, 0, sizeof(le)); @@ -570,29 +753,23 @@ static irqreturn_t edge_irq_thread(int irq, void *p) } line->timestamp_ns = 0; - eflags = READ_ONCE(line->eflags); - if (eflags == GPIO_V2_LINE_FLAG_EDGE_BOTH) { - int level = gpiod_get_value_cansleep(line->desc); - - if (level) - /* Emit low-to-high event */ - le.id = GPIO_V2_LINE_EVENT_RISING_EDGE; - else - /* Emit high-to-low event */ - le.id = GPIO_V2_LINE_EVENT_FALLING_EDGE; - } else if (eflags == GPIO_V2_LINE_FLAG_EDGE_RISING) { - /* Emit low-to-high event */ + switch (READ_ONCE(line->edflags) & GPIO_V2_LINE_EDGE_FLAGS) { + case GPIO_V2_LINE_FLAG_EDGE_BOTH: + le.id = line_event_id(gpiod_get_value_cansleep(line->desc)); + break; + case GPIO_V2_LINE_FLAG_EDGE_RISING: le.id = GPIO_V2_LINE_EVENT_RISING_EDGE; - } else if (eflags == GPIO_V2_LINE_FLAG_EDGE_FALLING) { - /* Emit high-to-low event */ + break; + case GPIO_V2_LINE_FLAG_EDGE_FALLING: le.id = GPIO_V2_LINE_EVENT_FALLING_EDGE; - } else { + break; + default: return IRQ_NONE; } line->line_seqno++; le.line_seqno = line->line_seqno; le.seqno = (lr->num_lines == 1) ? le.line_seqno : line->req_seqno; - le.offset = gpio_chip_hwgpio(line->desc); + le.offset = gpiod_hwgpio(line->desc); linereq_put_event(lr, &le); @@ -630,7 +807,7 @@ static bool debounced_value(struct line *line) */ value = READ_ONCE(line->level); - if (test_bit(FLAG_ACTIVE_LOW, &line->desc->flags)) + if (test_bit(GPIOD_FLAG_ACTIVE_LOW, &line->desc->flags)) value = !value; return value; @@ -640,7 +817,7 @@ static irqreturn_t debounce_irq_handler(int irq, void *p) { struct line *line = p; - mod_delayed_work(system_wq, &line->work, + mod_delayed_work(system_percpu_wq, &line->work, usecs_to_jiffies(READ_ONCE(line->desc->debounce_period_us))); return IRQ_HANDLED; @@ -651,10 +828,16 @@ static void debounce_work_func(struct work_struct *work) struct gpio_v2_line_event le; struct line *line = container_of(work, struct line, work.work); struct linereq *lr; - int level; - u64 eflags; + u64 eflags, edflags = READ_ONCE(line->edflags); + int level = -1; +#ifdef CONFIG_HTE + int diff_seqno; - level = gpiod_get_raw_value_cansleep(line->desc); + if (edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE) + level = line->raw_level; +#endif + if (level < 0) + level = gpiod_get_raw_value_cansleep(line->desc); if (level < 0) { pr_debug_ratelimited("debouncer failed to read line value\n"); return; @@ -666,12 +849,12 @@ static void debounce_work_func(struct work_struct *work) WRITE_ONCE(line->level, level); /* -- edge detection -- */ - eflags = READ_ONCE(line->eflags); + eflags = edflags & GPIO_V2_LINE_EDGE_FLAGS; if (!eflags) return; /* switch from physical level to logical - if they differ */ - if (test_bit(FLAG_ACTIVE_LOW, &line->desc->flags)) + if (edflags & GPIO_V2_LINE_FLAG_ACTIVE_LOW) level = !level; /* ignore edges that are not being monitored */ @@ -684,34 +867,44 @@ static void debounce_work_func(struct work_struct *work) lr = line->req; le.timestamp_ns = line_event_timestamp(line); - le.offset = gpio_chip_hwgpio(line->desc); - line->line_seqno++; - le.line_seqno = line->line_seqno; - le.seqno = (lr->num_lines == 1) ? - le.line_seqno : atomic_inc_return(&lr->seqno); + le.offset = gpiod_hwgpio(line->desc); +#ifdef CONFIG_HTE + if (edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE) { + /* discard events except the last one */ + line->total_discard_seq -= 1; + diff_seqno = line->last_seqno - line->total_discard_seq - + line->line_seqno; + line->line_seqno = line->last_seqno - line->total_discard_seq; + le.line_seqno = line->line_seqno; + le.seqno = (lr->num_lines == 1) ? + le.line_seqno : atomic_add_return(diff_seqno, &lr->seqno); + } else +#endif /* CONFIG_HTE */ + { + line->line_seqno++; + le.line_seqno = line->line_seqno; + le.seqno = (lr->num_lines == 1) ? + le.line_seqno : atomic_inc_return(&lr->seqno); + } - if (level) - /* Emit low-to-high event */ - le.id = GPIO_V2_LINE_EVENT_RISING_EDGE; - else - /* Emit high-to-low event */ - le.id = GPIO_V2_LINE_EVENT_FALLING_EDGE; + le.id = line_event_id(level); linereq_put_event(lr, &le); } -static int debounce_setup(struct line *line, - unsigned int debounce_period_us) +static int debounce_setup(struct line *line, unsigned int debounce_period_us) { unsigned long irqflags; int ret, level, irq; + char *label; - /* try hardware */ - ret = gpiod_set_debounce(line->desc, debounce_period_us); - if (!ret) { - WRITE_ONCE(line->desc->debounce_period_us, debounce_period_us); - return ret; - } + /* + * Try hardware. Skip gpiod_set_config() to avoid emitting two + * CHANGED_CONFIG line state events. + */ + ret = gpio_do_set_config(line->desc, + pinconf_to_config_packed(PIN_CONFIG_INPUT_DEBOUNCE, + debounce_period_us)); if (ret != -ENOTSUPP) return ret; @@ -721,19 +914,32 @@ static int debounce_setup(struct line *line, if (level < 0) return level; - irq = gpiod_to_irq(line->desc); - if (irq < 0) - return -ENXIO; + if (!(IS_ENABLED(CONFIG_HTE) && + test_bit(GPIOD_FLAG_EVENT_CLOCK_HTE, &line->desc->flags))) { + irq = gpiod_to_irq(line->desc); + if (irq < 0) + return -ENXIO; - WRITE_ONCE(line->level, level); - irqflags = IRQF_TRIGGER_FALLING | IRQF_TRIGGER_RISING; - ret = request_irq(irq, debounce_irq_handler, irqflags, - line->req->label, line); - if (ret) - return ret; + label = make_irq_label(line->req->label); + if (IS_ERR(label)) + return -ENOMEM; + + irqflags = IRQF_TRIGGER_FALLING | IRQF_TRIGGER_RISING; + ret = request_irq(irq, debounce_irq_handler, irqflags, + label, line); + if (ret) { + free_irq_label(label); + return ret; + } + line->irq = irq; + } else { + ret = hte_edge_setup(line, GPIO_V2_LINE_FLAG_EDGE_BOTH); + if (ret) + return ret; + } + WRITE_ONCE(line->level, level); WRITE_ONCE(line->sw_debounced, 1); - line->irq = irq; } return 0; } @@ -769,34 +975,47 @@ static u32 gpio_v2_line_config_debounce_period(struct gpio_v2_line_config *lc, static void edge_detector_stop(struct line *line) { if (line->irq) { - free_irq(line->irq, line); + free_irq_label(free_irq(line->irq, line)); line->irq = 0; } +#ifdef CONFIG_HTE + if (READ_ONCE(line->edflags) & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE) + hte_ts_put(&line->hdesc); +#endif + cancel_delayed_work_sync(&line->work); WRITE_ONCE(line->sw_debounced, 0); - WRITE_ONCE(line->eflags, 0); + WRITE_ONCE(line->edflags, 0); if (line->desc) WRITE_ONCE(line->desc->debounce_period_us, 0); /* do not change line->level - see comment in debounced_value() */ } +static int edge_detector_fifo_init(struct linereq *req) +{ + if (kfifo_initialized(&req->events)) + return 0; + + return kfifo_alloc(&req->events, req->event_buffer_size, GFP_KERNEL); +} + static int edge_detector_setup(struct line *line, struct gpio_v2_line_config *lc, - unsigned int line_idx, - u64 eflags) + unsigned int line_idx, u64 edflags) { u32 debounce_period_us; unsigned long irqflags = 0; + u64 eflags; int irq, ret; + char *label; - if (eflags && !kfifo_initialized(&line->req->events)) { - ret = kfifo_alloc(&line->req->events, - line->req->event_buffer_size, GFP_KERNEL); + eflags = edflags & GPIO_V2_LINE_EDGE_FLAGS; + if (eflags) { + ret = edge_detector_fifo_init(line->req); if (ret) return ret; } - WRITE_ONCE(line->eflags, eflags); if (gpio_v2_line_config_debounced(lc, line_idx)) { debounce_period_us = gpio_v2_line_config_debounce_period(lc, line_idx); ret = debounce_setup(line, debounce_period_us); @@ -809,23 +1028,33 @@ static int edge_detector_setup(struct line *line, if (!eflags || READ_ONCE(line->sw_debounced)) return 0; + if (IS_ENABLED(CONFIG_HTE) && + (edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE)) + return hte_edge_setup(line, edflags); + irq = gpiod_to_irq(line->desc); if (irq < 0) return -ENXIO; if (eflags & GPIO_V2_LINE_FLAG_EDGE_RISING) - irqflags |= test_bit(FLAG_ACTIVE_LOW, &line->desc->flags) ? + irqflags |= test_bit(GPIOD_FLAG_ACTIVE_LOW, &line->desc->flags) ? IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING; if (eflags & GPIO_V2_LINE_FLAG_EDGE_FALLING) - irqflags |= test_bit(FLAG_ACTIVE_LOW, &line->desc->flags) ? + irqflags |= test_bit(GPIOD_FLAG_ACTIVE_LOW, &line->desc->flags) ? IRQF_TRIGGER_RISING : IRQF_TRIGGER_FALLING; irqflags |= IRQF_ONESHOT; + label = make_irq_label(line->req->label); + if (IS_ERR(label)) + return PTR_ERR(label); + /* Request a thread to read the events */ ret = request_threaded_irq(irq, edge_irq_handler, edge_irq_thread, - irqflags, line->req->label, line); - if (ret) + irqflags, label, line); + if (ret) { + free_irq_label(label); return ret; + } line->irq = irq; return 0; @@ -833,29 +1062,36 @@ static int edge_detector_setup(struct line *line, static int edge_detector_update(struct line *line, struct gpio_v2_line_config *lc, - unsigned int line_idx, - u64 eflags, bool polarity_change) + unsigned int line_idx, u64 edflags) { + u64 active_edflags = READ_ONCE(line->edflags); unsigned int debounce_period_us = - gpio_v2_line_config_debounce_period(lc, line_idx); + gpio_v2_line_config_debounce_period(lc, line_idx); - if ((READ_ONCE(line->eflags) == eflags) && !polarity_change && + if ((active_edflags == edflags) && (READ_ONCE(line->desc->debounce_period_us) == debounce_period_us)) return 0; /* sw debounced and still will be...*/ if (debounce_period_us && READ_ONCE(line->sw_debounced)) { - WRITE_ONCE(line->eflags, eflags); WRITE_ONCE(line->desc->debounce_period_us, debounce_period_us); + /* + * ensure event fifo is initialised if edge detection + * is now enabled. + */ + if (edflags & GPIO_V2_LINE_EDGE_FLAGS) + return edge_detector_fifo_init(line->req); + return 0; } /* reconfiguring edge detection or sw debounce being disabled */ if ((line->irq && !READ_ONCE(line->sw_debounced)) || + (active_edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE) || (!debounce_period_us && READ_ONCE(line->sw_debounced))) edge_detector_stop(line); - return edge_detector_setup(line, lc, line_idx, eflags); + return edge_detector_setup(line, lc, line_idx, edflags); } static u64 gpio_v2_line_config_flags(struct gpio_v2_line_config *lc, @@ -892,6 +1128,10 @@ static int gpio_v2_line_flags_validate(u64 flags) if (flags & ~GPIO_V2_LINE_VALID_FLAGS) return -EINVAL; + if (!IS_ENABLED(CONFIG_HTE) && + (flags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE)) + return -EOPNOTSUPP; + /* * Do not allow both INPUT and OUTPUT flags to be set as they are * contradictory. @@ -900,6 +1140,12 @@ static int gpio_v2_line_flags_validate(u64 flags) (flags & GPIO_V2_LINE_FLAG_OUTPUT)) return -EINVAL; + /* Only allow one event clock source */ + if (IS_ENABLED(CONFIG_HTE) && + (flags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME) && + (flags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE)) + return -EINVAL; + /* Edge detection requires explicit input. */ if ((flags & GPIO_V2_LINE_EDGE_FLAGS) && !(flags & GPIO_V2_LINE_FLAG_INPUT)) @@ -945,7 +1191,7 @@ static int gpio_v2_line_config_validate(struct gpio_v2_line_config *lc, if (lc->num_attrs > GPIO_V2_LINE_NUM_ATTRS_MAX) return -EINVAL; - if (memchr_inv(lc->padding, 0, sizeof(lc->padding))) + if (!mem_is_zero(lc->padding, sizeof(lc->padding))) return -EINVAL; for (i = 0; i < num_lines; i++) { @@ -962,36 +1208,42 @@ static int gpio_v2_line_config_validate(struct gpio_v2_line_config *lc, return 0; } -static void gpio_v2_line_config_flags_to_desc_flags(u64 flags, +static void gpio_v2_line_config_flags_to_desc_flags(u64 lflags, unsigned long *flagsp) { - assign_bit(FLAG_ACTIVE_LOW, flagsp, - flags & GPIO_V2_LINE_FLAG_ACTIVE_LOW); + unsigned long flags = READ_ONCE(*flagsp); - if (flags & GPIO_V2_LINE_FLAG_OUTPUT) - set_bit(FLAG_IS_OUT, flagsp); - else if (flags & GPIO_V2_LINE_FLAG_INPUT) - clear_bit(FLAG_IS_OUT, flagsp); + assign_bit(GPIOD_FLAG_ACTIVE_LOW, &flags, + lflags & GPIO_V2_LINE_FLAG_ACTIVE_LOW); - assign_bit(FLAG_EDGE_RISING, flagsp, - flags & GPIO_V2_LINE_FLAG_EDGE_RISING); - assign_bit(FLAG_EDGE_FALLING, flagsp, - flags & GPIO_V2_LINE_FLAG_EDGE_FALLING); + if (lflags & GPIO_V2_LINE_FLAG_OUTPUT) + set_bit(GPIOD_FLAG_IS_OUT, &flags); + else if (lflags & GPIO_V2_LINE_FLAG_INPUT) + clear_bit(GPIOD_FLAG_IS_OUT, &flags); - assign_bit(FLAG_OPEN_DRAIN, flagsp, - flags & GPIO_V2_LINE_FLAG_OPEN_DRAIN); - assign_bit(FLAG_OPEN_SOURCE, flagsp, - flags & GPIO_V2_LINE_FLAG_OPEN_SOURCE); + assign_bit(GPIOD_FLAG_EDGE_RISING, &flags, + lflags & GPIO_V2_LINE_FLAG_EDGE_RISING); + assign_bit(GPIOD_FLAG_EDGE_FALLING, &flags, + lflags & GPIO_V2_LINE_FLAG_EDGE_FALLING); - assign_bit(FLAG_PULL_UP, flagsp, - flags & GPIO_V2_LINE_FLAG_BIAS_PULL_UP); - assign_bit(FLAG_PULL_DOWN, flagsp, - flags & GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN); - assign_bit(FLAG_BIAS_DISABLE, flagsp, - flags & GPIO_V2_LINE_FLAG_BIAS_DISABLED); + assign_bit(GPIOD_FLAG_OPEN_DRAIN, &flags, + lflags & GPIO_V2_LINE_FLAG_OPEN_DRAIN); + assign_bit(GPIOD_FLAG_OPEN_SOURCE, &flags, + lflags & GPIO_V2_LINE_FLAG_OPEN_SOURCE); - assign_bit(FLAG_EVENT_CLOCK_REALTIME, flagsp, - flags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME); + assign_bit(GPIOD_FLAG_PULL_UP, &flags, + lflags & GPIO_V2_LINE_FLAG_BIAS_PULL_UP); + assign_bit(GPIOD_FLAG_PULL_DOWN, &flags, + lflags & GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN); + assign_bit(GPIOD_FLAG_BIAS_DISABLE, &flags, + lflags & GPIO_V2_LINE_FLAG_BIAS_DISABLED); + + assign_bit(GPIOD_FLAG_EVENT_CLOCK_REALTIME, &flags, + lflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME); + assign_bit(GPIOD_FLAG_EVENT_CLOCK_HTE, &flags, + lflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE); + + WRITE_ONCE(*flagsp, flags); } static long linereq_get_values(struct linereq *lr, void __user *ip) @@ -1007,9 +1259,18 @@ static long linereq_get_values(struct linereq *lr, void __user *ip) if (copy_from_user(&lv, ip, sizeof(lv))) return -EFAULT; + /* + * gpiod_get_array_value_complex() requires compacted desc and val + * arrays, rather than the sparse ones in lv. + * Calculation of num_get and construction of the desc array is + * optimized to avoid allocation for the desc array for the common + * num_get == 1 case. + */ + /* scan requested lines to calculate the subset to get */ for (num_get = 0, i = 0; i < lr->num_lines; i++) { if (lv.mask & BIT_ULL(i)) { num_get++; + /* capture desc for the num_get == 1 case */ descs = &lr->lines[i].desc; } } @@ -1018,6 +1279,7 @@ static long linereq_get_values(struct linereq *lr, void __user *ip) return -EINVAL; if (num_get != 1) { + /* build compacted desc array */ descs = kmalloc_array(num_get, sizeof(*descs), GFP_KERNEL); if (!descs) return -ENOMEM; @@ -1038,6 +1300,7 @@ static long linereq_get_values(struct linereq *lr, void __user *ip) lv.bits = 0; for (didx = 0, i = 0; i < lr->num_lines; i++) { + /* unpack compacted vals for the response */ if (lv.mask & BIT_ULL(i)) { if (lr->lines[i].sw_debounced) val = debounced_value(&lr->lines[i]); @@ -1055,22 +1318,35 @@ static long linereq_get_values(struct linereq *lr, void __user *ip) return 0; } -static long linereq_set_values_unlocked(struct linereq *lr, - struct gpio_v2_line_values *lv) +static long linereq_set_values(struct linereq *lr, void __user *ip) { DECLARE_BITMAP(vals, GPIO_V2_LINES_MAX); + struct gpio_v2_line_values lv; struct gpio_desc **descs; unsigned int i, didx, num_set; int ret; + if (copy_from_user(&lv, ip, sizeof(lv))) + return -EFAULT; + + guard(mutex)(&lr->config_mutex); + + /* + * gpiod_set_array_value_complex() requires compacted desc and val + * arrays, rather than the sparse ones in lv. + * Calculation of num_set and construction of the descs and vals arrays + * is optimized to minimize scanning the lv->mask, and to avoid + * allocation for the desc array for the common num_set == 1 case. + */ bitmap_zero(vals, GPIO_V2_LINES_MAX); + /* scan requested lines to determine the subset to be set */ for (num_set = 0, i = 0; i < lr->num_lines; i++) { - if (lv->mask & BIT_ULL(i)) { - if (!test_bit(FLAG_IS_OUT, &lr->lines[i].desc->flags)) - return -EPERM; - if (lv->bits & BIT_ULL(i)) + if (lv.mask & BIT_ULL(i)) { + /* add to compacted values */ + if (lv.bits & BIT_ULL(i)) __set_bit(num_set, vals); num_set++; + /* capture desc for the num_set == 1 case */ descs = &lr->lines[i].desc; } } @@ -1078,12 +1354,12 @@ static long linereq_set_values_unlocked(struct linereq *lr, return -EINVAL; if (num_set != 1) { - /* build compacted desc array and values */ + /* build compacted desc array */ descs = kmalloc_array(num_set, sizeof(*descs), GFP_KERNEL); if (!descs) return -ENOMEM; for (didx = 0, i = 0; i < lr->num_lines; i++) { - if (lv->mask & BIT_ULL(i)) { + if (lv.mask & BIT_ULL(i)) { descs[didx] = lr->lines[i].desc; didx++; } @@ -1097,105 +1373,81 @@ static long linereq_set_values_unlocked(struct linereq *lr, return ret; } -static long linereq_set_values(struct linereq *lr, void __user *ip) +static long linereq_set_config(struct linereq *lr, void __user *ip) { - struct gpio_v2_line_values lv; + struct gpio_v2_line_config lc; + struct gpio_desc *desc; + struct line *line; + unsigned int i; + u64 flags, edflags; int ret; - if (copy_from_user(&lv, ip, sizeof(lv))) + if (copy_from_user(&lc, ip, sizeof(lc))) return -EFAULT; - mutex_lock(&lr->config_mutex); - - ret = linereq_set_values_unlocked(lr, &lv); - - mutex_unlock(&lr->config_mutex); + ret = gpio_v2_line_config_validate(&lc, lr->num_lines); + if (ret) + return ret; - return ret; -} - -static long linereq_set_config_unlocked(struct linereq *lr, - struct gpio_v2_line_config *lc) -{ - struct gpio_desc *desc; - unsigned int i; - u64 flags; - bool polarity_change; - int ret; + guard(mutex)(&lr->config_mutex); for (i = 0; i < lr->num_lines; i++) { + line = &lr->lines[i]; desc = lr->lines[i].desc; - flags = gpio_v2_line_config_flags(lc, i); - polarity_change = - (!!test_bit(FLAG_ACTIVE_LOW, &desc->flags) != - ((flags & GPIO_V2_LINE_FLAG_ACTIVE_LOW) != 0)); - - gpio_v2_line_config_flags_to_desc_flags(flags, &desc->flags); + flags = gpio_v2_line_config_flags(&lc, i); /* - * Lines have to be requested explicitly for input - * or output, else the line will be treated "as is". + * Lines not explicitly reconfigured as input or output + * are left unchanged. */ + if (!(flags & GPIO_V2_LINE_DIRECTION_FLAGS)) + continue; + gpio_v2_line_config_flags_to_desc_flags(flags, &desc->flags); + edflags = flags & GPIO_V2_LINE_EDGE_DETECTOR_FLAGS; if (flags & GPIO_V2_LINE_FLAG_OUTPUT) { - int val = gpio_v2_line_config_output_value(lc, i); + int val = gpio_v2_line_config_output_value(&lc, i); - edge_detector_stop(&lr->lines[i]); - ret = gpiod_direction_output(desc, val); + edge_detector_stop(line); + ret = gpiod_direction_output_nonotify(desc, val); if (ret) return ret; - } else if (flags & GPIO_V2_LINE_FLAG_INPUT) { - ret = gpiod_direction_input(desc); + } else { + ret = gpiod_direction_input_nonotify(desc); if (ret) return ret; - ret = edge_detector_update(&lr->lines[i], lc, i, - flags & GPIO_V2_LINE_EDGE_FLAGS, - polarity_change); + ret = edge_detector_update(line, &lc, i, edflags); if (ret) return ret; } - blocking_notifier_call_chain(&desc->gdev->notifier, - GPIO_V2_LINE_CHANGED_CONFIG, - desc); + WRITE_ONCE(line->edflags, edflags); + + gpiod_line_state_notify(desc, GPIO_V2_LINE_CHANGED_CONFIG); } return 0; } -static long linereq_set_config(struct linereq *lr, void __user *ip) -{ - struct gpio_v2_line_config lc; - int ret; - - if (copy_from_user(&lc, ip, sizeof(lc))) - return -EFAULT; - - ret = gpio_v2_line_config_validate(&lc, lr->num_lines); - if (ret) - return ret; - - mutex_lock(&lr->config_mutex); - - ret = linereq_set_config_unlocked(lr, &lc); - - mutex_unlock(&lr->config_mutex); - - return ret; -} - static long linereq_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct linereq *lr = file->private_data; void __user *ip = (void __user *)arg; - if (cmd == GPIO_V2_LINE_GET_VALUES_IOCTL) + guard(srcu)(&lr->gdev->srcu); + + if (!rcu_access_pointer(lr->gdev->chip)) + return -ENODEV; + + switch (cmd) { + case GPIO_V2_LINE_GET_VALUES_IOCTL: return linereq_get_values(lr, ip); - else if (cmd == GPIO_V2_LINE_SET_VALUES_IOCTL) + case GPIO_V2_LINE_SET_VALUES_IOCTL: return linereq_set_values(lr, ip); - else if (cmd == GPIO_V2_LINE_SET_CONFIG_IOCTL) + case GPIO_V2_LINE_SET_CONFIG_IOCTL: return linereq_set_config(lr, ip); - - return -EINVAL; + default: + return -EINVAL; + } } #ifdef CONFIG_COMPAT @@ -1207,11 +1459,16 @@ static long linereq_ioctl_compat(struct file *file, unsigned int cmd, #endif static __poll_t linereq_poll(struct file *file, - struct poll_table_struct *wait) + struct poll_table_struct *wait) { struct linereq *lr = file->private_data; __poll_t events = 0; + guard(srcu)(&lr->gdev->srcu); + + if (!rcu_access_pointer(lr->gdev->chip)) + return EPOLLHUP | EPOLLERR; + poll_wait(file, &lr->wait, wait); if (!kfifo_is_empty_spinlocked_noirqsave(&lr->events, @@ -1221,52 +1478,48 @@ static __poll_t linereq_poll(struct file *file, return events; } -static ssize_t linereq_read(struct file *file, - char __user *buf, - size_t count, - loff_t *f_ps) +static ssize_t linereq_read(struct file *file, char __user *buf, + size_t count, loff_t *f_ps) { struct linereq *lr = file->private_data; struct gpio_v2_line_event le; ssize_t bytes_read = 0; int ret; + guard(srcu)(&lr->gdev->srcu); + + if (!rcu_access_pointer(lr->gdev->chip)) + return -ENODEV; + if (count < sizeof(le)) return -EINVAL; do { - spin_lock(&lr->wait.lock); - if (kfifo_is_empty(&lr->events)) { - if (bytes_read) { - spin_unlock(&lr->wait.lock); - return bytes_read; + scoped_guard(spinlock, &lr->wait.lock) { + if (kfifo_is_empty(&lr->events)) { + if (bytes_read) + return bytes_read; + + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + + ret = wait_event_interruptible_locked(lr->wait, + !kfifo_is_empty(&lr->events)); + if (ret) + return ret; } - if (file->f_flags & O_NONBLOCK) { - spin_unlock(&lr->wait.lock); - return -EAGAIN; - } - - ret = wait_event_interruptible_locked(lr->wait, - !kfifo_is_empty(&lr->events)); - if (ret) { - spin_unlock(&lr->wait.lock); - return ret; + if (kfifo_out(&lr->events, &le, 1) != 1) { + /* + * This should never happen - we hold the + * lock from the moment we learned the fifo + * is no longer empty until now. + */ + WARN(1, "failed to read from non-empty kfifo"); + return -EIO; } } - ret = kfifo_out(&lr->events, &le, 1); - spin_unlock(&lr->wait.lock); - if (ret != 1) { - /* - * This should never happen - we were holding the - * lock from the moment we learned the fifo is no - * longer empty until now. - */ - ret = -EIO; - break; - } - if (copy_to_user(buf + bytes_read, &le, sizeof(le))) return -EFAULT; bytes_read += sizeof(le); @@ -1279,15 +1532,20 @@ static void linereq_free(struct linereq *lr) { unsigned int i; + if (lr->device_unregistered_nb.notifier_call) + blocking_notifier_chain_unregister(&lr->gdev->device_notifier, + &lr->device_unregistered_nb); + for (i = 0; i < lr->num_lines; i++) { - edge_detector_stop(&lr->lines[i]); - if (lr->lines[i].desc) + if (lr->lines[i].desc) { + edge_detector_stop(&lr->lines[i]); gpiod_free(lr->lines[i].desc); + } } kfifo_free(&lr->events); kfree(lr->label); - put_device(&lr->gdev->dev); - kfree(lr); + gpio_device_put(lr->gdev); + kvfree(lr); } static int linereq_release(struct inode *inode, struct file *file) @@ -1298,6 +1556,21 @@ static int linereq_release(struct inode *inode, struct file *file) return 0; } +#ifdef CONFIG_PROC_FS +static void linereq_show_fdinfo(struct seq_file *out, struct file *file) +{ + struct linereq *lr = file->private_data; + struct device *dev = &lr->gdev->dev; + u16 i; + + seq_printf(out, "gpio-chip:\t%s\n", dev_name(dev)); + + for (i = 0; i < lr->num_lines; i++) + seq_printf(out, "gpio-line:\t%d\n", + gpiod_hwgpio(lr->lines[i].desc)); +} +#endif + static const struct file_operations line_fileops = { .release = linereq_release, .read = linereq_read, @@ -1308,6 +1581,9 @@ static const struct file_operations line_fileops = { #ifdef CONFIG_COMPAT .compat_ioctl = linereq_ioctl_compat, #endif +#ifdef CONFIG_PROC_FS + .show_fdinfo = linereq_show_fdinfo, +#endif }; static int linereq_create(struct gpio_device *gdev, void __user *ip) @@ -1316,7 +1592,7 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip) struct gpio_v2_line_config *lc; struct linereq *lr; struct file *file; - u64 flags; + u64 flags, edflags; unsigned int i; int fd, ret; @@ -1326,7 +1602,7 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip) if ((ulr.num_lines == 0) || (ulr.num_lines > GPIO_V2_LINES_MAX)) return -EINVAL; - if (memchr_inv(ulr.padding, 0, sizeof(ulr.padding))) + if (!mem_is_zero(ulr.padding, sizeof(ulr.padding))) return -EINVAL; lc = &ulr.config; @@ -1334,12 +1610,12 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip) if (ret) return ret; - lr = kzalloc(struct_size(lr, lines, ulr.num_lines), GFP_KERNEL); + lr = kvzalloc(struct_size(lr, lines, ulr.num_lines), GFP_KERNEL); if (!lr) return -ENOMEM; + lr->num_lines = ulr.num_lines; - lr->gdev = gdev; - get_device(&gdev->dev); + lr->gdev = gpio_device_get(gdev); for (i = 0; i < ulr.num_lines; i++) { lr->lines[i].req = lr; @@ -1359,6 +1635,7 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip) mutex_init(&lr->config_mutex); init_waitqueue_head(&lr->wait); + INIT_KFIFO(lr->events); lr->event_buffer_size = ulr.event_buffer_size; if (lr->event_buffer_size == 0) lr->event_buffer_size = ulr.num_lines * 16; @@ -1366,19 +1643,18 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip) lr->event_buffer_size = GPIO_V2_LINES_MAX * 16; atomic_set(&lr->seqno, 0); - lr->num_lines = ulr.num_lines; /* Request each GPIO */ for (i = 0; i < ulr.num_lines; i++) { u32 offset = ulr.offsets[i]; - struct gpio_desc *desc = gpiochip_get_desc(gdev->chip, offset); + struct gpio_desc *desc = gpio_device_get_desc(gdev, offset); if (IS_ERR(desc)) { ret = PTR_ERR(desc); goto out_free_linereq; } - ret = gpiod_request(desc, lr->label); + ret = gpiod_request_user(desc, lr->label); if (ret) goto out_free_linereq; @@ -1390,6 +1666,7 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip) if (ret < 0) goto out_free_linereq; + edflags = flags & GPIO_V2_LINE_EDGE_DETECTOR_FLAGS; /* * Lines have to be requested explicitly for input * or output, else the line will be treated "as is". @@ -1397,27 +1674,34 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip) if (flags & GPIO_V2_LINE_FLAG_OUTPUT) { int val = gpio_v2_line_config_output_value(lc, i); - ret = gpiod_direction_output(desc, val); + ret = gpiod_direction_output_nonotify(desc, val); if (ret) goto out_free_linereq; } else if (flags & GPIO_V2_LINE_FLAG_INPUT) { - ret = gpiod_direction_input(desc); + ret = gpiod_direction_input_nonotify(desc); if (ret) goto out_free_linereq; ret = edge_detector_setup(&lr->lines[i], lc, i, - flags & GPIO_V2_LINE_EDGE_FLAGS); + edflags); if (ret) goto out_free_linereq; } - blocking_notifier_call_chain(&desc->gdev->notifier, - GPIO_V2_LINE_CHANGED_REQUESTED, desc); + lr->lines[i].edflags = edflags; + + gpiod_line_state_notify(desc, GPIO_V2_LINE_CHANGED_REQUESTED); dev_dbg(&gdev->dev, "registered chardev handle for line %d\n", offset); } + lr->device_unregistered_nb.notifier_call = linereq_unregistered_notify; + ret = blocking_notifier_chain_register(&gdev->device_notifier, + &lr->device_unregistered_nb); + if (ret) + goto out_free_linereq; + fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC); if (fd < 0) { ret = fd; @@ -1470,6 +1754,7 @@ out_free_linereq: * @eflags: the event flags this line was requested with * @irq: the interrupt that trigger in response to events on this GPIO * @wait: wait queue that handles blocking reads of events + * @device_unregistered_nb: notifier block for receiving gdev unregister events * @events: KFIFO for the GPIO events * @timestamp: cache for the timestamp storing it between hardirq * and IRQ thread, used to bring the timestamp close to the actual @@ -1482,6 +1767,7 @@ struct lineevent_state { u32 eflags; int irq; wait_queue_head_t wait; + struct notifier_block device_unregistered_nb; DECLARE_KFIFO(events, struct gpioevent_data, 16); u64 timestamp; }; @@ -1496,6 +1782,11 @@ static __poll_t lineevent_poll(struct file *file, struct lineevent_state *le = file->private_data; __poll_t events = 0; + guard(srcu)(&le->gdev->srcu); + + if (!rcu_access_pointer(le->gdev->chip)) + return EPOLLHUP | EPOLLERR; + poll_wait(file, &le->wait, wait); if (!kfifo_is_empty_spinlocked_noirqsave(&le->events, &le->wait.lock)) @@ -1504,15 +1795,24 @@ static __poll_t lineevent_poll(struct file *file, return events; } +static int lineevent_unregistered_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct lineevent_state *le = container_of(nb, struct lineevent_state, + device_unregistered_nb); + + wake_up_poll(&le->wait, EPOLLIN | EPOLLERR); + + return NOTIFY_OK; +} + struct compat_gpioeevent_data { compat_u64 timestamp; u32 id; }; -static ssize_t lineevent_read(struct file *file, - char __user *buf, - size_t count, - loff_t *f_ps) +static ssize_t lineevent_read(struct file *file, char __user *buf, + size_t count, loff_t *f_ps) { struct lineevent_state *le = file->private_data; struct gpioevent_data ge; @@ -1520,6 +1820,11 @@ static ssize_t lineevent_read(struct file *file, ssize_t ge_size; int ret; + guard(srcu)(&le->gdev->srcu); + + if (!rcu_access_pointer(le->gdev->chip)) + return -ENODEV; + /* * When compatible system call is being used the struct gpioevent_data, * in case of at least ia32, has different size due to the alignment @@ -1537,38 +1842,31 @@ static ssize_t lineevent_read(struct file *file, return -EINVAL; do { - spin_lock(&le->wait.lock); - if (kfifo_is_empty(&le->events)) { - if (bytes_read) { - spin_unlock(&le->wait.lock); - return bytes_read; + scoped_guard(spinlock, &le->wait.lock) { + if (kfifo_is_empty(&le->events)) { + if (bytes_read) + return bytes_read; + + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + + ret = wait_event_interruptible_locked(le->wait, + !kfifo_is_empty(&le->events)); + if (ret) + return ret; } - if (file->f_flags & O_NONBLOCK) { - spin_unlock(&le->wait.lock); - return -EAGAIN; - } - - ret = wait_event_interruptible_locked(le->wait, - !kfifo_is_empty(&le->events)); - if (ret) { - spin_unlock(&le->wait.lock); - return ret; + if (kfifo_out(&le->events, &ge, 1) != 1) { + /* + * This should never happen - we hold the + * lock from the moment we learned the fifo + * is no longer empty until now. + */ + WARN(1, "failed to read from non-empty kfifo"); + return -EIO; } } - ret = kfifo_out(&le->events, &ge, 1); - spin_unlock(&le->wait.lock); - if (ret != 1) { - /* - * This should never happen - we were holding the lock - * from the moment we learned the fifo is no longer - * empty until now. - */ - ret = -EIO; - break; - } - if (copy_to_user(buf + bytes_read, &ge, ge_size)) return -EFAULT; bytes_read += ge_size; @@ -1579,12 +1877,15 @@ static ssize_t lineevent_read(struct file *file, static void lineevent_free(struct lineevent_state *le) { + if (le->device_unregistered_nb.notifier_call) + blocking_notifier_chain_unregister(&le->gdev->device_notifier, + &le->device_unregistered_nb); if (le->irq) - free_irq(le->irq, le); + free_irq_label(free_irq(le->irq, le)); if (le->desc) gpiod_free(le->desc); kfree(le->label); - put_device(&le->gdev->dev); + gpio_device_put(le->gdev); kfree(le); } @@ -1601,6 +1902,11 @@ static long lineevent_ioctl(struct file *file, unsigned int cmd, void __user *ip = (void __user *)arg; struct gpiohandle_data ghd; + guard(srcu)(&le->gdev->srcu); + + if (!rcu_access_pointer(le->gdev->chip)) + return -ENODEV; + /* * We can get the value for an event line but not set it, * because it is input by definition. @@ -1716,6 +2022,7 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) int fd; int ret; int irq, irqflags = 0; + char *label; if (copy_from_user(&eventreq, ip, sizeof(eventreq))) return -EFAULT; @@ -1724,7 +2031,7 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) lflags = eventreq.handleflags; eflags = eventreq.eventflags; - desc = gpiochip_get_desc(gdev->chip, offset); + desc = gpio_device_get_desc(gdev, offset); if (IS_ERR(desc)) return PTR_ERR(desc); @@ -1750,8 +2057,7 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) le = kzalloc(sizeof(*le), GFP_KERNEL); if (!le) return -ENOMEM; - le->gdev = gdev; - get_device(&gdev->dev); + le->gdev = gpio_device_get(gdev); if (eventreq.consumer_label[0] != '\0') { /* label is only initialized if consumer_label is set */ @@ -1764,7 +2070,7 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) } } - ret = gpiod_request(desc, le->label); + ret = gpiod_request_user(desc, le->label); if (ret) goto out_free_le; le->desc = desc; @@ -1776,36 +2082,50 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) if (ret) goto out_free_le; - blocking_notifier_call_chain(&desc->gdev->notifier, - GPIO_V2_LINE_CHANGED_REQUESTED, desc); + gpiod_line_state_notify(desc, GPIO_V2_LINE_CHANGED_REQUESTED); irq = gpiod_to_irq(desc); if (irq <= 0) { ret = -ENODEV; goto out_free_le; } - le->irq = irq; if (eflags & GPIOEVENT_REQUEST_RISING_EDGE) - irqflags |= test_bit(FLAG_ACTIVE_LOW, &desc->flags) ? + irqflags |= test_bit(GPIOD_FLAG_ACTIVE_LOW, &desc->flags) ? IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING; if (eflags & GPIOEVENT_REQUEST_FALLING_EDGE) - irqflags |= test_bit(FLAG_ACTIVE_LOW, &desc->flags) ? + irqflags |= test_bit(GPIOD_FLAG_ACTIVE_LOW, &desc->flags) ? IRQF_TRIGGER_RISING : IRQF_TRIGGER_FALLING; irqflags |= IRQF_ONESHOT; INIT_KFIFO(le->events); init_waitqueue_head(&le->wait); + le->device_unregistered_nb.notifier_call = lineevent_unregistered_notify; + ret = blocking_notifier_chain_register(&gdev->device_notifier, + &le->device_unregistered_nb); + if (ret) + goto out_free_le; + + label = make_irq_label(le->label); + if (IS_ERR(label)) { + ret = PTR_ERR(label); + goto out_free_le; + } + /* Request a thread to read the events */ - ret = request_threaded_irq(le->irq, + ret = request_threaded_irq(irq, lineevent_irq_handler, lineevent_irq_thread, irqflags, - le->label, + label, le); - if (ret) + if (ret) { + free_irq_label(label); goto out_free_le; + } + + le->irq = irq; fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC); if (fd < 0) { @@ -1889,86 +2209,91 @@ static void gpio_v2_line_info_changed_to_v1( #endif /* CONFIG_GPIO_CDEV_V1 */ static void gpio_desc_to_lineinfo(struct gpio_desc *desc, - struct gpio_v2_line_info *info) + struct gpio_v2_line_info *info, bool atomic) { - struct gpio_chip *gc = desc->gdev->chip; - bool ok_for_pinctrl; - unsigned long flags; u32 debounce_period_us; - unsigned int num_attrs = 0; - - memset(info, 0, sizeof(*info)); - info->offset = gpio_chip_hwgpio(desc); + unsigned long dflags; + const char *label; - /* - * This function takes a mutex so we must check this before taking - * the spinlock. - * - * FIXME: find a non-racy way to retrieve this information. Maybe a - * lock common to both frameworks? - */ - ok_for_pinctrl = - pinctrl_gpio_can_use_line(gc->base + info->offset); + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return; - spin_lock_irqsave(&gpio_lock, flags); + memset(info, 0, sizeof(*info)); + info->offset = gpiod_hwgpio(desc); if (desc->name) strscpy(info->name, desc->name, sizeof(info->name)); - if (desc->label) - strscpy(info->consumer, desc->label, sizeof(info->consumer)); + dflags = READ_ONCE(desc->flags); + + scoped_guard(srcu, &desc->gdev->desc_srcu) { + label = gpiod_get_label(desc); + if (label && test_bit(GPIOD_FLAG_REQUESTED, &dflags)) + strscpy(info->consumer, label, + sizeof(info->consumer)); + } /* - * Userspace only need to know that the kernel is using this GPIO so - * it can't use it. + * Userspace only need know that the kernel is using this GPIO so it + * can't use it. + * The calculation of the used flag is slightly racy, as it may read + * desc, gc and pinctrl state without a lock covering all three at + * once. Worst case if the line is in transition and the calculation + * is inconsistent then it looks to the user like they performed the + * read on the other side of the transition - but that can always + * happen. + * The definitive test that a line is available to userspace is to + * request it. */ - info->flags = 0; - if (test_bit(FLAG_REQUESTED, &desc->flags) || - test_bit(FLAG_IS_HOGGED, &desc->flags) || - test_bit(FLAG_USED_AS_IRQ, &desc->flags) || - test_bit(FLAG_EXPORT, &desc->flags) || - test_bit(FLAG_SYSFS, &desc->flags) || - !gpiochip_line_is_valid(gc, info->offset) || - !ok_for_pinctrl) + if (test_bit(GPIOD_FLAG_REQUESTED, &dflags) || + test_bit(GPIOD_FLAG_IS_HOGGED, &dflags) || + test_bit(GPIOD_FLAG_EXPORT, &dflags) || + test_bit(GPIOD_FLAG_SYSFS, &dflags) || + !gpiochip_line_is_valid(guard.gc, info->offset)) { info->flags |= GPIO_V2_LINE_FLAG_USED; + } else if (!atomic) { + if (!pinctrl_gpio_can_use_line(guard.gc, info->offset)) + info->flags |= GPIO_V2_LINE_FLAG_USED; + } - if (test_bit(FLAG_IS_OUT, &desc->flags)) + if (test_bit(GPIOD_FLAG_IS_OUT, &dflags)) info->flags |= GPIO_V2_LINE_FLAG_OUTPUT; else info->flags |= GPIO_V2_LINE_FLAG_INPUT; - if (test_bit(FLAG_ACTIVE_LOW, &desc->flags)) + if (test_bit(GPIOD_FLAG_ACTIVE_LOW, &dflags)) info->flags |= GPIO_V2_LINE_FLAG_ACTIVE_LOW; - if (test_bit(FLAG_OPEN_DRAIN, &desc->flags)) + if (test_bit(GPIOD_FLAG_OPEN_DRAIN, &dflags)) info->flags |= GPIO_V2_LINE_FLAG_OPEN_DRAIN; - if (test_bit(FLAG_OPEN_SOURCE, &desc->flags)) + if (test_bit(GPIOD_FLAG_OPEN_SOURCE, &dflags)) info->flags |= GPIO_V2_LINE_FLAG_OPEN_SOURCE; - if (test_bit(FLAG_BIAS_DISABLE, &desc->flags)) + if (test_bit(GPIOD_FLAG_BIAS_DISABLE, &dflags)) info->flags |= GPIO_V2_LINE_FLAG_BIAS_DISABLED; - if (test_bit(FLAG_PULL_DOWN, &desc->flags)) + if (test_bit(GPIOD_FLAG_PULL_DOWN, &dflags)) info->flags |= GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN; - if (test_bit(FLAG_PULL_UP, &desc->flags)) + if (test_bit(GPIOD_FLAG_PULL_UP, &dflags)) info->flags |= GPIO_V2_LINE_FLAG_BIAS_PULL_UP; - if (test_bit(FLAG_EDGE_RISING, &desc->flags)) + if (test_bit(GPIOD_FLAG_EDGE_RISING, &dflags)) info->flags |= GPIO_V2_LINE_FLAG_EDGE_RISING; - if (test_bit(FLAG_EDGE_FALLING, &desc->flags)) + if (test_bit(GPIOD_FLAG_EDGE_FALLING, &dflags)) info->flags |= GPIO_V2_LINE_FLAG_EDGE_FALLING; - if (test_bit(FLAG_EVENT_CLOCK_REALTIME, &desc->flags)) + if (test_bit(GPIOD_FLAG_EVENT_CLOCK_REALTIME, &dflags)) info->flags |= GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME; + else if (test_bit(GPIOD_FLAG_EVENT_CLOCK_HTE, &dflags)) + info->flags |= GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE; debounce_period_us = READ_ONCE(desc->debounce_period_us); if (debounce_period_us) { - info->attrs[num_attrs].id = GPIO_V2_LINE_ATTR_ID_DEBOUNCE; - info->attrs[num_attrs].debounce_period_us = debounce_period_us; - num_attrs++; + info->attrs[info->num_attrs].id = GPIO_V2_LINE_ATTR_ID_DEBOUNCE; + info->attrs[info->num_attrs].debounce_period_us = + debounce_period_us; + info->num_attrs++; } - info->num_attrs = num_attrs; - - spin_unlock_irqrestore(&gpio_lock, flags); } struct gpio_chardev_data { @@ -1976,10 +2301,12 @@ struct gpio_chardev_data { wait_queue_head_t wait; DECLARE_KFIFO(events, struct gpio_v2_line_info_changed, 32); struct notifier_block lineinfo_changed_nb; + struct notifier_block device_unregistered_nb; unsigned long *watched_lines; #ifdef CONFIG_GPIO_CDEV_V1 atomic_t watch_abi_version; #endif + struct file *fp; }; static int chipinfo_get(struct gpio_chardev_data *cdev, void __user *ip) @@ -2023,7 +2350,7 @@ static int lineinfo_get_v1(struct gpio_chardev_data *cdev, void __user *ip, return -EFAULT; /* this doubles as a range check on line_offset */ - desc = gpiochip_get_desc(cdev->gdev->chip, lineinfo.line_offset); + desc = gpio_device_get_desc(cdev->gdev, lineinfo.line_offset); if (IS_ERR(desc)) return PTR_ERR(desc); @@ -2035,7 +2362,7 @@ static int lineinfo_get_v1(struct gpio_chardev_data *cdev, void __user *ip, return -EBUSY; } - gpio_desc_to_lineinfo(desc, &lineinfo_v2); + gpio_desc_to_lineinfo(desc, &lineinfo_v2, false); gpio_v2_line_info_to_v1(&lineinfo_v2, &lineinfo); if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) { @@ -2057,10 +2384,10 @@ static int lineinfo_get(struct gpio_chardev_data *cdev, void __user *ip, if (copy_from_user(&lineinfo, ip, sizeof(lineinfo))) return -EFAULT; - if (memchr_inv(lineinfo.padding, 0, sizeof(lineinfo.padding))) + if (!mem_is_zero(lineinfo.padding, sizeof(lineinfo.padding))) return -EINVAL; - desc = gpiochip_get_desc(cdev->gdev->chip, lineinfo.offset); + desc = gpio_device_get_desc(cdev->gdev, lineinfo.offset); if (IS_ERR(desc)) return PTR_ERR(desc); @@ -2072,7 +2399,7 @@ static int lineinfo_get(struct gpio_chardev_data *cdev, void __user *ip, if (test_and_set_bit(lineinfo.offset, cdev->watched_lines)) return -EBUSY; } - gpio_desc_to_lineinfo(desc, &lineinfo); + gpio_desc_to_lineinfo(desc, &lineinfo, false); if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) { if (watch) @@ -2108,33 +2435,37 @@ static long gpio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) struct gpio_device *gdev = cdev->gdev; void __user *ip = (void __user *)arg; + guard(srcu)(&gdev->srcu); + /* We fail any subsequent ioctl():s when the chip is gone */ - if (!gdev->chip) + if (!rcu_access_pointer(gdev->chip)) return -ENODEV; /* Fill in the struct and pass to userspace */ - if (cmd == GPIO_GET_CHIPINFO_IOCTL) { + switch (cmd) { + case GPIO_GET_CHIPINFO_IOCTL: return chipinfo_get(cdev, ip); #ifdef CONFIG_GPIO_CDEV_V1 - } else if (cmd == GPIO_GET_LINEHANDLE_IOCTL) { + case GPIO_GET_LINEHANDLE_IOCTL: return linehandle_create(gdev, ip); - } else if (cmd == GPIO_GET_LINEEVENT_IOCTL) { + case GPIO_GET_LINEEVENT_IOCTL: return lineevent_create(gdev, ip); - } else if (cmd == GPIO_GET_LINEINFO_IOCTL || - cmd == GPIO_GET_LINEINFO_WATCH_IOCTL) { - return lineinfo_get_v1(cdev, ip, - cmd == GPIO_GET_LINEINFO_WATCH_IOCTL); + case GPIO_GET_LINEINFO_IOCTL: + return lineinfo_get_v1(cdev, ip, false); + case GPIO_GET_LINEINFO_WATCH_IOCTL: + return lineinfo_get_v1(cdev, ip, true); #endif /* CONFIG_GPIO_CDEV_V1 */ - } else if (cmd == GPIO_V2_GET_LINEINFO_IOCTL || - cmd == GPIO_V2_GET_LINEINFO_WATCH_IOCTL) { - return lineinfo_get(cdev, ip, - cmd == GPIO_V2_GET_LINEINFO_WATCH_IOCTL); - } else if (cmd == GPIO_V2_GET_LINE_IOCTL) { + case GPIO_V2_GET_LINEINFO_IOCTL: + return lineinfo_get(cdev, ip, false); + case GPIO_V2_GET_LINEINFO_WATCH_IOCTL: + return lineinfo_get(cdev, ip, true); + case GPIO_V2_GET_LINE_IOCTL: return linereq_create(gdev, ip); - } else if (cmd == GPIO_GET_LINEINFO_UNWATCH_IOCTL) { + case GPIO_GET_LINEINFO_UNWATCH_IOCTL: return lineinfo_unwatch(cdev, ip); + default: + return -EINVAL; } - return -EINVAL; } #ifdef CONFIG_COMPAT @@ -2145,33 +2476,103 @@ static long gpio_ioctl_compat(struct file *file, unsigned int cmd, } #endif -static struct gpio_chardev_data * -to_gpio_chardev_data(struct notifier_block *nb) +struct lineinfo_changed_ctx { + struct work_struct work; + struct gpio_v2_line_info_changed chg; + struct gpio_device *gdev; + struct gpio_chardev_data *cdev; +}; + +static void lineinfo_changed_func(struct work_struct *work) { - return container_of(nb, struct gpio_chardev_data, lineinfo_changed_nb); + struct lineinfo_changed_ctx *ctx = + container_of(work, struct lineinfo_changed_ctx, work); + struct gpio_chip *gc; + int ret; + + if (!(ctx->chg.info.flags & GPIO_V2_LINE_FLAG_USED)) { + /* + * If nobody set the USED flag earlier, let's see with pinctrl + * now. We're doing this late because it's a sleeping function. + * Pin functions are in general much more static and while it's + * not 100% bullet-proof, it's good enough for most cases. + */ + scoped_guard(srcu, &ctx->gdev->srcu) { + gc = srcu_dereference(ctx->gdev->chip, &ctx->gdev->srcu); + if (gc && + !pinctrl_gpio_can_use_line(gc, ctx->chg.info.offset)) + ctx->chg.info.flags |= GPIO_V2_LINE_FLAG_USED; + } + } + + ret = kfifo_in_spinlocked(&ctx->cdev->events, &ctx->chg, 1, + &ctx->cdev->wait.lock); + if (ret) + wake_up_poll(&ctx->cdev->wait, EPOLLIN); + else + pr_debug_ratelimited("lineinfo event FIFO is full - event dropped\n"); + + gpio_device_put(ctx->gdev); + fput(ctx->cdev->fp); + kfree(ctx); } static int lineinfo_changed_notify(struct notifier_block *nb, unsigned long action, void *data) { - struct gpio_chardev_data *cdev = to_gpio_chardev_data(nb); - struct gpio_v2_line_info_changed chg; + struct gpio_chardev_data *cdev = + container_of(nb, struct gpio_chardev_data, lineinfo_changed_nb); + struct lineinfo_changed_ctx *ctx; struct gpio_desc *desc = data; - int ret; + struct file *fp; - if (!test_bit(gpio_chip_hwgpio(desc), cdev->watched_lines)) + if (!test_bit(gpiod_hwgpio(desc), cdev->watched_lines)) return NOTIFY_DONE; - memset(&chg, 0, sizeof(chg)); - chg.event_type = action; - chg.timestamp_ns = ktime_get_ns(); - gpio_desc_to_lineinfo(desc, &chg.info); + /* Keep the file descriptor alive for the duration of the notification. */ + fp = get_file_active(&cdev->fp); + if (!fp) + /* Chardev file descriptor was or is being released. */ + return NOTIFY_DONE; - ret = kfifo_in_spinlocked(&cdev->events, &chg, 1, &cdev->wait.lock); - if (ret) - wake_up_poll(&cdev->wait, EPOLLIN); - else - pr_debug_ratelimited("lineinfo event FIFO is full - event dropped\n"); + /* + * If this is called from atomic context (for instance: with a spinlock + * taken by the atomic notifier chain), any sleeping calls must be done + * outside of this function in process context of the dedicated + * workqueue. + * + * Let's gather as much info as possible from the descriptor and + * postpone just the call to pinctrl_gpio_can_use_line() until the work + * is executed. + */ + + ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC); + if (!ctx) { + pr_err("Failed to allocate memory for line info notification\n"); + return NOTIFY_DONE; + } + + ctx->chg.event_type = action; + ctx->chg.timestamp_ns = ktime_get_ns(); + gpio_desc_to_lineinfo(desc, &ctx->chg.info, true); + /* Keep the GPIO device alive until we emit the event. */ + ctx->gdev = gpio_device_get(desc->gdev); + ctx->cdev = cdev; + + INIT_WORK(&ctx->work, lineinfo_changed_func); + queue_work(ctx->gdev->line_state_wq, &ctx->work); + + return NOTIFY_OK; +} + +static int gpio_device_unregistered_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct gpio_chardev_data *cdev = container_of(nb, + struct gpio_chardev_data, + device_unregistered_nb); + + wake_up_poll(&cdev->wait, EPOLLIN | EPOLLERR); return NOTIFY_OK; } @@ -2182,6 +2583,11 @@ static __poll_t lineinfo_watch_poll(struct file *file, struct gpio_chardev_data *cdev = file->private_data; __poll_t events = 0; + guard(srcu)(&cdev->gdev->srcu); + + if (!rcu_access_pointer(cdev->gdev->chip)) + return EPOLLHUP | EPOLLERR; + poll_wait(file, &cdev->wait, pollt); if (!kfifo_is_empty_spinlocked_noirqsave(&cdev->events, @@ -2200,6 +2606,11 @@ static ssize_t lineinfo_watch_read(struct file *file, char __user *buf, int ret; size_t event_size; + guard(srcu)(&cdev->gdev->srcu); + + if (!rcu_access_pointer(cdev->gdev->chip)) + return -ENODEV; + #ifndef CONFIG_GPIO_CDEV_V1 event_size = sizeof(struct gpio_v2_line_info_changed); if (count < event_size) @@ -2207,42 +2618,37 @@ static ssize_t lineinfo_watch_read(struct file *file, char __user *buf, #endif do { - spin_lock(&cdev->wait.lock); - if (kfifo_is_empty(&cdev->events)) { - if (bytes_read) { - spin_unlock(&cdev->wait.lock); - return bytes_read; - } - - if (file->f_flags & O_NONBLOCK) { - spin_unlock(&cdev->wait.lock); - return -EAGAIN; + scoped_guard(spinlock, &cdev->wait.lock) { + if (kfifo_is_empty(&cdev->events)) { + if (bytes_read) + return bytes_read; + + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + + ret = wait_event_interruptible_locked(cdev->wait, + !kfifo_is_empty(&cdev->events)); + if (ret) + return ret; } - - ret = wait_event_interruptible_locked(cdev->wait, - !kfifo_is_empty(&cdev->events)); - if (ret) { - spin_unlock(&cdev->wait.lock); - return ret; - } - } #ifdef CONFIG_GPIO_CDEV_V1 - /* must be after kfifo check so watch_abi_version is set */ - if (atomic_read(&cdev->watch_abi_version) == 2) - event_size = sizeof(struct gpio_v2_line_info_changed); - else - event_size = sizeof(struct gpioline_info_changed); - if (count < event_size) { - spin_unlock(&cdev->wait.lock); - return -EINVAL; - } + /* must be after kfifo check so watch_abi_version is set */ + if (atomic_read(&cdev->watch_abi_version) == 2) + event_size = sizeof(struct gpio_v2_line_info_changed); + else + event_size = sizeof(struct gpioline_info_changed); + if (count < event_size) + return -EINVAL; #endif - ret = kfifo_out(&cdev->events, &event, 1); - spin_unlock(&cdev->wait.lock); - if (ret != 1) { - ret = -EIO; - break; - /* We should never get here. See lineevent_read(). */ + if (kfifo_out(&cdev->events, &event, 1) != 1) { + /* + * This should never happen - we hold the + * lock from the moment we learned the fifo + * is no longer empty until now. + */ + WARN(1, "failed to read from non-empty kfifo"); + return -EIO; + } } #ifdef CONFIG_GPIO_CDEV_V1 @@ -2271,7 +2677,9 @@ static ssize_t lineinfo_watch_read(struct file *file, char __user *buf, * gpio_chrdev_open() - open the chardev for ioctl operations * @inode: inode for this chardev * @file: file struct for storing private data - * Returns 0 on success + * + * Returns: + * 0 on success, or negative errno on failure. */ static int gpio_chrdev_open(struct inode *inode, struct file *file) { @@ -2280,41 +2688,56 @@ static int gpio_chrdev_open(struct inode *inode, struct file *file) struct gpio_chardev_data *cdev; int ret = -ENOMEM; + guard(srcu)(&gdev->srcu); + /* Fail on open if the backing gpiochip is gone */ - if (!gdev->chip) + if (!rcu_access_pointer(gdev->chip)) return -ENODEV; cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); if (!cdev) - return -ENOMEM; + return -ENODEV; - cdev->watched_lines = bitmap_zalloc(gdev->chip->ngpio, GFP_KERNEL); + cdev->watched_lines = bitmap_zalloc(gdev->ngpio, GFP_KERNEL); if (!cdev->watched_lines) goto out_free_cdev; init_waitqueue_head(&cdev->wait); INIT_KFIFO(cdev->events); - cdev->gdev = gdev; + cdev->gdev = gpio_device_get(gdev); cdev->lineinfo_changed_nb.notifier_call = lineinfo_changed_notify; - ret = blocking_notifier_chain_register(&gdev->notifier, - &cdev->lineinfo_changed_nb); + scoped_guard(write_lock_irqsave, &gdev->line_state_lock) + ret = raw_notifier_chain_register(&gdev->line_state_notifier, + &cdev->lineinfo_changed_nb); if (ret) goto out_free_bitmap; - get_device(&gdev->dev); + cdev->device_unregistered_nb.notifier_call = + gpio_device_unregistered_notify; + ret = blocking_notifier_chain_register(&gdev->device_notifier, + &cdev->device_unregistered_nb); + if (ret) + goto out_unregister_line_notifier; + file->private_data = cdev; + cdev->fp = file; ret = nonseekable_open(inode, file); if (ret) - goto out_unregister_notifier; + goto out_unregister_device_notifier; return ret; -out_unregister_notifier: - blocking_notifier_chain_unregister(&gdev->notifier, - &cdev->lineinfo_changed_nb); +out_unregister_device_notifier: + blocking_notifier_chain_unregister(&gdev->device_notifier, + &cdev->device_unregistered_nb); +out_unregister_line_notifier: + scoped_guard(write_lock_irqsave, &gdev->line_state_lock) + raw_notifier_chain_unregister(&gdev->line_state_notifier, + &cdev->lineinfo_changed_nb); out_free_bitmap: + gpio_device_put(gdev); bitmap_free(cdev->watched_lines); out_free_cdev: kfree(cdev); @@ -2325,17 +2748,22 @@ out_free_cdev: * gpio_chrdev_release() - close chardev after ioctl operations * @inode: inode for this chardev * @file: file struct for storing private data - * Returns 0 on success + * + * Returns: + * 0 on success, or negative errno on failure. */ static int gpio_chrdev_release(struct inode *inode, struct file *file) { struct gpio_chardev_data *cdev = file->private_data; struct gpio_device *gdev = cdev->gdev; + blocking_notifier_chain_unregister(&gdev->device_notifier, + &cdev->device_unregistered_nb); + scoped_guard(write_lock_irqsave, &gdev->line_state_lock) + raw_notifier_chain_unregister(&gdev->line_state_notifier, + &cdev->lineinfo_changed_nb); bitmap_free(cdev->watched_lines); - blocking_notifier_chain_unregister(&gdev->notifier, - &cdev->lineinfo_changed_nb); - put_device(&gdev->dev); + gpio_device_put(gdev); kfree(cdev); return 0; @@ -2347,7 +2775,6 @@ static const struct file_operations gpio_fileops = { .poll = lineinfo_watch_poll, .read = lineinfo_watch_read, .owner = THIS_MODULE, - .llseek = no_llseek, .unlocked_ioctl = gpio_ioctl, #ifdef CONFIG_COMPAT .compat_ioctl = gpio_ioctl_compat, @@ -2356,23 +2783,35 @@ static const struct file_operations gpio_fileops = { int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt) { + struct gpio_chip *gc; int ret; cdev_init(&gdev->chrdev, &gpio_fileops); gdev->chrdev.owner = THIS_MODULE; gdev->dev.devt = MKDEV(MAJOR(devt), gdev->id); + gdev->line_state_wq = alloc_ordered_workqueue("%s", WQ_HIGHPRI, + dev_name(&gdev->dev)); + if (!gdev->line_state_wq) + return -ENOMEM; + ret = cdev_device_add(&gdev->chrdev, &gdev->dev); if (ret) return ret; - chip_dbg(gdev->chip, "added GPIO chardev (%d:%d)\n", - MAJOR(devt), gdev->id); + guard(srcu)(&gdev->srcu); + gc = srcu_dereference(gdev->chip, &gdev->srcu); + if (!gc) + return -ENODEV; + + gpiochip_dbg(gc, "added GPIO chardev (%d:%d)\n", MAJOR(devt), gdev->id); return 0; } void gpiolib_cdev_unregister(struct gpio_device *gdev) { + destroy_workqueue(gdev->line_state_wq); cdev_device_del(&gdev->chrdev, &gdev->dev); + blocking_notifier_call_chain(&gdev->device_notifier, 0, NULL); } |