diff options
Diffstat (limited to 'drivers/tty/n_tty.c')
| -rw-r--r-- | drivers/tty/n_tty.c | 1077 |
1 files changed, 545 insertions, 532 deletions
diff --git a/drivers/tty/n_tty.c b/drivers/tty/n_tty.c index c8f56c9b1a1c..e6a0f5b40d0a 100644 --- a/drivers/tty/n_tty.c +++ b/drivers/tty/n_tty.c @@ -28,27 +28,26 @@ * EAGAIN */ -#include <linux/types.h> -#include <linux/major.h> +#include <linux/bitmap.h> +#include <linux/bitops.h> +#include <linux/ctype.h> #include <linux/errno.h> -#include <linux/signal.h> +#include <linux/export.h> #include <linux/fcntl.h> +#include <linux/file.h> +#include <linux/jiffies.h> +#include <linux/math.h> +#include <linux/poll.h> +#include <linux/ratelimit.h> #include <linux/sched.h> -#include <linux/interrupt.h> -#include <linux/tty.h> -#include <linux/timer.h> -#include <linux/ctype.h> -#include <linux/mm.h> -#include <linux/string.h> +#include <linux/signal.h> #include <linux/slab.h> -#include <linux/poll.h> -#include <linux/bitops.h> -#include <linux/audit.h> -#include <linux/file.h> +#include <linux/string.h> +#include <linux/tty.h> +#include <linux/types.h> #include <linux/uaccess.h> -#include <linux/module.h> -#include <linux/ratelimit.h> #include <linux/vmalloc.h> + #include "tty.h" /* @@ -57,6 +56,8 @@ */ #define WAKEUP_CHARS 256 +#define N_TTY_BUF_SIZE 4096 + /* * This defines the low- and high-watermarks for throttling and * unthrottling the TTY driver. These watermarks are used for @@ -80,14 +81,6 @@ #define ECHO_BLOCK 256 #define ECHO_DISCARD_WATERMARK N_TTY_BUF_SIZE - (ECHO_BLOCK + 32) - -#undef N_TTY_TRACE -#ifdef N_TTY_TRACE -# define n_tty_trace(f, args...) trace_printk(f, ##args) -#else -# define n_tty_trace(f, args...) no_printk(f, ##args) -#endif - struct n_tty_data { /* producer-published */ size_t read_head; @@ -100,7 +93,7 @@ struct n_tty_data { /* private to n_tty_receive_overrun (single-threaded) */ unsigned long overrun_time; - int num_overrun; + unsigned int num_overrun; /* non-atomic */ bool no_room; @@ -110,9 +103,9 @@ struct n_tty_data { unsigned char push:1; /* shared by producer and consumer */ - char read_buf[N_TTY_BUF_SIZE]; + u8 read_buf[N_TTY_BUF_SIZE]; DECLARE_BITMAP(read_flags, N_TTY_BUF_SIZE); - unsigned char echo_buf[N_TTY_BUF_SIZE]; + u8 echo_buf[N_TTY_BUF_SIZE]; /* consumer-published */ size_t read_tail; @@ -137,38 +130,36 @@ static inline size_t read_cnt(struct n_tty_data *ldata) return ldata->read_head - ldata->read_tail; } -static inline unsigned char read_buf(struct n_tty_data *ldata, size_t i) +static inline u8 read_buf(struct n_tty_data *ldata, size_t i) { - return ldata->read_buf[i & (N_TTY_BUF_SIZE - 1)]; + return ldata->read_buf[MASK(i)]; } -static inline unsigned char *read_buf_addr(struct n_tty_data *ldata, size_t i) +static inline u8 *read_buf_addr(struct n_tty_data *ldata, size_t i) { - return &ldata->read_buf[i & (N_TTY_BUF_SIZE - 1)]; + return &ldata->read_buf[MASK(i)]; } -static inline unsigned char echo_buf(struct n_tty_data *ldata, size_t i) +static inline u8 echo_buf(struct n_tty_data *ldata, size_t i) { smp_rmb(); /* Matches smp_wmb() in add_echo_byte(). */ - return ldata->echo_buf[i & (N_TTY_BUF_SIZE - 1)]; + return ldata->echo_buf[MASK(i)]; } -static inline unsigned char *echo_buf_addr(struct n_tty_data *ldata, size_t i) +static inline u8 *echo_buf_addr(struct n_tty_data *ldata, size_t i) { - return &ldata->echo_buf[i & (N_TTY_BUF_SIZE - 1)]; + return &ldata->echo_buf[MASK(i)]; } /* If we are not echoing the data, perhaps this is a secret so erase it */ -static void zero_buffer(struct tty_struct *tty, u8 *buffer, int size) +static void zero_buffer(const struct tty_struct *tty, u8 *buffer, size_t size) { - bool icanon = !!L_ICANON(tty); - bool no_echo = !L_ECHO(tty); - - if (icanon && no_echo) - memset(buffer, 0x00, size); + if (L_ICANON(tty) && !L_ECHO(tty)) + memset(buffer, 0, size); } -static void tty_copy(struct tty_struct *tty, void *to, size_t tail, size_t n) +static void tty_copy(const struct tty_struct *tty, void *to, size_t tail, + size_t n) { struct n_tty_data *ldata = tty->disc_data; size_t size = N_TTY_BUF_SIZE - tail; @@ -199,13 +190,13 @@ static void tty_copy(struct tty_struct *tty, void *to, size_t tail, size_t n) * * n_tty_read()/consumer path: * holds non-exclusive %termios_rwsem */ -static void n_tty_kick_worker(struct tty_struct *tty) +static void n_tty_kick_worker(const struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; /* Did the input worker stop? Restart it */ - if (unlikely(ldata->no_room)) { - ldata->no_room = 0; + if (unlikely(READ_ONCE(ldata->no_room))) { + WRITE_ONCE(ldata->no_room, 0); WARN_RATELIMIT(tty->port->itty == NULL, "scheduling with invalid itty\n"); @@ -219,16 +210,12 @@ static void n_tty_kick_worker(struct tty_struct *tty) } } -static ssize_t chars_in_buffer(struct tty_struct *tty) +static ssize_t chars_in_buffer(const struct tty_struct *tty) { - struct n_tty_data *ldata = tty->disc_data; - ssize_t n = 0; + const struct n_tty_data *ldata = tty->disc_data; + size_t head = ldata->icanon ? ldata->canon_head : ldata->commit_head; - if (!ldata->icanon) - n = ldata->commit_head - ldata->read_tail; - else - n = ldata->canon_head - ldata->read_tail; - return n; + return head - ldata->read_tail; } /** @@ -256,15 +243,12 @@ static void n_tty_check_throttle(struct tty_struct *tty) if (ldata->icanon && ldata->canon_head == ldata->read_tail) return; - while (1) { - int throttled; + do { tty_set_flow_change(tty, TTY_THROTTLE_SAFE); if (N_TTY_BUF_SIZE - read_cnt(ldata) >= TTY_THRESHOLD_THROTTLE) break; - throttled = tty_throttle_safe(tty); - if (!throttled) - break; - } + } while (!tty_throttle_safe(tty)); + __tty_set_flow_change(tty, 0); } @@ -286,16 +270,14 @@ static void n_tty_check_unthrottle(struct tty_struct *tty) * we won't get any more characters. */ - while (1) { - int unthrottled; + do { tty_set_flow_change(tty, TTY_UNTHROTTLE_SAFE); if (chars_in_buffer(tty) > TTY_THRESHOLD_UNTHROTTLE) break; + n_tty_kick_worker(tty); - unthrottled = tty_unthrottle_safe(tty); - if (!unthrottled) - break; - } + } while (!tty_unthrottle_safe(tty)); + __tty_set_flow_change(tty, 0); } @@ -310,7 +292,7 @@ static void n_tty_check_unthrottle(struct tty_struct *tty) * * n_tty_receive_buf()/producer path: * caller holds non-exclusive %termios_rwsem */ -static inline void put_tty_queue(unsigned char c, struct n_tty_data *ldata) +static inline void put_tty_queue(u8 c, struct n_tty_data *ldata) { *read_buf_addr(ldata, ldata->read_head) = c; ldata->read_head++; @@ -342,12 +324,9 @@ static void reset_buffer_flags(struct n_tty_data *ldata) static void n_tty_packet_mode_flush(struct tty_struct *tty) { - unsigned long flags; - if (tty->link->ctrl.packet) { - spin_lock_irqsave(&tty->ctrl.lock, flags); - tty->ctrl.pktstatus |= TIOCPKT_FLUSHREAD; - spin_unlock_irqrestore(&tty->ctrl.lock, flags); + scoped_guard(spinlock_irqsave, &tty->ctrl.lock) + tty->ctrl.pktstatus |= TIOCPKT_FLUSHREAD; wake_up_interruptible(&tty->link->read_wait); } } @@ -367,13 +346,12 @@ static void n_tty_packet_mode_flush(struct tty_struct *tty) */ static void n_tty_flush_buffer(struct tty_struct *tty) { - down_write(&tty->termios_rwsem); + guard(rwsem_write)(&tty->termios_rwsem); reset_buffer_flags(tty->disc_data); n_tty_kick_worker(tty); if (tty->link) n_tty_packet_mode_flush(tty); - up_write(&tty->termios_rwsem); } /** @@ -384,7 +362,7 @@ static void n_tty_flush_buffer(struct tty_struct *tty) * character. We use this to correctly compute the on-screen size of the * character when printing. */ -static inline int is_utf8_continuation(unsigned char c) +static inline int is_utf8_continuation(u8 c) { return (c & 0xc0) == 0x80; } @@ -397,7 +375,7 @@ static inline int is_utf8_continuation(unsigned char c) * Returns: true if the utf8 character @c is a multibyte continuation character * and the terminal is in unicode mode. */ -static inline int is_continuation(unsigned char c, struct tty_struct *tty) +static inline int is_continuation(u8 c, const struct tty_struct *tty) { return I_IUTF8(tty) && is_utf8_continuation(c); } @@ -421,7 +399,7 @@ static inline int is_continuation(unsigned char c, struct tty_struct *tty) * Locking: should be called under the %output_lock to protect the column state * and space left in the buffer. */ -static int do_output_char(unsigned char c, struct tty_struct *tty, int space) +static int do_output_char(u8 c, struct tty_struct *tty, int space) { struct n_tty_data *ldata = tty->disc_data; int spaces; @@ -495,21 +473,16 @@ static int do_output_char(unsigned char c, struct tty_struct *tty, int space) * Locking: %output_lock to protect column state and space left (also, this is *called from n_tty_write() under the tty layer write lock). */ -static int process_output(unsigned char c, struct tty_struct *tty) +static int process_output(u8 c, struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; - int space, retval; - - mutex_lock(&ldata->output_lock); - space = tty_write_room(tty); - retval = do_output_char(c, tty, space); + guard(mutex)(&ldata->output_lock); - mutex_unlock(&ldata->output_lock); - if (retval < 0) + if (do_output_char(c, tty, tty_write_room(tty)) < 0) return -1; - else - return 0; + + return 0; } /** @@ -531,43 +504,41 @@ static int process_output(unsigned char c, struct tty_struct *tty) * called from n_tty_write() under the tty layer write lock). */ static ssize_t process_output_block(struct tty_struct *tty, - const unsigned char *buf, unsigned int nr) + const u8 *buf, unsigned int nr) { struct n_tty_data *ldata = tty->disc_data; - int space; - int i; - const unsigned char *cp; + unsigned int space, i; + const u8 *cp; - mutex_lock(&ldata->output_lock); + guard(mutex)(&ldata->output_lock); space = tty_write_room(tty); - if (space <= 0) { - mutex_unlock(&ldata->output_lock); - return space; - } + if (space == 0) + return 0; + if (nr > space) nr = space; for (i = 0, cp = buf; i < nr; i++, cp++) { - unsigned char c = *cp; + u8 c = *cp; switch (c) { case '\n': if (O_ONLRET(tty)) ldata->column = 0; if (O_ONLCR(tty)) - goto break_out; + goto do_write; ldata->canon_column = ldata->column; break; case '\r': if (O_ONOCR(tty) && ldata->column == 0) - goto break_out; + goto do_write; if (O_OCRNL(tty)) - goto break_out; + goto do_write; ldata->canon_column = ldata->column = 0; break; case '\t': - goto break_out; + goto do_write; case '\b': if (ldata->column > 0) ldata->column--; @@ -575,18 +546,109 @@ static ssize_t process_output_block(struct tty_struct *tty, default: if (!iscntrl(c)) { if (O_OLCUC(tty)) - goto break_out; + goto do_write; if (!is_continuation(c, tty)) ldata->column++; } break; } } -break_out: - i = tty->ops->write(tty, buf, i); +do_write: + return tty->ops->write(tty, buf, i); +} + +static int n_tty_process_echo_ops(struct tty_struct *tty, size_t *tail, + int space) +{ + struct n_tty_data *ldata = tty->disc_data; + u8 op; + + /* + * Since add_echo_byte() is called without holding output_lock, we + * might see only portion of multi-byte operation. + */ + if (MASK(ldata->echo_commit) == MASK(*tail + 1)) + return -ENODATA; + + /* + * If the buffer byte is the start of a multi-byte operation, get the + * next byte, which is either the op code or a control character value. + */ + op = echo_buf(ldata, *tail + 1); + + switch (op) { + case ECHO_OP_ERASE_TAB: { + unsigned int num_chars, num_bs; + + if (MASK(ldata->echo_commit) == MASK(*tail + 2)) + return -ENODATA; + + num_chars = echo_buf(ldata, *tail + 2); + + /* + * Determine how many columns to go back in order to erase the + * tab. This depends on the number of columns used by other + * characters within the tab area. If this (modulo 8) count is + * from the start of input rather than from a previous tab, we + * offset by canon column. Otherwise, tab spacing is normal. + */ + if (!(num_chars & 0x80)) + num_chars += ldata->canon_column; + num_bs = 8 - (num_chars & 7); + + if (num_bs > space) + return -ENOSPC; + + space -= num_bs; + while (num_bs--) { + tty_put_char(tty, '\b'); + if (ldata->column > 0) + ldata->column--; + } + *tail += 3; + break; + } + case ECHO_OP_SET_CANON_COL: + ldata->canon_column = ldata->column; + *tail += 2; + break; + + case ECHO_OP_MOVE_BACK_COL: + if (ldata->column > 0) + ldata->column--; + *tail += 2; + break; + + case ECHO_OP_START: + /* This is an escaped echo op start code */ + if (!space) + return -ENOSPC; + + tty_put_char(tty, ECHO_OP_START); + ldata->column++; + space--; + *tail += 2; + break; + + default: + /* + * If the op is not a special byte code, it is a ctrl char + * tagged to be echoed as "^X" (where X is the letter + * representing the control char). Note that we must ensure + * there is enough space for the whole ctrl pair. + */ + if (space < 2) + return -ENOSPC; + + tty_put_char(tty, '^'); + tty_put_char(tty, op ^ 0100); + ldata->column += 2; + space -= 2; + *tail += 2; + break; + } - mutex_unlock(&ldata->output_lock); - return i; + return space; } /** @@ -614,9 +676,9 @@ break_out: static size_t __process_echoes(struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; - int space, old_space; + unsigned int space, old_space; size_t tail; - unsigned char c; + u8 c; old_space = space = tty_write_room(tty); @@ -624,104 +686,12 @@ static size_t __process_echoes(struct tty_struct *tty) while (MASK(ldata->echo_commit) != MASK(tail)) { c = echo_buf(ldata, tail); if (c == ECHO_OP_START) { - unsigned char op; - int no_space_left = 0; - - /* - * Since add_echo_byte() is called without holding - * output_lock, we might see only portion of multi-byte - * operation. - */ - if (MASK(ldata->echo_commit) == MASK(tail + 1)) + int ret = n_tty_process_echo_ops(tty, &tail, space); + if (ret == -ENODATA) goto not_yet_stored; - /* - * If the buffer byte is the start of a multi-byte - * operation, get the next byte, which is either the - * op code or a control character value. - */ - op = echo_buf(ldata, tail + 1); - - switch (op) { - case ECHO_OP_ERASE_TAB: { - unsigned int num_chars, num_bs; - - if (MASK(ldata->echo_commit) == MASK(tail + 2)) - goto not_yet_stored; - num_chars = echo_buf(ldata, tail + 2); - - /* - * Determine how many columns to go back - * in order to erase the tab. - * This depends on the number of columns - * used by other characters within the tab - * area. If this (modulo 8) count is from - * the start of input rather than from a - * previous tab, we offset by canon column. - * Otherwise, tab spacing is normal. - */ - if (!(num_chars & 0x80)) - num_chars += ldata->canon_column; - num_bs = 8 - (num_chars & 7); - - if (num_bs > space) { - no_space_left = 1; - break; - } - space -= num_bs; - while (num_bs--) { - tty_put_char(tty, '\b'); - if (ldata->column > 0) - ldata->column--; - } - tail += 3; - break; - } - case ECHO_OP_SET_CANON_COL: - ldata->canon_column = ldata->column; - tail += 2; - break; - - case ECHO_OP_MOVE_BACK_COL: - if (ldata->column > 0) - ldata->column--; - tail += 2; - break; - - case ECHO_OP_START: - /* This is an escaped echo op start code */ - if (!space) { - no_space_left = 1; - break; - } - tty_put_char(tty, ECHO_OP_START); - ldata->column++; - space--; - tail += 2; - break; - - default: - /* - * If the op is not a special byte code, - * it is a ctrl char tagged to be echoed - * as "^X" (where X is the letter - * representing the control char). - * Note that we must ensure there is - * enough space for the whole ctrl pair. - * - */ - if (space < 2) { - no_space_left = 1; - break; - } - tty_put_char(tty, '^'); - tty_put_char(tty, op ^ 0100); - ldata->column += 2; - space -= 2; - tail += 2; - } - - if (no_space_left) + if (ret < 0) break; + space = ret; } else { if (O_OPOST(tty)) { int retval = do_output_char(c, tty, space); @@ -763,24 +733,22 @@ static void commit_echoes(struct tty_struct *tty) size_t nr, old, echoed; size_t head; - mutex_lock(&ldata->output_lock); - head = ldata->echo_head; - ldata->echo_mark = head; - old = ldata->echo_commit - ldata->echo_tail; - - /* Process committed echoes if the accumulated # of bytes - * is over the threshold (and try again each time another - * block is accumulated) */ - nr = head - ldata->echo_tail; - if (nr < ECHO_COMMIT_WATERMARK || - (nr % ECHO_BLOCK > old % ECHO_BLOCK)) { - mutex_unlock(&ldata->output_lock); - return; - } + scoped_guard(mutex, &ldata->output_lock) { + head = ldata->echo_head; + ldata->echo_mark = head; + old = ldata->echo_commit - ldata->echo_tail; - ldata->echo_commit = head; - echoed = __process_echoes(tty); - mutex_unlock(&ldata->output_lock); + /* + * Process committed echoes if the accumulated # of bytes is over the threshold + * (and try again each time another block is accumulated) + */ + nr = head - ldata->echo_tail; + if (nr < ECHO_COMMIT_WATERMARK || (nr % ECHO_BLOCK > old % ECHO_BLOCK)) + return; + + ldata->echo_commit = head; + echoed = __process_echoes(tty); + } if (echoed && tty->ops->flush_chars) tty->ops->flush_chars(tty); @@ -794,10 +762,10 @@ static void process_echoes(struct tty_struct *tty) if (ldata->echo_mark == ldata->echo_tail) return; - mutex_lock(&ldata->output_lock); - ldata->echo_commit = ldata->echo_mark; - echoed = __process_echoes(tty); - mutex_unlock(&ldata->output_lock); + scoped_guard(mutex, &ldata->output_lock) { + ldata->echo_commit = ldata->echo_mark; + echoed = __process_echoes(tty); + } if (echoed && tty->ops->flush_chars) tty->ops->flush_chars(tty); @@ -812,10 +780,9 @@ static void flush_echoes(struct tty_struct *tty) ldata->echo_commit == ldata->echo_head) return; - mutex_lock(&ldata->output_lock); + guard(mutex)(&ldata->output_lock); ldata->echo_commit = ldata->echo_head; __process_echoes(tty); - mutex_unlock(&ldata->output_lock); } /** @@ -825,7 +792,7 @@ static void flush_echoes(struct tty_struct *tty) * * Add a character or operation byte to the echo buffer. */ -static inline void add_echo_byte(unsigned char c, struct n_tty_data *ldata) +static inline void add_echo_byte(u8 c, struct n_tty_data *ldata) { *echo_buf_addr(ldata, ldata->echo_head) = c; smp_wmb(); /* Matches smp_rmb() in echo_buf(). */ @@ -896,7 +863,7 @@ static void echo_erase_tab(unsigned int num_chars, int after_tab, * * This variant does not treat control characters specially. */ -static void echo_char_raw(unsigned char c, struct n_tty_data *ldata) +static void echo_char_raw(u8 c, struct n_tty_data *ldata) { if (c == ECHO_OP_START) { add_echo_byte(ECHO_OP_START, ldata); @@ -917,7 +884,7 @@ static void echo_char_raw(unsigned char c, struct n_tty_data *ldata) * This variant tags control characters to be echoed as "^X" (where X is the * letter representing the control char). */ -static void echo_char(unsigned char c, struct tty_struct *tty) +static void echo_char(u8 c, const struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; @@ -955,7 +922,7 @@ static inline void finish_erasing(struct n_tty_data *ldata) * Locking: n_tty_receive_buf()/producer path: * caller holds non-exclusive %termios_rwsem */ -static void eraser(unsigned char c, struct tty_struct *tty) +static void eraser(u8 c, const struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; enum { ERASE, WERASE, KILL } kill_type; @@ -1104,18 +1071,19 @@ static void isig(int sig, struct tty_struct *tty) if (L_NOFLSH(tty)) { /* signal only */ __isig(sig, tty); + return; + } - } else { /* signal and flush */ - up_read(&tty->termios_rwsem); - down_write(&tty->termios_rwsem); - + /* signal and flush */ + up_read(&tty->termios_rwsem); + scoped_guard(rwsem_write, &tty->termios_rwsem) { __isig(sig, tty); /* clear echo buffer */ - mutex_lock(&ldata->output_lock); - ldata->echo_head = ldata->echo_tail = 0; - ldata->echo_mark = ldata->echo_commit = 0; - mutex_unlock(&ldata->output_lock); + scoped_guard(mutex, &ldata->output_lock) { + ldata->echo_head = ldata->echo_tail = 0; + ldata->echo_mark = ldata->echo_commit = 0; + } /* clear output buffer */ tty_driver_flush_buffer(tty); @@ -1126,10 +1094,8 @@ static void isig(int sig, struct tty_struct *tty) /* notify pty master of flush */ if (tty->link) n_tty_packet_mode_flush(tty); - - up_write(&tty->termios_rwsem); - down_read(&tty->termios_rwsem); } + down_read(&tty->termios_rwsem); } /** @@ -1171,14 +1137,13 @@ static void n_tty_receive_break(struct tty_struct *tty) * Called from the receive_buf path so single threaded. Does not need locking * as num_overrun and overrun_time are function private. */ -static void n_tty_receive_overrun(struct tty_struct *tty) +static void n_tty_receive_overrun(const struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; ldata->num_overrun++; - if (time_after(jiffies, ldata->overrun_time + HZ) || - time_after(ldata->overrun_time, jiffies)) { - tty_warn(tty, "%d input overrun(s)\n", ldata->num_overrun); + if (time_is_before_jiffies(ldata->overrun_time + HZ)) { + tty_warn(tty, "%u input overrun(s)\n", ldata->num_overrun); ldata->overrun_time = jiffies; ldata->num_overrun = 0; } @@ -1195,7 +1160,8 @@ static void n_tty_receive_overrun(struct tty_struct *tty) * Locking: n_tty_receive_buf()/producer path: * caller holds non-exclusive %termios_rwsem */ -static void n_tty_receive_parity_error(struct tty_struct *tty, unsigned char c) +static void n_tty_receive_parity_error(const struct tty_struct *tty, + u8 c) { struct n_tty_data *ldata = tty->disc_data; @@ -1213,7 +1179,7 @@ static void n_tty_receive_parity_error(struct tty_struct *tty, unsigned char c) } static void -n_tty_receive_signal_char(struct tty_struct *tty, int signal, unsigned char c) +n_tty_receive_signal_char(struct tty_struct *tty, int signal, u8 c) { isig(signal, tty); if (I_IXON(tty)) @@ -1225,7 +1191,7 @@ n_tty_receive_signal_char(struct tty_struct *tty, int signal, unsigned char c) process_echoes(tty); } -static bool n_tty_is_char_flow_ctrl(struct tty_struct *tty, unsigned char c) +static bool n_tty_is_char_flow_ctrl(struct tty_struct *tty, u8 c) { return c == START_CHAR(tty) || c == STOP_CHAR(tty); } @@ -1245,7 +1211,7 @@ static bool n_tty_is_char_flow_ctrl(struct tty_struct *tty, unsigned char c) * Returns true if @c is consumed as flow-control character, the character * must not be treated as normal character. */ -static bool n_tty_receive_char_flow_ctrl(struct tty_struct *tty, unsigned char c, +static bool n_tty_receive_char_flow_ctrl(struct tty_struct *tty, u8 c, bool lookahead_done) { if (!n_tty_is_char_flow_ctrl(tty, c)) @@ -1265,7 +1231,103 @@ static bool n_tty_receive_char_flow_ctrl(struct tty_struct *tty, unsigned char c return true; } -static void n_tty_receive_char_special(struct tty_struct *tty, unsigned char c, +static void n_tty_receive_handle_newline(struct tty_struct *tty, u8 c) +{ + struct n_tty_data *ldata = tty->disc_data; + + set_bit(MASK(ldata->read_head), ldata->read_flags); + put_tty_queue(c, ldata); + smp_store_release(&ldata->canon_head, ldata->read_head); + kill_fasync(&tty->fasync, SIGIO, POLL_IN); + wake_up_interruptible_poll(&tty->read_wait, EPOLLIN | EPOLLRDNORM); +} + +static bool n_tty_receive_char_canon(struct tty_struct *tty, u8 c) +{ + struct n_tty_data *ldata = tty->disc_data; + + if (c == ERASE_CHAR(tty) || c == KILL_CHAR(tty) || + (c == WERASE_CHAR(tty) && L_IEXTEN(tty))) { + eraser(c, tty); + commit_echoes(tty); + + return true; + } + + if (c == LNEXT_CHAR(tty) && L_IEXTEN(tty)) { + ldata->lnext = 1; + if (L_ECHO(tty)) { + finish_erasing(ldata); + if (L_ECHOCTL(tty)) { + echo_char_raw('^', ldata); + echo_char_raw('\b', ldata); + commit_echoes(tty); + } + } + + return true; + } + + if (c == REPRINT_CHAR(tty) && L_ECHO(tty) && L_IEXTEN(tty)) { + size_t tail = ldata->canon_head; + + finish_erasing(ldata); + echo_char(c, tty); + echo_char_raw('\n', ldata); + while (MASK(tail) != MASK(ldata->read_head)) { + echo_char(read_buf(ldata, tail), tty); + tail++; + } + commit_echoes(tty); + + return true; + } + + if (c == '\n') { + if (L_ECHO(tty) || L_ECHONL(tty)) { + echo_char_raw('\n', ldata); + commit_echoes(tty); + } + n_tty_receive_handle_newline(tty, c); + + return true; + } + + if (c == EOF_CHAR(tty)) { + c = __DISABLED_CHAR; + n_tty_receive_handle_newline(tty, c); + + return true; + } + + if ((c == EOL_CHAR(tty)) || + (c == EOL2_CHAR(tty) && L_IEXTEN(tty))) { + /* + * XXX are EOL_CHAR and EOL2_CHAR echoed?!? + */ + if (L_ECHO(tty)) { + /* Record the column of first canon char. */ + if (ldata->canon_head == ldata->read_head) + echo_set_canon_col(ldata); + echo_char(c, tty); + commit_echoes(tty); + } + /* + * XXX does PARMRK doubling happen for + * EOL_CHAR and EOL2_CHAR? + */ + if (c == '\377' && I_PARMRK(tty)) + put_tty_queue(c, ldata); + + n_tty_receive_handle_newline(tty, c); + + return true; + } + + return false; +} + +static void n_tty_receive_char_special(struct tty_struct *tty, u8 c, bool lookahead_done) { struct n_tty_data *ldata = tty->disc_data; @@ -1299,77 +1361,8 @@ static void n_tty_receive_char_special(struct tty_struct *tty, unsigned char c, } else if (c == '\n' && I_INLCR(tty)) c = '\r'; - if (ldata->icanon) { - if (c == ERASE_CHAR(tty) || c == KILL_CHAR(tty) || - (c == WERASE_CHAR(tty) && L_IEXTEN(tty))) { - eraser(c, tty); - commit_echoes(tty); - return; - } - if (c == LNEXT_CHAR(tty) && L_IEXTEN(tty)) { - ldata->lnext = 1; - if (L_ECHO(tty)) { - finish_erasing(ldata); - if (L_ECHOCTL(tty)) { - echo_char_raw('^', ldata); - echo_char_raw('\b', ldata); - commit_echoes(tty); - } - } - return; - } - if (c == REPRINT_CHAR(tty) && L_ECHO(tty) && L_IEXTEN(tty)) { - size_t tail = ldata->canon_head; - - finish_erasing(ldata); - echo_char(c, tty); - echo_char_raw('\n', ldata); - while (MASK(tail) != MASK(ldata->read_head)) { - echo_char(read_buf(ldata, tail), tty); - tail++; - } - commit_echoes(tty); - return; - } - if (c == '\n') { - if (L_ECHO(tty) || L_ECHONL(tty)) { - echo_char_raw('\n', ldata); - commit_echoes(tty); - } - goto handle_newline; - } - if (c == EOF_CHAR(tty)) { - c = __DISABLED_CHAR; - goto handle_newline; - } - if ((c == EOL_CHAR(tty)) || - (c == EOL2_CHAR(tty) && L_IEXTEN(tty))) { - /* - * XXX are EOL_CHAR and EOL2_CHAR echoed?!? - */ - if (L_ECHO(tty)) { - /* Record the column of first canon char. */ - if (ldata->canon_head == ldata->read_head) - echo_set_canon_col(ldata); - echo_char(c, tty); - commit_echoes(tty); - } - /* - * XXX does PARMRK doubling happen for - * EOL_CHAR and EOL2_CHAR? - */ - if (c == (unsigned char) '\377' && I_PARMRK(tty)) - put_tty_queue(c, ldata); - -handle_newline: - set_bit(ldata->read_head & (N_TTY_BUF_SIZE - 1), ldata->read_flags); - put_tty_queue(c, ldata); - smp_store_release(&ldata->canon_head, ldata->read_head); - kill_fasync(&tty->fasync, SIGIO, POLL_IN); - wake_up_interruptible_poll(&tty->read_wait, EPOLLIN | EPOLLRDNORM); - return; - } - } + if (ldata->icanon && n_tty_receive_char_canon(tty, c)) + return; if (L_ECHO(tty)) { finish_erasing(ldata); @@ -1385,7 +1378,7 @@ handle_newline: } /* PARMRK doubling check */ - if (c == (unsigned char) '\377' && I_PARMRK(tty)) + if (c == '\377' && I_PARMRK(tty)) put_tty_queue(c, ldata); put_tty_queue(c, ldata); @@ -1403,7 +1396,7 @@ handle_newline: * caller holds non-exclusive %termios_rwsem * publishes canon_head if canonical mode is active */ -static void n_tty_receive_char(struct tty_struct *tty, unsigned char c) +static void n_tty_receive_char(struct tty_struct *tty, u8 c) { struct n_tty_data *ldata = tty->disc_data; @@ -1420,12 +1413,12 @@ static void n_tty_receive_char(struct tty_struct *tty, unsigned char c) commit_echoes(tty); } /* PARMRK doubling check */ - if (c == (unsigned char) '\377' && I_PARMRK(tty)) + if (c == '\377' && I_PARMRK(tty)) put_tty_queue(c, ldata); put_tty_queue(c, ldata); } -static void n_tty_receive_char_closing(struct tty_struct *tty, unsigned char c, +static void n_tty_receive_char_closing(struct tty_struct *tty, u8 c, bool lookahead_done) { if (I_ISTRIP(tty)) @@ -1445,7 +1438,7 @@ static void n_tty_receive_char_closing(struct tty_struct *tty, unsigned char c, } static void -n_tty_receive_char_flagged(struct tty_struct *tty, unsigned char c, char flag) +n_tty_receive_char_flagged(struct tty_struct *tty, u8 c, u8 flag) { switch (flag) { case TTY_BREAK: @@ -1459,13 +1452,13 @@ n_tty_receive_char_flagged(struct tty_struct *tty, unsigned char c, char flag) n_tty_receive_overrun(tty); break; default: - tty_err(tty, "unknown flag %d\n", flag); + tty_err(tty, "unknown flag %u\n", flag); break; } } static void -n_tty_receive_char_lnext(struct tty_struct *tty, unsigned char c, char flag) +n_tty_receive_char_lnext(struct tty_struct *tty, u8 c, u8 flag) { struct n_tty_data *ldata = tty->disc_data; @@ -1481,11 +1474,11 @@ n_tty_receive_char_lnext(struct tty_struct *tty, unsigned char c, char flag) } /* Caller must ensure count > 0 */ -static void n_tty_lookahead_flow_ctrl(struct tty_struct *tty, const unsigned char *cp, - const unsigned char *fp, unsigned int count) +static void n_tty_lookahead_flow_ctrl(struct tty_struct *tty, const u8 *cp, + const u8 *fp, size_t count) { struct n_tty_data *ldata = tty->disc_data; - unsigned char flag = TTY_NORMAL; + u8 flag = TTY_NORMAL; ldata->lookahead_count += count; @@ -1502,31 +1495,30 @@ static void n_tty_lookahead_flow_ctrl(struct tty_struct *tty, const unsigned cha } static void -n_tty_receive_buf_real_raw(struct tty_struct *tty, const unsigned char *cp, - const char *fp, int count) +n_tty_receive_buf_real_raw(const struct tty_struct *tty, const u8 *cp, + size_t count) { struct n_tty_data *ldata = tty->disc_data; - size_t n, head; - head = ldata->read_head & (N_TTY_BUF_SIZE - 1); - n = min_t(size_t, count, N_TTY_BUF_SIZE - head); - memcpy(read_buf_addr(ldata, head), cp, n); - ldata->read_head += n; - cp += n; - count -= n; + /* handle buffer wrap-around by a loop */ + for (unsigned int i = 0; i < 2; i++) { + size_t head = MASK(ldata->read_head); + size_t n = min(count, N_TTY_BUF_SIZE - head); - head = ldata->read_head & (N_TTY_BUF_SIZE - 1); - n = min_t(size_t, count, N_TTY_BUF_SIZE - head); - memcpy(read_buf_addr(ldata, head), cp, n); - ldata->read_head += n; + memcpy(read_buf_addr(ldata, head), cp, n); + + ldata->read_head += n; + cp += n; + count -= n; + } } static void -n_tty_receive_buf_raw(struct tty_struct *tty, const unsigned char *cp, - const char *fp, int count) +n_tty_receive_buf_raw(struct tty_struct *tty, const u8 *cp, const u8 *fp, + size_t count) { struct n_tty_data *ldata = tty->disc_data; - char flag = TTY_NORMAL; + u8 flag = TTY_NORMAL; while (count--) { if (fp) @@ -1539,10 +1531,10 @@ n_tty_receive_buf_raw(struct tty_struct *tty, const unsigned char *cp, } static void -n_tty_receive_buf_closing(struct tty_struct *tty, const unsigned char *cp, - const char *fp, int count, bool lookahead_done) +n_tty_receive_buf_closing(struct tty_struct *tty, const u8 *cp, const u8 *fp, + size_t count, bool lookahead_done) { - char flag = TTY_NORMAL; + u8 flag = TTY_NORMAL; while (count--) { if (fp) @@ -1552,14 +1544,15 @@ n_tty_receive_buf_closing(struct tty_struct *tty, const unsigned char *cp, } } -static void n_tty_receive_buf_standard(struct tty_struct *tty, - const unsigned char *cp, const char *fp, int count, bool lookahead_done) +static void n_tty_receive_buf_standard(struct tty_struct *tty, const u8 *cp, + const u8 *fp, size_t count, + bool lookahead_done) { struct n_tty_data *ldata = tty->disc_data; - char flag = TTY_NORMAL; + u8 flag = TTY_NORMAL; while (count--) { - unsigned char c = *cp++; + u8 c = *cp++; if (fp) flag = *fp++; @@ -1590,27 +1583,37 @@ static void n_tty_receive_buf_standard(struct tty_struct *tty, } } -static void __receive_buf(struct tty_struct *tty, const unsigned char *cp, - const char *fp, int count) +static void __receive_buf(struct tty_struct *tty, const u8 *cp, const u8 *fp, + size_t count) { struct n_tty_data *ldata = tty->disc_data; bool preops = I_ISTRIP(tty) || (I_IUCLC(tty) && L_IEXTEN(tty)); - size_t la_count = min_t(size_t, ldata->lookahead_count, count); + size_t la_count = min(ldata->lookahead_count, count); if (ldata->real_raw) - n_tty_receive_buf_real_raw(tty, cp, fp, count); + n_tty_receive_buf_real_raw(tty, cp, count); else if (ldata->raw || (L_EXTPROC(tty) && !preops)) n_tty_receive_buf_raw(tty, cp, fp, count); else if (tty->closing && !L_EXTPROC(tty)) { - if (la_count > 0) + if (la_count > 0) { n_tty_receive_buf_closing(tty, cp, fp, la_count, true); - if (count > la_count) - n_tty_receive_buf_closing(tty, cp, fp, count - la_count, false); + cp += la_count; + if (fp) + fp += la_count; + count -= la_count; + } + if (count > 0) + n_tty_receive_buf_closing(tty, cp, fp, count, false); } else { - if (la_count > 0) + if (la_count > 0) { n_tty_receive_buf_standard(tty, cp, fp, la_count, true); - if (count > la_count) - n_tty_receive_buf_standard(tty, cp, fp, count - la_count, false); + cp += la_count; + if (fp) + fp += la_count; + count -= la_count; + } + if (count > 0) + n_tty_receive_buf_standard(tty, cp, fp, count, false); flush_echoes(tty); if (tty->ops->flush_chars) @@ -1664,14 +1667,15 @@ static void __receive_buf(struct tty_struct *tty, const unsigned char *cp, * claims non-exclusive %termios_rwsem * publishes commit_head or canon_head */ -static int -n_tty_receive_buf_common(struct tty_struct *tty, const unsigned char *cp, - const char *fp, int count, int flow) +static size_t +n_tty_receive_buf_common(struct tty_struct *tty, const u8 *cp, const u8 *fp, + size_t count, bool flow) { struct n_tty_data *ldata = tty->disc_data; - int room, n, rcvd = 0, overflow; + size_t n, rcvd = 0; + int room, overflow; - down_read(&tty->termios_rwsem); + guard(rwsem_read)(&tty->termios_rwsem); do { /* @@ -1691,18 +1695,18 @@ n_tty_receive_buf_common(struct tty_struct *tty, const unsigned char *cp, room = N_TTY_BUF_SIZE - (ldata->read_head - tail); if (I_PARMRK(tty)) - room = (room + 2) / 3; + room = DIV_ROUND_UP(room, 3); room--; if (room <= 0) { overflow = ldata->icanon && ldata->canon_head == tail; if (overflow && room < 0) ldata->read_head--; room = overflow; - ldata->no_room = flow && !room; + WRITE_ONCE(ldata->no_room, flow && !room); } else overflow = 0; - n = min(count, room); + n = min_t(size_t, count, room); if (!n) break; @@ -1729,21 +1733,30 @@ n_tty_receive_buf_common(struct tty_struct *tty, const unsigned char *cp, } else n_tty_check_throttle(tty); - up_read(&tty->termios_rwsem); + if (unlikely(ldata->no_room)) { + /* + * Barrier here is to ensure to read the latest read_tail in + * chars_in_buffer() and to make sure that read_tail is not loaded + * before ldata->no_room is set. + */ + smp_mb(); + if (!chars_in_buffer(tty)) + n_tty_kick_worker(tty); + } return rcvd; } -static void n_tty_receive_buf(struct tty_struct *tty, const unsigned char *cp, - const char *fp, int count) +static void n_tty_receive_buf(struct tty_struct *tty, const u8 *cp, + const u8 *fp, size_t count) { - n_tty_receive_buf_common(tty, cp, fp, count, 0); + n_tty_receive_buf_common(tty, cp, fp, count, false); } -static int n_tty_receive_buf2(struct tty_struct *tty, const unsigned char *cp, - const char *fp, int count) +static size_t n_tty_receive_buf2(struct tty_struct *tty, const u8 *cp, + const u8 *fp, size_t count) { - return n_tty_receive_buf_common(tty, cp, fp, count, 1); + return n_tty_receive_buf_common(tty, cp, fp, count, true); } /** @@ -1769,8 +1782,7 @@ static void n_tty_set_termios(struct tty_struct *tty, const struct ktermios *old ldata->canon_head = ldata->read_tail; ldata->push = 0; } else { - set_bit((ldata->read_head - 1) & (N_TTY_BUF_SIZE - 1), - ldata->read_flags); + set_bit(MASK(ldata->read_head - 1), ldata->read_flags); ldata->canon_head = ldata->read_head; ldata->push = 1; } @@ -1857,10 +1869,9 @@ static void n_tty_close(struct tty_struct *tty) if (tty->link) n_tty_packet_mode_flush(tty); - down_write(&tty->termios_rwsem); + guard(rwsem_write)(&tty->termios_rwsem); vfree(ldata); tty->disc_data = NULL; - up_write(&tty->termios_rwsem); } /** @@ -1893,9 +1904,9 @@ static int n_tty_open(struct tty_struct *tty) return 0; } -static inline int input_available_p(struct tty_struct *tty, int poll) +static inline int input_available_p(const struct tty_struct *tty, int poll) { - struct n_tty_data *ldata = tty->disc_data; + const struct n_tty_data *ldata = tty->disc_data; int amt = poll && !TIME_CHAR(tty) && MIN_CHAR(tty) ? MIN_CHAR(tty) : 1; if (ldata->icanon && !L_EXTPROC(tty)) @@ -1922,37 +1933,37 @@ static inline int input_available_p(struct tty_struct *tty, int poll) * caller holds non-exclusive %termios_rwsem; * read_tail published */ -static bool copy_from_read_buf(struct tty_struct *tty, - unsigned char **kbp, - size_t *nr) +static bool copy_from_read_buf(const struct tty_struct *tty, u8 **kbp, + size_t *nr) { struct n_tty_data *ldata = tty->disc_data; size_t n; bool is_eof; size_t head = smp_load_acquire(&ldata->commit_head); - size_t tail = ldata->read_tail & (N_TTY_BUF_SIZE - 1); - - n = min(head - ldata->read_tail, N_TTY_BUF_SIZE - tail); - n = min(*nr, n); - if (n) { - unsigned char *from = read_buf_addr(ldata, tail); - memcpy(*kbp, from, n); - is_eof = n == 1 && *from == EOF_CHAR(tty); - tty_audit_add_data(tty, from, n); - zero_buffer(tty, from, n); - smp_store_release(&ldata->read_tail, ldata->read_tail + n); - /* Turn single EOF into zero-length read */ - if (L_EXTPROC(tty) && ldata->icanon && is_eof && - (head == ldata->read_tail)) - return false; - *kbp += n; - *nr -= n; - - /* If we have more to copy, let the caller know */ - return head != ldata->read_tail; - } - return false; + size_t tail = MASK(ldata->read_tail); + + n = min3(head - ldata->read_tail, N_TTY_BUF_SIZE - tail, *nr); + if (!n) + return false; + + u8 *from = read_buf_addr(ldata, tail); + memcpy(*kbp, from, n); + is_eof = n == 1 && *from == EOF_CHAR(tty); + tty_audit_add_data(tty, from, n); + zero_buffer(tty, from, n); + smp_store_release(&ldata->read_tail, ldata->read_tail + n); + + /* Turn single EOF into zero-length read */ + if (L_EXTPROC(tty) && ldata->icanon && is_eof && + head == ldata->read_tail) + return false; + + *kbp += n; + *nr -= n; + + /* If we have more to copy, let the caller know */ + return head != ldata->read_tail; } /** @@ -1977,8 +1988,7 @@ static bool copy_from_read_buf(struct tty_struct *tty, * caller holds non-exclusive %termios_rwsem; * read_tail published */ -static bool canon_copy_from_read_buf(struct tty_struct *tty, - unsigned char **kbp, +static bool canon_copy_from_read_buf(const struct tty_struct *tty, u8 **kbp, size_t *nr) { struct n_tty_data *ldata = tty->disc_data; @@ -1994,12 +2004,9 @@ static bool canon_copy_from_read_buf(struct tty_struct *tty, canon_head = smp_load_acquire(&ldata->canon_head); n = min(*nr, canon_head - ldata->read_tail); - tail = ldata->read_tail & (N_TTY_BUF_SIZE - 1); + tail = MASK(ldata->read_tail); size = min_t(size_t, tail + n, N_TTY_BUF_SIZE); - n_tty_trace("%s: nr:%zu tail:%zu n:%zu size:%zu\n", - __func__, *nr, tail, n, size); - eol = find_next_bit(ldata->read_flags, size, tail); more = n - (size - tail); if (eol == N_TTY_BUF_SIZE && more) { @@ -2017,9 +2024,6 @@ static bool canon_copy_from_read_buf(struct tty_struct *tty, if (!found || read_buf(ldata, eol) != __DISABLED_CHAR) n = c; - n_tty_trace("%s: eol:%zu found:%d n:%zu c:%zu tail:%zu more:%zu\n", - __func__, eol, found, n, c, tail, more); - tty_copy(tty, *kbp, tail, n); *kbp += n; *nr -= n; @@ -2046,9 +2050,8 @@ static bool canon_copy_from_read_buf(struct tty_struct *tty, * EOF (special EOL character that's a __DISABLED_CHAR) * in the stream, silently eat the EOF. */ -static void canon_skip_eof(struct tty_struct *tty) +static void canon_skip_eof(struct n_tty_data *ldata) { - struct n_tty_data *ldata = tty->disc_data; size_t tail, canon_head; canon_head = smp_load_acquire(&ldata->canon_head); @@ -2097,6 +2100,66 @@ static int job_control(struct tty_struct *tty, struct file *file) return __tty_check_change(tty, SIGTTIN); } +/* + * We still hold the atomic_read_lock and the termios_rwsem, and can just + * continue to copy data. + */ +static ssize_t n_tty_continue_cookie(struct tty_struct *tty, u8 *kbuf, + size_t nr, void **cookie) +{ + struct n_tty_data *ldata = tty->disc_data; + u8 *kb = kbuf; + + if (ldata->icanon && !L_EXTPROC(tty)) { + /* + * If we have filled the user buffer, see if we should skip an + * EOF character before releasing the lock and returning done. + */ + if (!nr) + canon_skip_eof(ldata); + else if (canon_copy_from_read_buf(tty, &kb, &nr)) + return kb - kbuf; + } else { + if (copy_from_read_buf(tty, &kb, &nr)) + return kb - kbuf; + } + + /* No more data - release locks and stop retries */ + n_tty_kick_worker(tty); + n_tty_check_unthrottle(tty); + up_read(&tty->termios_rwsem); + mutex_unlock(&ldata->atomic_read_lock); + *cookie = NULL; + + return kb - kbuf; +} + +static int n_tty_wait_for_input(struct tty_struct *tty, struct file *file, + struct wait_queue_entry *wait, long *timeout) +{ + if (test_bit(TTY_OTHER_CLOSED, &tty->flags)) + return -EIO; + if (tty_hung_up_p(file)) + return 0; + /* + * Abort readers for ttys which never actually get hung up. + * See __tty_hangup(). + */ + if (test_bit(TTY_HUPPING, &tty->flags)) + return 0; + if (!*timeout) + return 0; + if (tty_io_nonblock(tty, file)) + return -EAGAIN; + if (signal_pending(current)) + return -ERESTARTSYS; + + up_read(&tty->termios_rwsem); + *timeout = wait_woken(wait, TASK_INTERRUPTIBLE, *timeout); + down_read(&tty->termios_rwsem); + + return 1; +} /** * n_tty_read - read function for tty @@ -2118,54 +2181,25 @@ static int job_control(struct tty_struct *tty, struct file *file) * claims non-exclusive termios_rwsem; * publishes read_tail */ -static ssize_t n_tty_read(struct tty_struct *tty, struct file *file, - unsigned char *kbuf, size_t nr, - void **cookie, unsigned long offset) +static ssize_t n_tty_read(struct tty_struct *tty, struct file *file, u8 *kbuf, + size_t nr, void **cookie, unsigned long offset) { struct n_tty_data *ldata = tty->disc_data; - unsigned char *kb = kbuf; + u8 *kb = kbuf; DEFINE_WAIT_FUNC(wait, woken_wake_function); - int c; int minimum, time; - ssize_t retval = 0; + ssize_t retval; long timeout; bool packet; size_t old_tail; - /* - * Is this a continuation of a read started earler? - * - * If so, we still hold the atomic_read_lock and the - * termios_rwsem, and can just continue to copy data. - */ - if (*cookie) { - if (ldata->icanon && !L_EXTPROC(tty)) { - /* - * If we have filled the user buffer, see - * if we should skip an EOF character before - * releasing the lock and returning done. - */ - if (!nr) - canon_skip_eof(tty); - else if (canon_copy_from_read_buf(tty, &kb, &nr)) - return kb - kbuf; - } else { - if (copy_from_read_buf(tty, &kb, &nr)) - return kb - kbuf; - } + /* Is this a continuation of a read started earlier? */ + if (*cookie) + return n_tty_continue_cookie(tty, kbuf, nr, cookie); - /* No more data - release locks and stop retries */ - n_tty_kick_worker(tty); - n_tty_check_unthrottle(tty); - up_read(&tty->termios_rwsem); - mutex_unlock(&ldata->atomic_read_lock); - *cookie = NULL; - return kb - kbuf; - } - - c = job_control(tty, file); - if (c < 0) - return c; + retval = job_control(tty, file); + if (retval < 0) + return retval; /* * Internal serialization of reads. @@ -2199,13 +2233,13 @@ static ssize_t n_tty_read(struct tty_struct *tty, struct file *file, while (nr) { /* First test for status change. */ if (packet && tty->link->ctrl.pktstatus) { - unsigned char cs; + u8 cs; if (kb != kbuf) break; - spin_lock_irq(&tty->link->ctrl.lock); - cs = tty->link->ctrl.pktstatus; - tty->link->ctrl.pktstatus = 0; - spin_unlock_irq(&tty->link->ctrl.lock); + scoped_guard(spinlock_irq, &tty->link->ctrl.lock) { + cs = tty->link->ctrl.pktstatus; + tty->link->ctrl.pktstatus = 0; + } *kb++ = cs; nr--; break; @@ -2216,34 +2250,12 @@ static ssize_t n_tty_read(struct tty_struct *tty, struct file *file, tty_buffer_flush_work(tty->port); down_read(&tty->termios_rwsem); if (!input_available_p(tty, 0)) { - if (test_bit(TTY_OTHER_CLOSED, &tty->flags)) { - retval = -EIO; + int ret = n_tty_wait_for_input(tty, file, &wait, + &timeout); + if (ret <= 0) { + retval = ret; break; } - if (tty_hung_up_p(file)) - break; - /* - * Abort readers for ttys which never actually - * get hung up. See __tty_hangup(). - */ - if (test_bit(TTY_HUPPING, &tty->flags)) - break; - if (!timeout) - break; - if (tty_io_nonblock(tty, file)) { - retval = -EAGAIN; - break; - } - if (signal_pending(current)) { - retval = -ERESTARTSYS; - break; - } - up_read(&tty->termios_rwsem); - - timeout = wait_woken(&wait, TASK_INTERRUPTIBLE, - timeout); - - down_read(&tty->termios_rwsem); continue; } } @@ -2258,21 +2270,8 @@ static ssize_t n_tty_read(struct tty_struct *tty, struct file *file, nr--; } - /* - * Copy data, and if there is more to be had - * and we have nothing more to wait for, then - * let's mark us for retries. - * - * NOTE! We return here with both the termios_sem - * and atomic_read_lock still held, the retries - * will release them when done. - */ - if (copy_from_read_buf(tty, &kb, &nr) && kb - kbuf >= minimum) { -more_to_be_read: - remove_wait_queue(&tty->read_wait, &wait); - *cookie = cookie; - return kb - kbuf; - } + if (copy_from_read_buf(tty, &kb, &nr) && kb - kbuf >= minimum) + goto more_to_be_read; } n_tty_check_unthrottle(tty); @@ -2282,8 +2281,14 @@ more_to_be_read: if (time) timeout = time; } - if (old_tail != ldata->read_tail) + if (old_tail != ldata->read_tail) { + /* + * Make sure no_room is not read in n_tty_kick_worker() + * before setting ldata->read_tail in copy_from_read_buf(). + */ + smp_mb(); n_tty_kick_worker(tty); + } up_read(&tty->termios_rwsem); remove_wait_queue(&tty->read_wait, &wait); @@ -2293,6 +2298,18 @@ more_to_be_read: retval = kb - kbuf; return retval; +more_to_be_read: + /* + * There is more to be had and we have nothing more to wait for, so + * let's mark us for retries. + * + * NOTE! We return here with both the termios_sem and atomic_read_lock + * still held, the retries will release them when done. + */ + remove_wait_queue(&tty->read_wait, &wait); + *cookie = cookie; + + return kb - kbuf; } /** @@ -2316,12 +2333,11 @@ more_to_be_read: */ static ssize_t n_tty_write(struct tty_struct *tty, struct file *file, - const unsigned char *buf, size_t nr) + const u8 *buf, size_t nr) { - const unsigned char *b = buf; + const u8 *b = buf; DEFINE_WAIT_FUNC(wait, woken_wake_function); - int c; - ssize_t retval = 0; + ssize_t num, retval = 0; /* Job control check -- must be done at start (POSIX.1 7.1.1.4). */ if (L_TOSTOP(tty) && file->f_op->write_iter != redirected_tty_write) { @@ -2330,7 +2346,7 @@ static ssize_t n_tty_write(struct tty_struct *tty, struct file *file, return retval; } - down_read(&tty->termios_rwsem); + guard(rwsem_read)(&tty->termios_rwsem); /* Write out any echoed characters that are still pending */ process_echoes(tty); @@ -2347,7 +2363,7 @@ static ssize_t n_tty_write(struct tty_struct *tty, struct file *file, } if (O_OPOST(tty)) { while (nr > 0) { - ssize_t num = process_output_block(tty, b, nr); + num = process_output_block(tty, b, nr); if (num < 0) { if (num == -EAGAIN) break; @@ -2358,8 +2374,7 @@ static ssize_t n_tty_write(struct tty_struct *tty, struct file *file, nr -= num; if (nr == 0) break; - c = *b; - if (process_output(c, tty) < 0) + if (process_output(*b, tty) < 0) break; b++; nr--; } @@ -2369,17 +2384,16 @@ static ssize_t n_tty_write(struct tty_struct *tty, struct file *file, struct n_tty_data *ldata = tty->disc_data; while (nr > 0) { - mutex_lock(&ldata->output_lock); - c = tty->ops->write(tty, b, nr); - mutex_unlock(&ldata->output_lock); - if (c < 0) { - retval = c; + scoped_guard(mutex, &ldata->output_lock) + num = tty->ops->write(tty, b, nr); + if (num < 0) { + retval = num; goto break_out; } - if (!c) + if (!num) break; - b += c; - nr -= c; + b += num; + nr -= num; } } if (!nr) @@ -2398,7 +2412,7 @@ break_out: remove_wait_queue(&tty->write_wait, &wait); if (nr && tty->fasync) set_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); - up_read(&tty->termios_rwsem); + return (b - buf) ? b - buf : retval; } @@ -2454,7 +2468,7 @@ static unsigned long inq_canon(struct n_tty_data *ldata) nr = head - tail; /* Skip EOF-chars.. */ while (MASK(head) != MASK(tail)) { - if (test_bit(tail & (N_TTY_BUF_SIZE - 1), ldata->read_flags) && + if (test_bit(MASK(tail), ldata->read_flags) && read_buf(ldata, tail) == __DISABLED_CHAR) nr--; tail++; @@ -2466,19 +2480,18 @@ static int n_tty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { struct n_tty_data *ldata = tty->disc_data; - int retval; + unsigned int num; switch (cmd) { case TIOCOUTQ: return put_user(tty_chars_in_buffer(tty), (int __user *) arg); case TIOCINQ: - down_write(&tty->termios_rwsem); - if (L_ICANON(tty) && !L_EXTPROC(tty)) - retval = inq_canon(ldata); - else - retval = read_cnt(ldata); - up_write(&tty->termios_rwsem); - return put_user(retval, (unsigned int __user *) arg); + scoped_guard(rwsem_write, &tty->termios_rwsem) + if (L_ICANON(tty) && !L_EXTPROC(tty)) + num = inq_canon(ldata); + else + num = read_cnt(ldata); + return put_user(num, (unsigned int __user *) arg); default: return n_tty_ioctl_helper(tty, cmd, arg); } |