diff options
| author | Jiri Kosina <jkosina@suse.cz> | 2021-04-29 21:45:19 +0200 |
|---|---|---|
| committer | Jiri Kosina <jkosina@suse.cz> | 2021-04-29 21:45:19 +0200 |
| commit | e50fedec822efc7b7090f95862b782d91ca8aec0 (patch) | |
| tree | 22dd6b534d00a1f7fc3823f0d88e0d43547fc87b /drivers | |
| parent | cfc9bdfb6ba76de84a9ed8ee75dc56903b505a78 (diff) | |
| parent | 35a927f2848bd79586c6374ebb99e4207f3b0c7f (diff) | |
Merge branch 'for-5.13/surface-system-aggregator-intergration' into for-linus
- Surface Aggregator Module support from Maximilian Luz
Diffstat (limited to 'drivers')
696 files changed, 37466 insertions, 15879 deletions
diff --git a/drivers/Kconfig b/drivers/Kconfig index dcecc9f6e33f..62c753a73651 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -6,6 +6,7 @@ menu "Device Drivers" source "drivers/amba/Kconfig" source "drivers/eisa/Kconfig" source "drivers/pci/Kconfig" +source "drivers/cxl/Kconfig" source "drivers/pcmcia/Kconfig" source "drivers/rapidio/Kconfig" diff --git a/drivers/Makefile b/drivers/Makefile index fd11b9ac4cc3..6fba7daba591 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -27,7 +27,7 @@ obj-y += idle/ obj-y += char/ipmi/ obj-$(CONFIG_ACPI) += acpi/ -obj-$(CONFIG_SFI) += sfi/ + # PnP must come after ACPI since it will eventually need to check if acpi # was used and do nothing if so obj-$(CONFIG_PNP) += pnp/ @@ -73,6 +73,7 @@ obj-$(CONFIG_NVM) += lightnvm/ obj-y += base/ block/ misc/ mfd/ nfc/ obj-$(CONFIG_LIBNVDIMM) += nvdimm/ obj-$(CONFIG_DAX) += dax/ +obj-$(CONFIG_CXL_BUS) += cxl/ obj-$(CONFIG_DMA_SHARED_BUFFER) += dma-buf/ obj-$(CONFIG_NUBUS) += nubus/ obj-y += macintosh/ diff --git a/drivers/accessibility/speakup/serialio.c b/drivers/accessibility/speakup/serialio.c index 403b01d66367..53580bdc5baa 100644 --- a/drivers/accessibility/speakup/serialio.c +++ b/drivers/accessibility/speakup/serialio.c @@ -27,11 +27,11 @@ static const struct old_serial_port *serstate; static int timeouts; static int spk_serial_out(struct spk_synth *in_synth, const char ch); -static void spk_serial_send_xchar(char ch); -static void spk_serial_tiocmset(unsigned int set, unsigned int clear); -static unsigned char spk_serial_in(void); -static unsigned char spk_serial_in_nowait(void); -static void spk_serial_flush_buffer(void); +static void spk_serial_send_xchar(struct spk_synth *in_synth, char ch); +static void spk_serial_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear); +static unsigned char spk_serial_in(struct spk_synth *in_synth); +static unsigned char spk_serial_in_nowait(struct spk_synth *in_synth); +static void spk_serial_flush_buffer(struct spk_synth *in_synth); static int spk_serial_wait_for_xmitr(struct spk_synth *in_synth); struct spk_io_ops spk_serial_io_ops = { @@ -150,7 +150,7 @@ static void start_serial_interrupt(int irq) outb(1, speakup_info.port_tts + UART_FCR); /* Turn FIFO On */ } -static void spk_serial_send_xchar(char ch) +static void spk_serial_send_xchar(struct spk_synth *synth, char ch) { int timeout = SPK_XMITR_TIMEOUT; @@ -162,7 +162,7 @@ static void spk_serial_send_xchar(char ch) outb(ch, speakup_info.port_tts); } -static void spk_serial_tiocmset(unsigned int set, unsigned int clear) +static void spk_serial_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear) { int old = inb(speakup_info.port_tts + UART_MCR); @@ -251,7 +251,7 @@ static int spk_serial_wait_for_xmitr(struct spk_synth *in_synth) return 1; } -static unsigned char spk_serial_in(void) +static unsigned char spk_serial_in(struct spk_synth *in_synth) { int tmout = SPK_SERIAL_TIMEOUT; @@ -265,7 +265,7 @@ static unsigned char spk_serial_in(void) return inb_p(speakup_info.port_tts + UART_RX); } -static unsigned char spk_serial_in_nowait(void) +static unsigned char spk_serial_in_nowait(struct spk_synth *in_synth) { unsigned char lsr; @@ -275,7 +275,7 @@ static unsigned char spk_serial_in_nowait(void) return inb_p(speakup_info.port_tts + UART_RX); } -static void spk_serial_flush_buffer(void) +static void spk_serial_flush_buffer(struct spk_synth *in_synth) { /* TODO: flush the UART 16550 buffer */ } @@ -307,7 +307,7 @@ const char *spk_serial_synth_immediate(struct spk_synth *synth, } EXPORT_SYMBOL_GPL(spk_serial_synth_immediate); -void spk_serial_release(void) +void spk_serial_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts == 0) diff --git a/drivers/accessibility/speakup/speakup_acntpc.c b/drivers/accessibility/speakup/speakup_acntpc.c index c94328a5bd4a..c1ec087dca13 100644 --- a/drivers/accessibility/speakup/speakup_acntpc.c +++ b/drivers/accessibility/speakup/speakup_acntpc.c @@ -25,7 +25,7 @@ #define PROCSPEECH '\r' static int synth_probe(struct spk_synth *synth); -static void accent_release(void); +static void accent_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -294,7 +294,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void accent_release(void) +static void accent_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts) diff --git a/drivers/accessibility/speakup/speakup_apollo.c b/drivers/accessibility/speakup/speakup_apollo.c index 0877b4044c28..cd63581b2e99 100644 --- a/drivers/accessibility/speakup/speakup_apollo.c +++ b/drivers/accessibility/speakup/speakup_apollo.c @@ -163,8 +163,8 @@ static void do_catch_up(struct spk_synth *synth) full_time_val = full_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); if (!synth->io_ops->synth_out(synth, ch)) { - synth->io_ops->tiocmset(0, UART_MCR_RTS); - synth->io_ops->tiocmset(UART_MCR_RTS, 0); + synth->io_ops->tiocmset(synth, 0, UART_MCR_RTS); + synth->io_ops->tiocmset(synth, UART_MCR_RTS, 0); schedule_timeout(msecs_to_jiffies(full_time_val)); continue; } diff --git a/drivers/accessibility/speakup/speakup_audptr.c b/drivers/accessibility/speakup/speakup_audptr.c index e6a6a9665d8f..e89fd72579e6 100644 --- a/drivers/accessibility/speakup/speakup_audptr.c +++ b/drivers/accessibility/speakup/speakup_audptr.c @@ -119,8 +119,8 @@ static struct spk_synth synth_audptr = { static void synth_flush(struct spk_synth *synth) { - synth->io_ops->flush_buffer(); - synth->io_ops->send_xchar(SYNTH_CLEAR); + synth->io_ops->flush_buffer(synth); + synth->io_ops->send_xchar(synth, SYNTH_CLEAR); synth->io_ops->synth_out(synth, PROCSPEECH); } @@ -130,11 +130,11 @@ static void synth_version(struct spk_synth *synth) char synth_id[40] = ""; synth->synth_immediate(synth, "\x05[Q]"); - synth_id[test] = synth->io_ops->synth_in(); + synth_id[test] = synth->io_ops->synth_in(synth); if (synth_id[test] == 'A') { do { /* read version string from synth */ - synth_id[++test] = synth->io_ops->synth_in(); + synth_id[++test] = synth->io_ops->synth_in(synth); } while (synth_id[test] != '\n' && test < 32); synth_id[++test] = 0x00; } diff --git a/drivers/accessibility/speakup/speakup_decext.c b/drivers/accessibility/speakup/speakup_decext.c index 7408eb29cf38..092cfd08a9e1 100644 --- a/drivers/accessibility/speakup/speakup_decext.c +++ b/drivers/accessibility/speakup/speakup_decext.c @@ -218,7 +218,7 @@ static void do_catch_up(struct spk_synth *synth) static void synth_flush(struct spk_synth *synth) { in_escape = 0; - synth->io_ops->flush_buffer(); + synth->io_ops->flush_buffer(synth); synth->synth_immediate(synth, "\033P;10z\033\\"); } diff --git a/drivers/accessibility/speakup/speakup_decpc.c b/drivers/accessibility/speakup/speakup_decpc.c index 96f24c848cc5..dec314dee214 100644 --- a/drivers/accessibility/speakup/speakup_decpc.c +++ b/drivers/accessibility/speakup/speakup_decpc.c @@ -125,7 +125,7 @@ enum { PRIMARY_DIC = 0, USER_DIC, COMMAND_DIC, ABBREV_DIC }; #define SYNTH_IO_EXTENT 8 static int synth_probe(struct spk_synth *synth); -static void dtpc_release(void); +static void dtpc_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -474,7 +474,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void dtpc_release(void) +static void dtpc_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts) diff --git a/drivers/accessibility/speakup/speakup_dectlk.c b/drivers/accessibility/speakup/speakup_dectlk.c index ab6d61e80b1c..580ec796816b 100644 --- a/drivers/accessibility/speakup/speakup_dectlk.c +++ b/drivers/accessibility/speakup/speakup_dectlk.c @@ -78,6 +78,8 @@ static struct kobj_attribute direct_attribute = __ATTR(direct, 0644, spk_var_show, spk_var_store); static struct kobj_attribute full_time_attribute = __ATTR(full_time, 0644, spk_var_show, spk_var_store); +static struct kobj_attribute flush_time_attribute = + __ATTR(flush_time, 0644, spk_var_show, spk_var_store); static struct kobj_attribute jiffy_delta_attribute = __ATTR(jiffy_delta, 0644, spk_var_show, spk_var_store); static struct kobj_attribute trigger_time_attribute = @@ -99,6 +101,7 @@ static struct attribute *synth_attrs[] = { &delay_time_attribute.attr, &direct_attribute.attr, &full_time_attribute.attr, + &flush_time_attribute.attr, &jiffy_delta_attribute.attr, &trigger_time_attribute.attr, NULL, /* need to NULL terminate the list of attributes */ @@ -118,6 +121,7 @@ static struct spk_synth synth_dectlk = { .trigger = 50, .jiffies = 50, .full = 40000, + .flush_time = 4000, .dev_name = SYNTH_DEFAULT_DEV, .startup = SYNTH_START, .checkval = SYNTH_CHECK, @@ -200,18 +204,23 @@ static void do_catch_up(struct spk_synth *synth) static u_char last = '\0'; unsigned long flags; unsigned long jiff_max; - unsigned long timeout = msecs_to_jiffies(4000); + unsigned long timeout; DEFINE_WAIT(wait); struct var_t *jiffy_delta; struct var_t *delay_time; + struct var_t *flush_time; int jiffy_delta_val; int delay_time_val; + int timeout_val; jiffy_delta = spk_get_var(JIFFY); delay_time = spk_get_var(DELAY); + flush_time = spk_get_var(FLUSH); spin_lock_irqsave(&speakup_info.spinlock, flags); jiffy_delta_val = jiffy_delta->u.n.value; + timeout_val = flush_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); + timeout = msecs_to_jiffies(timeout_val); jiff_max = jiffies + jiffy_delta_val; while (!kthread_should_stop()) { @@ -289,7 +298,7 @@ static void synth_flush(struct spk_synth *synth) synth->io_ops->synth_out(synth, ']'); in_escape = 0; is_flushing = 1; - synth->io_ops->flush_buffer(); + synth->io_ops->flush_buffer(synth); synth->io_ops->synth_out(synth, SYNTH_CLEAR); } diff --git a/drivers/accessibility/speakup/speakup_dtlk.c b/drivers/accessibility/speakup/speakup_dtlk.c index dbebed0eeeec..92838d3ae9eb 100644 --- a/drivers/accessibility/speakup/speakup_dtlk.c +++ b/drivers/accessibility/speakup/speakup_dtlk.c @@ -24,7 +24,7 @@ #define PROCSPEECH 0x00 static int synth_probe(struct spk_synth *synth); -static void dtlk_release(void); +static void dtlk_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -365,7 +365,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void dtlk_release(void) +static void dtlk_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts) diff --git a/drivers/accessibility/speakup/speakup_keypc.c b/drivers/accessibility/speakup/speakup_keypc.c index 414827e888fc..311f4aa0be22 100644 --- a/drivers/accessibility/speakup/speakup_keypc.c +++ b/drivers/accessibility/speakup/speakup_keypc.c @@ -24,7 +24,7 @@ #define SYNTH_CLEAR 0x03 static int synth_probe(struct spk_synth *synth); -static void keynote_release(void); +static void keynote_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -295,7 +295,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void keynote_release(void) +static void keynote_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (synth_port) diff --git a/drivers/accessibility/speakup/speakup_ltlk.c b/drivers/accessibility/speakup/speakup_ltlk.c index 3c59519a871f..3e59b387d0c4 100644 --- a/drivers/accessibility/speakup/speakup_ltlk.c +++ b/drivers/accessibility/speakup/speakup_ltlk.c @@ -132,7 +132,7 @@ static void synth_interrogate(struct spk_synth *synth) synth->synth_immediate(synth, "\x18\x01?"); for (i = 0; i < 50; i++) { - buf[i] = synth->io_ops->synth_in(); + buf[i] = synth->io_ops->synth_in(synth); if (i > 2 && buf[i] == 0x7f) break; } diff --git a/drivers/accessibility/speakup/speakup_soft.c b/drivers/accessibility/speakup/speakup_soft.c index 9a7029539f35..c3f97c572fb6 100644 --- a/drivers/accessibility/speakup/speakup_soft.c +++ b/drivers/accessibility/speakup/speakup_soft.c @@ -24,7 +24,7 @@ #define CLEAR_SYNTH 0x18 static int softsynth_probe(struct spk_synth *synth); -static void softsynth_release(void); +static void softsynth_release(struct spk_synth *synth); static int softsynth_is_alive(struct spk_synth *synth); static unsigned char get_index(struct spk_synth *synth); @@ -402,7 +402,7 @@ static int softsynth_probe(struct spk_synth *synth) return 0; } -static void softsynth_release(void) +static void softsynth_release(struct spk_synth *synth) { misc_deregister(&synth_device); misc_deregister(&synthu_device); diff --git a/drivers/accessibility/speakup/speakup_spkout.c b/drivers/accessibility/speakup/speakup_spkout.c index 6e933bf1de2e..bd3d8dc300ff 100644 --- a/drivers/accessibility/speakup/speakup_spkout.c +++ b/drivers/accessibility/speakup/speakup_spkout.c @@ -117,8 +117,8 @@ static struct spk_synth synth_spkout = { static void synth_flush(struct spk_synth *synth) { - synth->io_ops->flush_buffer(); - synth->io_ops->send_xchar(SYNTH_CLEAR); + synth->io_ops->flush_buffer(synth); + synth->io_ops->send_xchar(synth, SYNTH_CLEAR); } module_param_named(ser, synth_spkout.ser, int, 0444); diff --git a/drivers/accessibility/speakup/spk_priv.h b/drivers/accessibility/speakup/spk_priv.h index 0f4bcbe5ddb9..9da57ead17cb 100644 --- a/drivers/accessibility/speakup/spk_priv.h +++ b/drivers/accessibility/speakup/spk_priv.h @@ -34,8 +34,8 @@ const struct old_serial_port *spk_serial_init(int index); void spk_stop_serial_interrupt(void); -void spk_serial_release(void); -void spk_ttyio_release(void); +void spk_serial_release(struct spk_synth *synth); +void spk_ttyio_release(struct spk_synth *synth); void spk_ttyio_register_ldisc(void); void spk_ttyio_unregister_ldisc(void); diff --git a/drivers/accessibility/speakup/spk_ttyio.c b/drivers/accessibility/speakup/spk_ttyio.c index 835d17455fcd..9af1d4c124d3 100644 --- a/drivers/accessibility/speakup/spk_ttyio.c +++ b/drivers/accessibility/speakup/spk_ttyio.c @@ -12,14 +12,15 @@ struct spk_ldisc_data { char buf; struct completion completion; bool buf_free; + struct spk_synth *synth; }; -static struct spk_synth *spk_ttyio_synth; -static struct tty_struct *speakup_tty; -/* mutex to protect against speakup_tty disappearing from underneath us while - * we are using it. this can happen when the device physically unplugged, - * while in use. it also serialises access to speakup_tty. +/* + * This allows to catch within spk_ttyio_ldisc_open whether it is getting set + * on for a speakup-driven device. */ +static struct tty_struct *speakup_tty; +/* This mutex serializes the use of such global speakup_tty variable */ static DEFINE_MUTEX(speakup_tty_mutex); static int ser_to_dev(int ser, dev_t *dev_no) @@ -67,22 +68,20 @@ static int spk_ttyio_ldisc_open(struct tty_struct *tty) static void spk_ttyio_ldisc_close(struct tty_struct *tty) { - mutex_lock(&speakup_tty_mutex); - kfree(speakup_tty->disc_data); - speakup_tty = NULL; - mutex_unlock(&speakup_tty_mutex); + kfree(tty->disc_data); } static int spk_ttyio_receive_buf2(struct tty_struct *tty, const unsigned char *cp, char *fp, int count) { struct spk_ldisc_data *ldisc_data = tty->disc_data; + struct spk_synth *synth = ldisc_data->synth; - if (spk_ttyio_synth->read_buff_add) { + if (synth->read_buff_add) { int i; for (i = 0; i < count; i++) - spk_ttyio_synth->read_buff_add(cp[i]); + synth->read_buff_add(cp[i]); return count; } @@ -114,11 +113,11 @@ static struct tty_ldisc_ops spk_ttyio_ldisc_ops = { static int spk_ttyio_out(struct spk_synth *in_synth, const char ch); static int spk_ttyio_out_unicode(struct spk_synth *in_synth, u16 ch); -static void spk_ttyio_send_xchar(char ch); -static void spk_ttyio_tiocmset(unsigned int set, unsigned int clear); -static unsigned char spk_ttyio_in(void); -static unsigned char spk_ttyio_in_nowait(void); -static void spk_ttyio_flush_buffer(void); +static void spk_ttyio_send_xchar(struct spk_synth *in_synth, char ch); +static void spk_ttyio_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear); +static unsigned char spk_ttyio_in(struct spk_synth *in_synth); +static unsigned char spk_ttyio_in_nowait(struct spk_synth *in_synth); +static void spk_ttyio_flush_buffer(struct spk_synth *in_synth); static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth); struct spk_io_ops spk_ttyio_ops = { @@ -187,13 +186,17 @@ static int spk_ttyio_initialise_ldisc(struct spk_synth *synth) mutex_lock(&speakup_tty_mutex); speakup_tty = tty; ret = tty_set_ldisc(tty, N_SPEAKUP); - if (ret) - speakup_tty = NULL; + speakup_tty = NULL; mutex_unlock(&speakup_tty_mutex); - if (!ret) + if (!ret) { /* Success */ + struct spk_ldisc_data *ldisc_data = tty->disc_data; + + ldisc_data->synth = synth; + synth->dev = tty; return 0; + } pr_err("speakup: Failed to set N_SPEAKUP on tty\n"); @@ -221,29 +224,30 @@ void spk_ttyio_unregister_ldisc(void) static int spk_ttyio_out(struct spk_synth *in_synth, const char ch) { - mutex_lock(&speakup_tty_mutex); - if (in_synth->alive && speakup_tty && speakup_tty->ops->write) { - int ret = speakup_tty->ops->write(speakup_tty, &ch, 1); - - mutex_unlock(&speakup_tty_mutex); - if (ret == 0) - /* No room */ - return 0; - if (ret < 0) { - pr_warn("%s: I/O error, deactivating speakup\n", - in_synth->long_name); - /* No synth any more, so nobody will restart TTYs, - * and we thus need to do it ourselves. Now that there - * is no synth we can let application flood anyway - */ - in_synth->alive = 0; - speakup_start_ttys(); - return 0; - } + struct tty_struct *tty = in_synth->dev; + int ret; + + if (!in_synth->alive || !tty->ops->write) + return 0; + + ret = tty->ops->write(tty, &ch, 1); + + if (ret == 0) + /* No room */ + return 0; + + if (ret > 0) + /* Success */ return 1; - } - mutex_unlock(&speakup_tty_mutex); + pr_warn("%s: I/O error, deactivating speakup\n", + in_synth->long_name); + /* No synth any more, so nobody will restart TTYs, + * and we thus need to do it ourselves. Now that there + * is no synth we can let application flood anyway + */ + in_synth->alive = 0; + speakup_start_ttys(); return 0; } @@ -264,47 +268,20 @@ static int spk_ttyio_out_unicode(struct spk_synth *in_synth, u16 ch) return ret; } -static int check_tty(struct tty_struct *tty) -{ - if (!tty) { - pr_warn("%s: I/O error, deactivating speakup\n", - spk_ttyio_synth->long_name); - /* No synth any more, so nobody will restart TTYs, and we thus - * need to do it ourselves. Now that there is no synth we can - * let application flood anyway - */ - spk_ttyio_synth->alive = 0; - speakup_start_ttys(); - return 1; - } - - return 0; -} - -static void spk_ttyio_send_xchar(char ch) +static void spk_ttyio_send_xchar(struct spk_synth *in_synth, char ch) { - mutex_lock(&speakup_tty_mutex); - if (check_tty(speakup_tty)) { - mutex_unlock(&speakup_tty_mutex); - return; - } + struct tty_struct *tty = in_synth->dev; - if (speakup_tty->ops->send_xchar) - speakup_tty->ops->send_xchar(speakup_tty, ch); - mutex_unlock(&speakup_tty_mutex); + if (tty->ops->send_xchar) + tty->ops->send_xchar(tty, ch); } -static void spk_ttyio_tiocmset(unsigned int set, unsigned int clear) +static void spk_ttyio_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear) { - mutex_lock(&speakup_tty_mutex); - if (check_tty(speakup_tty)) { - mutex_unlock(&speakup_tty_mutex); - return; - } + struct tty_struct *tty = in_synth->dev; - if (speakup_tty->ops->tiocmset) - speakup_tty->ops->tiocmset(speakup_tty, set, clear); - mutex_unlock(&speakup_tty_mutex); + if (tty->ops->tiocmset) + tty->ops->tiocmset(tty, set, clear); } static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth) @@ -312,9 +289,10 @@ static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth) return 1; } -static unsigned char ttyio_in(int timeout) +static unsigned char ttyio_in(struct spk_synth *in_synth, int timeout) { - struct spk_ldisc_data *ldisc_data = speakup_tty->disc_data; + struct tty_struct *tty = in_synth->dev; + struct spk_ldisc_data *ldisc_data = tty->disc_data; char rv; if (!timeout) { @@ -334,35 +312,29 @@ static unsigned char ttyio_in(int timeout) mb(); ldisc_data->buf_free = true; /* Let TTY push more characters */ - tty_schedule_flip(speakup_tty->port); + tty_schedule_flip(tty->port); return rv; } -static unsigned char spk_ttyio_in(void) +static unsigned char spk_ttyio_in(struct spk_synth *in_synth) { - return ttyio_in(SPK_SYNTH_TIMEOUT); + return ttyio_in(in_synth, SPK_SYNTH_TIMEOUT); } -static unsigned char spk_ttyio_in_nowait(void) +static unsigned char spk_ttyio_in_nowait(struct spk_synth *in_synth) { - u8 rv = ttyio_in(0); + u8 rv = ttyio_in(in_synth, 0); return (rv == 0xff) ? 0 : rv; } -static void spk_ttyio_flush_buffer(void) +static void spk_ttyio_flush_buffer(struct spk_synth *in_synth) { - mutex_lock(&speakup_tty_mutex); - if (check_tty(speakup_tty)) { - mutex_unlock(&speakup_tty_mutex); - return; - } + struct tty_struct *tty = in_synth->dev; - if (speakup_tty->ops->flush_buffer) - speakup_tty->ops->flush_buffer(speakup_tty); - - mutex_unlock(&speakup_tty_mutex); + if (tty->ops->flush_buffer) + tty->ops->flush_buffer(tty); } int spk_ttyio_synth_probe(struct spk_synth *synth) @@ -373,37 +345,38 @@ int spk_ttyio_synth_probe(struct spk_synth *synth) return rv; synth->alive = 1; - spk_ttyio_synth = synth; return 0; } EXPORT_SYMBOL_GPL(spk_ttyio_synth_probe); -void spk_ttyio_release(void) +void spk_ttyio_release(struct spk_synth *in_synth) { - if (!speakup_tty) - return; + struct tty_struct *tty = in_synth->dev; - tty_lock(speakup_tty); + tty_lock(tty); - if (speakup_tty->ops->close) - speakup_tty->ops->close(speakup_tty, NULL); + if (tty->ops->close) + tty->ops->close(tty, NULL); + + tty_ldisc_flush(tty); + tty_unlock(tty); + tty_kclose(tty); - tty_ldisc_flush(speakup_tty); - tty_unlock(speakup_tty); - tty_kclose(speakup_tty); + in_synth->dev = NULL; } EXPORT_SYMBOL_GPL(spk_ttyio_release); -const char *spk_ttyio_synth_immediate(struct spk_synth *synth, const char *buff) +const char *spk_ttyio_synth_immediate(struct spk_synth *in_synth, const char *buff) { + struct tty_struct *tty = in_synth->dev; u_char ch; while ((ch = *buff)) { if (ch == '\n') - ch = synth->procspeech; - if (tty_write_room(speakup_tty) < 1 || - !synth->io_ops->synth_out(synth, ch)) + ch = in_synth->procspeech; + if (tty_write_room(tty) < 1 || + !in_synth->io_ops->synth_out(in_synth, ch)) return buff; buff++; } diff --git a/drivers/accessibility/speakup/spk_types.h b/drivers/accessibility/speakup/spk_types.h index 91fca3033a45..6a96ad94bc3f 100644 --- a/drivers/accessibility/speakup/spk_types.h +++ b/drivers/accessibility/speakup/spk_types.h @@ -48,7 +48,7 @@ enum var_id_t { ATTRIB_BLEEP, BLEEPS, RATE, PITCH, VOL, TONE, PUNCT, VOICE, FREQUENCY, LANG, DIRECT, PAUSE, - CAPS_START, CAPS_STOP, CHARTAB, INFLECTION, + CAPS_START, CAPS_STOP, CHARTAB, INFLECTION, FLUSH, MAXVARS }; @@ -157,11 +157,11 @@ struct spk_synth; struct spk_io_ops { int (*synth_out)(struct spk_synth *synth, const char ch); int (*synth_out_unicode)(struct spk_synth *synth, u16 ch); - void (*send_xchar)(char ch); - void (*tiocmset)(unsigned int set, unsigned int clear); - unsigned char (*synth_in)(void); - unsigned char (*synth_in_nowait)(void); - void (*flush_buffer)(void); + void (*send_xchar)(struct spk_synth *synth, char ch); + void (*tiocmset)(struct spk_synth *synth, unsigned int set, unsigned int clear); + unsigned char (*synth_in)(struct spk_synth *synth); + unsigned char (*synth_in_nowait)(struct spk_synth *synth); + void (*flush_buffer)(struct spk_synth *synth); int (*wait_for_xmitr)(struct spk_synth *synth); }; @@ -178,6 +178,7 @@ struct spk_synth { int trigger; int jiffies; int full; + int flush_time; int ser; char *dev_name; short flags; @@ -188,7 +189,7 @@ struct spk_synth { int *default_vol; struct spk_io_ops *io_ops; int (*probe)(struct spk_synth *synth); - void (*release)(void); + void (*release)(struct spk_synth *synth); const char *(*synth_immediate)(struct spk_synth *synth, const char *buff); void (*catch_up)(struct spk_synth *synth); @@ -200,6 +201,8 @@ struct spk_synth { struct synth_indexing indexing; int alive; struct attribute_group attributes; + + void *dev; }; /* diff --git a/drivers/accessibility/speakup/synth.c b/drivers/accessibility/speakup/synth.c index ac47dbac7207..2b8699673bac 100644 --- a/drivers/accessibility/speakup/synth.c +++ b/drivers/accessibility/speakup/synth.c @@ -137,14 +137,14 @@ EXPORT_SYMBOL_GPL(spk_do_catch_up_unicode); void spk_synth_flush(struct spk_synth *synth) { - synth->io_ops->flush_buffer(); + synth->io_ops->flush_buffer(synth); synth->io_ops->synth_out(synth, synth->clear); } EXPORT_SYMBOL_GPL(spk_synth_flush); unsigned char spk_synth_get_index(struct spk_synth *synth) { - return synth->io_ops->synth_in_nowait(); + return synth->io_ops->synth_in_nowait(synth); } EXPORT_SYMBOL_GPL(spk_synth_get_index); @@ -348,6 +348,7 @@ struct var_t synth_time_vars[] = { { TRIGGER, .u.n = {NULL, 20, 10, 2000, 0, 0, NULL } }, { JIFFY, .u.n = {NULL, 50, 20, 200, 0, 0, NULL } }, { FULL, .u.n = {NULL, 400, 200, 60000, 0, 0, NULL } }, + { FLUSH, .u.n = {NULL, 4000, 100, 4000, 0, 0, NULL } }, V_LAST_VAR }; @@ -408,6 +409,8 @@ static int do_synth_init(struct spk_synth *in_synth) synth_time_vars[2].u.n.default_val = synth->jiffies; synth_time_vars[3].u.n.value = synth_time_vars[3].u.n.default_val = synth->full; + synth_time_vars[4].u.n.value = + synth_time_vars[4].u.n.default_val = synth->flush_time; synth_printf("%s", synth->init); for (var = synth->vars; (var->var_id >= 0) && (var->var_id < MAXVARS); var++) @@ -440,7 +443,7 @@ void synth_release(void) sysfs_remove_group(speakup_kobj, &synth->attributes); for (var = synth->vars; var->var_id != MAXVARS; var++) speakup_unregister_var(var->var_id); - synth->release(); + synth->release(synth); synth = NULL; } diff --git a/drivers/accessibility/speakup/varhandlers.c b/drivers/accessibility/speakup/varhandlers.c index d7f6bec7ff06..067c0da97dcb 100644 --- a/drivers/accessibility/speakup/varhandlers.c +++ b/drivers/accessibility/speakup/varhandlers.c @@ -23,6 +23,7 @@ static struct st_var_header var_headers[] = { { "trigger_time", TRIGGER, VAR_TIME, NULL, NULL }, { "jiffy_delta", JIFFY, VAR_TIME, NULL, NULL }, { "full_time", FULL, VAR_TIME, NULL, NULL }, + { "flush_time", FLUSH, VAR_TIME, NULL, NULL }, { "spell_delay", SPELL_DELAY, VAR_NUM, &spk_spell_delay, NULL }, { "bleeps", BLEEPS, VAR_NUM, &spk_bleeps, NULL }, { "attrib_bleep", ATTRIB_BLEEP, VAR_NUM, &spk_attrib_bleep, NULL }, diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index fc06945d3f99..eedec61e3476 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -87,6 +87,14 @@ config ACPI_SPCR_TABLE This table provides information about the configuration of the earlycon console. +config ACPI_FPDT + bool "ACPI Firmware Performance Data Table (FPDT) support" + depends on X86_64 + help + Enable support for the Firmware Performance Data Table (FPDT). + This table provides information on the timing of the system + boot, S3 suspend and S3 resume firmware code paths. + config ACPI_LPIT bool depends on X86_64 @@ -327,21 +335,7 @@ config ACPI_THERMAL the module will be called thermal. config ACPI_PLATFORM_PROFILE - tristate "ACPI Platform Profile Driver" - default m - help - This driver adds support for platform-profiles on platforms that - support it. - - Platform-profiles can be used to control the platform behaviour. For - example whether to operate in a lower power mode, in a higher - power performance mode or between the two. - - This driver provides the sysfs interface and is used as the registration - point for platform specific drivers. - - Which profiles are supported is determined on a per-platform basis and - should be obtained from the platform specific driver. + tristate config ACPI_CUSTOM_DSDT_FILE string "Custom DSDT Table file to include" diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile index 52b627c7f977..700b41adf2db 100644 --- a/drivers/acpi/Makefile +++ b/drivers/acpi/Makefile @@ -57,6 +57,7 @@ acpi-$(CONFIG_X86) += x86/utils.o acpi-$(CONFIG_X86) += x86/s2idle.o acpi-$(CONFIG_DEBUG_FS) += debugfs.o acpi-y += acpi_lpat.o +acpi-$(CONFIG_ACPI_FPDT) += acpi_fpdt.o acpi-$(CONFIG_ACPI_LPIT) += acpi_lpit.o acpi-$(CONFIG_ACPI_GENERIC_GSI) += irq.o acpi-$(CONFIG_ACPI_WATCHDOG) += acpi_watchdog.o diff --git a/drivers/acpi/acpi_fpdt.c b/drivers/acpi/acpi_fpdt.c new file mode 100644 index 000000000000..a89a806a7a2a --- /dev/null +++ b/drivers/acpi/acpi_fpdt.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * FPDT support for exporting boot and suspend/resume performance data + * + * Copyright (C) 2021 Intel Corporation. All rights reserved. + */ + +#define pr_fmt(fmt) "ACPI FPDT: " fmt + +#include <linux/acpi.h> + +/* + * FPDT contains ACPI table header and a number of fpdt_subtable_entries. + * Each fpdt_subtable_entry points to a subtable: FBPT or S3PT. + * Each FPDT subtable (FBPT/S3PT) is composed of a fpdt_subtable_header + * and a number of fpdt performance records. + * Each FPDT performance record is composed of a fpdt_record_header and + * performance data fields, for boot or suspend or resume phase. + */ +enum fpdt_subtable_type { + SUBTABLE_FBPT, + SUBTABLE_S3PT, +}; + +struct fpdt_subtable_entry { + u16 type; /* refer to enum fpdt_subtable_type */ + u8 length; + u8 revision; + u32 reserved; + u64 address; /* physical address of the S3PT/FBPT table */ +}; + +struct fpdt_subtable_header { + u32 signature; + u32 length; +}; + +enum fpdt_record_type { + RECORD_S3_RESUME, + RECORD_S3_SUSPEND, + RECORD_BOOT, +}; + +struct fpdt_record_header { + u16 type; /* refer to enum fpdt_record_type */ + u8 length; + u8 revision; +}; + +struct resume_performance_record { + struct fpdt_record_header header; + u32 resume_count; + u64 resume_prev; + u64 resume_avg; +} __attribute__((packed)); + +struct boot_performance_record { + struct fpdt_record_header header; + u32 reserved; + u64 firmware_start; + u64 bootloader_load; + u64 bootloader_launch; + u64 exitbootservice_start; + u64 exitbootservice_end; +} __attribute__((packed)); + +struct suspend_performance_record { + struct fpdt_record_header header; + u64 suspend_start; + u64 suspend_end; +} __attribute__((packed)); + + +static struct resume_performance_record *record_resume; +static struct suspend_performance_record *record_suspend; +static struct boot_performance_record *record_boot; + +#define FPDT_ATTR(phase, name) \ +static ssize_t name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ +{ \ + return sprintf(buf, "%llu\n", record_##phase->name); \ +} \ +static struct kobj_attribute name##_attr = \ +__ATTR(name##_ns, 0444, name##_show, NULL) + +FPDT_ATTR(resume, resume_prev); +FPDT_ATTR(resume, resume_avg); +FPDT_ATTR(suspend, suspend_start); +FPDT_ATTR(suspend, suspend_end); +FPDT_ATTR(boot, firmware_start); +FPDT_ATTR(boot, bootloader_load); +FPDT_ATTR(boot, bootloader_launch); +FPDT_ATTR(boot, exitbootservice_start); +FPDT_ATTR(boot, exitbootservice_end); + +static ssize_t resume_count_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%u\n", record_resume->resume_count); +} + +static struct kobj_attribute resume_count_attr = +__ATTR_RO(resume_count); + +static struct attribute *resume_attrs[] = { + &resume_count_attr.attr, + &resume_prev_attr.attr, + &resume_avg_attr.attr, + NULL +}; + +static const struct attribute_group resume_attr_group = { + .attrs = resume_attrs, + .name = "resume", +}; + +static struct attribute *suspend_attrs[] = { + &suspend_start_attr.attr, + &suspend_end_attr.attr, + NULL +}; + +static const struct attribute_group suspend_attr_group = { + .attrs = suspend_attrs, + .name = "suspend", +}; + +static struct attribute *boot_attrs[] = { + &firmware_start_attr.attr, + &bootloader_load_attr.attr, + &bootloader_launch_attr.attr, + &exitbootservice_start_attr.attr, + &exitbootservice_end_attr.attr, + NULL +}; + +static const struct attribute_group boot_attr_group = { + .attrs = boot_attrs, + .name = "boot", +}; + +static struct kobject *fpdt_kobj; + +static int fpdt_process_subtable(u64 address, u32 subtable_type) +{ + struct fpdt_subtable_header *subtable_header; + struct fpdt_record_header *record_header; + char *signature = (subtable_type == SUBTABLE_FBPT ? "FBPT" : "S3PT"); + u32 length, offset; + int result; + + subtable_header = acpi_os_map_memory(address, sizeof(*subtable_header)); + if (!subtable_header) + return -ENOMEM; + + if (strncmp((char *)&subtable_header->signature, signature, 4)) { + pr_info(FW_BUG "subtable signature and type mismatch!\n"); + return -EINVAL; + } + + length = subtable_header->length; + acpi_os_unmap_memory(subtable_header, sizeof(*subtable_header)); + + subtable_header = acpi_os_map_memory(address, length); + if (!subtable_header) + return -ENOMEM; + + offset = sizeof(*subtable_header); + while (offset < length) { + record_header = (void *)subtable_header + offset; + offset += record_header->length; + + switch (record_header->type) { + case RECORD_S3_RESUME: + if (subtable_type != SUBTABLE_S3PT) { + pr_err(FW_BUG "Invalid record %d for subtable %s\n", + record_header->type, signature); + return -EINVAL; + } + if (record_resume) { + pr_err("Duplicate resume performance record found.\n"); + continue; + } + record_resume = (struct resume_performance_record *)record_header; + result = sysfs_create_group(fpdt_kobj, &resume_attr_group); + if (result) + return result; + break; + case RECORD_S3_SUSPEND: + if (subtable_type != SUBTABLE_S3PT) { + pr_err(FW_BUG "Invalid %d for subtable %s\n", + record_header->type, signature); + continue; + } + if (record_suspend) { + pr_err("Duplicate suspend performance record found.\n"); + continue; + } + record_suspend = (struct suspend_performance_record *)record_header; + result = sysfs_create_group(fpdt_kobj, &suspend_attr_group); + if (result) + return result; + break; + case RECORD_BOOT: + if (subtable_type != SUBTABLE_FBPT) { + pr_err(FW_BUG "Invalid %d for subtable %s\n", + record_header->type, signature); + return -EINVAL; + } + if (record_boot) { + pr_err("Duplicate boot performance record found.\n"); + continue; + } + record_boot = (struct boot_performance_record *)record_header; + result = sysfs_create_group(fpdt_kobj, &boot_attr_group); + if (result) + return result; + break; + + default: + pr_err(FW_BUG "Invalid record %d found.\n", record_header->type); + return -EINVAL; + } + } + return 0; +} + +static int __init acpi_init_fpdt(void) +{ + acpi_status status; + struct acpi_table_header *header; + struct fpdt_subtable_entry *subtable; + u32 offset = sizeof(*header); + + status = acpi_get_table(ACPI_SIG_FPDT, 0, &header); + + if (ACPI_FAILURE(status)) + return 0; + + fpdt_kobj = kobject_create_and_add("fpdt", acpi_kobj); + if (!fpdt_kobj) + return -ENOMEM; + + while (offset < header->length) { + subtable = (void *)header + offset; + switch (subtable->type) { + case SUBTABLE_FBPT: + case SUBTABLE_S3PT: + fpdt_process_subtable(subtable->address, + subtable->type); + break; + default: + pr_info(FW_BUG "Invalid subtable type %d found.\n", + subtable->type); + break; + } + offset += sizeof(*subtable); + } + return 0; +} + +fs_initcall(acpi_init_fpdt); diff --git a/drivers/acpi/acpica/acobject.h b/drivers/acpi/acpica/acobject.h index af47a3ffd2a4..9db5ae0f79ea 100644 --- a/drivers/acpi/acpica/acobject.h +++ b/drivers/acpi/acpica/acobject.h @@ -284,6 +284,7 @@ struct acpi_object_addr_handler { acpi_adr_space_handler handler; struct acpi_namespace_node *node; /* Parent device */ void *context; + acpi_mutex context_mutex; acpi_adr_space_setup setup; union acpi_operand_object *region_list; /* Regions using this handler */ union acpi_operand_object *next; diff --git a/drivers/acpi/acpica/evhandler.c b/drivers/acpi/acpica/evhandler.c index ea9485e6a475..c0cd7147a5a3 100644 --- a/drivers/acpi/acpica/evhandler.c +++ b/drivers/acpi/acpica/evhandler.c @@ -489,6 +489,13 @@ acpi_ev_install_space_handler(struct acpi_namespace_node *node, /* Init handler obj */ + status = + acpi_os_create_mutex(&handler_obj->address_space.context_mutex); + if (ACPI_FAILURE(status)) { + acpi_ut_remove_reference(handler_obj); + goto unlock_and_exit; + } + handler_obj->address_space.space_id = (u8)space_id; handler_obj->address_space.handler_flags = flags; handler_obj->address_space.region_list = NULL; diff --git a/drivers/acpi/acpica/evregion.c b/drivers/acpi/acpica/evregion.c index 3ed7d9ae95cf..4ef43c8ef5e7 100644 --- a/drivers/acpi/acpica/evregion.c +++ b/drivers/acpi/acpica/evregion.c @@ -112,6 +112,8 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, union acpi_operand_object *region_obj2; void *region_context = NULL; struct acpi_connection_info *context; + acpi_mutex context_mutex; + u8 context_locked; acpi_physical_address address; ACPI_FUNCTION_TRACE(ev_address_space_dispatch); @@ -136,6 +138,8 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, } context = handler_desc->address_space.context; + context_mutex = handler_desc->address_space.context_mutex; + context_locked = FALSE; /* * It may be the case that the region has never been initialized. @@ -204,6 +208,23 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, handler = handler_desc->address_space.handler; address = (region_obj->region.address + region_offset); + ACPI_DEBUG_PRINT((ACPI_DB_OPREGION, + "Handler %p (@%p) Address %8.8X%8.8X [%s]\n", + ®ion_obj->region.handler->address_space, handler, + ACPI_FORMAT_UINT64(address), + acpi_ut_get_region_name(region_obj->region. + space_id))); + + if (!(handler_desc->address_space.handler_flags & + ACPI_ADDR_HANDLER_DEFAULT_INSTALLED)) { + /* + * For handlers other than the default (supplied) handlers, we must + * exit the interpreter because the handler *might* block -- we don't + * know what it will do, so we can't hold the lock on the interpreter. + */ + acpi_ex_exit_interpreter(); + } + /* * Special handling for generic_serial_bus and general_purpose_io: * There are three extra parameters that must be passed to the @@ -212,48 +233,39 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, * 2) Length of the above buffer * 3) Actual access length from the access_as() op * + * Since we pass these extra parameters via the context, which is + * shared between threads, we must lock the context to avoid these + * parameters being changed from another thread before the handler + * has completed running. + * * In addition, for general_purpose_io, the Address and bit_width fields * are defined as follows: * 1) Address is the pin number index of the field (bit offset from * the previous Connection) * 2) bit_width is the actual bit length of the field (number of pins) */ - if ((region_obj->region.space_id == ACPI_ADR_SPACE_GSBUS) && + if ((region_obj->region.space_id == ACPI_ADR_SPACE_GSBUS || + region_obj->region.space_id == ACPI_ADR_SPACE_GPIO) && context && field_obj) { - /* Get the Connection (resource_template) buffer */ + status = + acpi_os_acquire_mutex(context_mutex, ACPI_WAIT_FOREVER); + if (ACPI_FAILURE(status)) { + goto re_enter_interpreter; + } - context->connection = field_obj->field.resource_buffer; - context->length = field_obj->field.resource_length; - context->access_length = field_obj->field.access_length; - } - if ((region_obj->region.space_id == ACPI_ADR_SPACE_GPIO) && - context && field_obj) { + context_locked = TRUE; /* Get the Connection (resource_template) buffer */ context->connection = field_obj->field.resource_buffer; context->length = field_obj->field.resource_length; context->access_length = field_obj->field.access_length; - address = field_obj->field.pin_number_index; - bit_width = field_obj->field.bit_length; - } - - ACPI_DEBUG_PRINT((ACPI_DB_OPREGION, - "Handler %p (@%p) Address %8.8X%8.8X [%s]\n", - ®ion_obj->region.handler->address_space, handler, - ACPI_FORMAT_UINT64(address), - acpi_ut_get_region_name(region_obj->region. - space_id))); - if (!(handler_desc->address_space.handler_flags & - ACPI_ADDR_HANDLER_DEFAULT_INSTALLED)) { - /* - * For handlers other than the default (supplied) handlers, we must - * exit the interpreter because the handler *might* block -- we don't - * know what it will do, so we can't hold the lock on the interpreter. - */ - acpi_ex_exit_interpreter(); + if (region_obj->region.space_id == ACPI_ADR_SPACE_GPIO) { + address = field_obj->field.pin_number_index; + bit_width = field_obj->field.bit_length; + } } /* Call the handler */ @@ -261,6 +273,10 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, status = handler(function, address, bit_width, value, context, region_obj2->extra.region_context); + if (context_locked) { + acpi_os_release_mutex(context_mutex); + } + if (ACPI_FAILURE(status)) { ACPI_EXCEPTION((AE_INFO, status, "Returned by Handler for [%s]", acpi_ut_get_region_name(region_obj->region. @@ -277,6 +293,7 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, } } +re_enter_interpreter: if (!(handler_desc->address_space.handler_flags & ACPI_ADDR_HANDLER_DEFAULT_INSTALLED)) { /* diff --git a/drivers/acpi/acpica/evxfregn.c b/drivers/acpi/acpica/evxfregn.c index 7672d70da850..b1ff0a8f9c14 100644 --- a/drivers/acpi/acpica/evxfregn.c +++ b/drivers/acpi/acpica/evxfregn.c @@ -201,6 +201,8 @@ acpi_remove_address_space_handler(acpi_handle device, /* Now we can delete the handler object */ + acpi_os_release_mutex(handler_obj->address_space. + context_mutex); acpi_ut_remove_reference(handler_obj); goto unlock_and_exit; } diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index 0bf072cef6cf..dcd593766a64 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -56,8 +56,6 @@ static struct acpi_scan_handler pci_root_handler = { }, }; -static DEFINE_MUTEX(osc_lock); - /** * acpi_is_root_bridge - determine whether an ACPI CA node is a PCI root bridge * @handle: the ACPI CA node in question. @@ -223,12 +221,7 @@ static acpi_status acpi_pci_query_osc(struct acpi_pci_root *root, static acpi_status acpi_pci_osc_support(struct acpi_pci_root *root, u32 flags) { - acpi_status status; - - mutex_lock(&osc_lock); - status = acpi_pci_query_osc(root, flags, NULL); - mutex_unlock(&osc_lock); - return status; + return acpi_pci_query_osc(root, flags, NULL); } struct acpi_pci_root *acpi_pci_find_root(acpi_handle handle) @@ -353,10 +346,10 @@ EXPORT_SYMBOL_GPL(acpi_get_pci_dev); * _OSC bits the BIOS has granted control of, but its contents are meaningless * on failure. **/ -acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) +static acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) { struct acpi_pci_root *root; - acpi_status status = AE_OK; + acpi_status status; u32 ctrl, capbuf[3]; if (!mask) @@ -370,18 +363,16 @@ acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) if (!root) return AE_NOT_EXIST; - mutex_lock(&osc_lock); - *mask = ctrl | root->osc_control_set; /* No need to evaluate _OSC if the control was already granted. */ if ((root->osc_control_set & ctrl) == ctrl) - goto out; + return AE_OK; /* Need to check the available controls bits before requesting them. */ while (*mask) { status = acpi_pci_query_osc(root, root->osc_support_set, mask); if (ACPI_FAILURE(status)) - goto out; + return status; if (ctrl == *mask) break; decode_osc_control(root, "platform does not support", @@ -392,21 +383,19 @@ acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) if ((ctrl & req) != req) { decode_osc_control(root, "not requesting control; platform does not support", req & ~(ctrl)); - status = AE_SUPPORT; - goto out; + return AE_SUPPORT; } capbuf[OSC_QUERY_DWORD] = 0; capbuf[OSC_SUPPORT_DWORD] = root->osc_support_set; capbuf[OSC_CONTROL_DWORD] = ctrl; status = acpi_pci_run_osc(handle, capbuf, mask); - if (ACPI_SUCCESS(status)) - root->osc_control_set = *mask; -out: - mutex_unlock(&osc_lock); - return status; + if (ACPI_FAILURE(status)) + return status; + + root->osc_control_set = *mask; + return AE_OK; } -EXPORT_SYMBOL(acpi_pci_osc_control_set); static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, bool is_pcie) @@ -452,9 +441,8 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, if ((status == AE_NOT_FOUND) && !is_pcie) return; - dev_info(&device->dev, "_OSC failed (%s)%s\n", - acpi_format_exception(status), - pcie_aspm_support_enabled() ? "; disabling ASPM" : ""); + dev_info(&device->dev, "_OSC: platform retains control of PCIe features (%s)\n", + acpi_format_exception(status)); return; } @@ -510,7 +498,7 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, } else { decode_osc_control(root, "OS requested", requested); decode_osc_control(root, "platform willing to grant", control); - dev_info(&device->dev, "_OSC failed (%s); disabling ASPM\n", + dev_info(&device->dev, "_OSC: platform retains control of PCIe features (%s)\n", acpi_format_exception(status)); /* diff --git a/drivers/acpi/platform_profile.c b/drivers/acpi/platform_profile.c index 4a59c5993bde..dd2fbf38e414 100644 --- a/drivers/acpi/platform_profile.c +++ b/drivers/acpi/platform_profile.c @@ -17,6 +17,7 @@ static const char * const profile_names[] = { [PLATFORM_PROFILE_COOL] = "cool", [PLATFORM_PROFILE_QUIET] = "quiet", [PLATFORM_PROFILE_BALANCED] = "balanced", + [PLATFORM_PROFILE_BALANCED_PERFORMANCE] = "balanced-performance", [PLATFORM_PROFILE_PERFORMANCE] = "performance", }; static_assert(ARRAY_SIZE(profile_names) == PLATFORM_PROFILE_LAST); diff --git a/drivers/auxdisplay/cfag12864b.c b/drivers/auxdisplay/cfag12864b.c index 7eebae7e322c..fd430e6866a1 100644 --- a/drivers/auxdisplay/cfag12864b.c +++ b/drivers/auxdisplay/cfag12864b.c @@ -5,7 +5,7 @@ * Description: cfag12864b LCD driver * Depends: ks0108 * - * Author: Copyright (C) Miguel Ojeda Sandonis + * Author: Copyright (C) Miguel Ojeda <ojeda@kernel.org> * Date: 2006-10-31 */ @@ -376,5 +376,5 @@ module_init(cfag12864b_init); module_exit(cfag12864b_exit); MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Miguel Ojeda Sandonis <miguel.ojeda.sandonis@gmail.com>"); +MODULE_AUTHOR("Miguel Ojeda <ojeda@kernel.org>"); MODULE_DESCRIPTION("cfag12864b LCD driver"); diff --git a/drivers/auxdisplay/cfag12864bfb.c b/drivers/auxdisplay/cfag12864bfb.c index 2002291ab338..d66821adf453 100644 --- a/drivers/auxdisplay/cfag12864bfb.c +++ b/drivers/auxdisplay/cfag12864bfb.c @@ -5,7 +5,7 @@ * Description: cfag12864b LCD framebuffer driver * Depends: cfag12864b * - * Author: Copyright (C) Miguel Ojeda Sandonis + * Author: Copyright (C) Miguel Ojeda <ojeda@kernel.org> * Date: 2006-10-31 */ @@ -171,5 +171,5 @@ module_init(cfag12864bfb_init); module_exit(cfag12864bfb_exit); MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Miguel Ojeda Sandonis <miguel.ojeda.sandonis@gmail.com>"); +MODULE_AUTHOR("Miguel Ojeda <ojeda@kernel.org>"); MODULE_DESCRIPTION("cfag12864b LCD framebuffer driver"); diff --git a/drivers/auxdisplay/ks0108.c b/drivers/auxdisplay/ks0108.c index abfe3fa9e6f4..03c95ad4216c 100644 --- a/drivers/auxdisplay/ks0108.c +++ b/drivers/auxdisplay/ks0108.c @@ -5,7 +5,7 @@ * Description: ks0108 LCD Controller driver * Depends: parport * - * Author: Copyright (C) Miguel Ojeda Sandonis + * Author: Copyright (C) Miguel Ojeda <ojeda@kernel.org> * Date: 2006-10-31 */ @@ -182,6 +182,6 @@ module_init(ks0108_init); module_exit(ks0108_exit); MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Miguel Ojeda Sandonis <miguel.ojeda.sandonis@gmail.com>"); +MODULE_AUTHOR("Miguel Ojeda <ojeda@kernel.org>"); MODULE_DESCRIPTION("ks0108 LCD Controller driver"); diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig index 040be48ce046..ffcbe2bc460e 100644 --- a/drivers/base/Kconfig +++ b/drivers/base/Kconfig @@ -161,7 +161,7 @@ config HMEM_REPORTING default n depends on NUMA help - Enable reporting for heterogenous memory access attributes under + Enable reporting for heterogeneous memory access attributes under their non-uniform memory nodes. source "drivers/base/test/Kconfig" @@ -213,4 +213,10 @@ config GENERIC_ARCH_TOPOLOGY appropriate scaling, sysfs interface for reading capacity values at runtime. +config GENERIC_ARCH_NUMA + bool + help + Enable support for generic NUMA implementation. Currently, RISC-V + and ARM64 use it. + endmenu diff --git a/drivers/base/Makefile b/drivers/base/Makefile index 5e7bf9669a81..8b93a7f291ec 100644 --- a/drivers/base/Makefile +++ b/drivers/base/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o obj-y += test/ diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c new file mode 100644 index 000000000000..4cc4e117727d --- /dev/null +++ b/drivers/base/arch_numa.c @@ -0,0 +1,484 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * NUMA support, based on the x86 implementation. + * + * Copyright (C) 2015 Cavium Inc. + * Author: Ganapatrao Kulkarni <gkulkarni@cavium.com> + */ + +#define pr_fmt(fmt) "NUMA: " fmt + +#include <linux/acpi.h> +#include <linux/memblock.h> +#include <linux/module.h> +#include <linux/of.h> + +#include <asm/sections.h> + +struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; +EXPORT_SYMBOL(node_data); +nodemask_t numa_nodes_parsed __initdata; +static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE }; + +static int numa_distance_cnt; +static u8 *numa_distance; +bool numa_off; + +static __init int numa_parse_early_param(char *opt) +{ + if (!opt) + return -EINVAL; + if (str_has_prefix(opt, "off")) + numa_off = true; + + return 0; +} +early_param("numa", numa_parse_early_param); + +cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; +EXPORT_SYMBOL(node_to_cpumask_map); + +#ifdef CONFIG_DEBUG_PER_CPU_MAPS + +/* + * Returns a pointer to the bitmask of CPUs on Node 'node'. + */ +const struct cpumask *cpumask_of_node(int node) +{ + + if (node == NUMA_NO_NODE) + return cpu_all_mask; + + if (WARN_ON(node < 0 || node >= nr_node_ids)) + return cpu_none_mask; + + if (WARN_ON(node_to_cpumask_map[node] == NULL)) + return cpu_online_mask; + + return node_to_cpumask_map[node]; +} +EXPORT_SYMBOL(cpumask_of_node); + +#endif + +static void numa_update_cpu(unsigned int cpu, bool remove) +{ + int nid = cpu_to_node(cpu); + + if (nid == NUMA_NO_NODE) + return; + + if (remove) + cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]); + else + cpumask_set_cpu(cpu, node_to_cpumask_map[nid]); +} + +void numa_add_cpu(unsigned int cpu) +{ + numa_update_cpu(cpu, false); +} + +void numa_remove_cpu(unsigned int cpu) +{ + numa_update_cpu(cpu, true); +} + +void numa_clear_node(unsigned int cpu) +{ + numa_remove_cpu(cpu); + set_cpu_numa_node(cpu, NUMA_NO_NODE); +} + +/* + * Allocate node_to_cpumask_map based on number of available nodes + * Requires node_possible_map to be valid. + * + * Note: cpumask_of_node() is not valid until after this is done. + * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.) + */ +static void __init setup_node_to_cpumask_map(void) +{ + int node; + + /* setup nr_node_ids if not done yet */ + if (nr_node_ids == MAX_NUMNODES) + setup_nr_node_ids(); + + /* allocate and clear the mapping */ + for (node = 0; node < nr_node_ids; node++) { + alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); + cpumask_clear(node_to_cpumask_map[node]); + } + + /* cpumask_of_node() will now work */ + pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids); +} + +/* + * Set the cpu to node and mem mapping + */ +void numa_store_cpu_info(unsigned int cpu) +{ + set_cpu_numa_node(cpu, cpu_to_node_map[cpu]); +} + +void __init early_map_cpu_to_node(unsigned int cpu, int nid) +{ + /* fallback to node 0 */ + if (nid < 0 || nid >= MAX_NUMNODES || numa_off) + nid = 0; + + cpu_to_node_map[cpu] = nid; + + /* + * We should set the numa node of cpu0 as soon as possible, because it + * has already been set up online before. cpu_to_node(0) will soon be + * called. + */ + if (!cpu) + set_cpu_numa_node(cpu, nid); +} + +#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA +unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; +EXPORT_SYMBOL(__per_cpu_offset); + +static int __init early_cpu_to_node(int cpu) +{ + return cpu_to_node_map[cpu]; +} + +static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) +{ + return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); +} + +static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, + size_t align) +{ + int nid = early_cpu_to_node(cpu); + + return memblock_alloc_try_nid(size, align, + __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid); +} + +static void __init pcpu_fc_free(void *ptr, size_t size) +{ + memblock_free_early(__pa(ptr), size); +} + +void __init setup_per_cpu_areas(void) +{ + unsigned long delta; + unsigned int cpu; + int rc; + + /* + * Always reserve area for module percpu variables. That's + * what the legacy allocator did. + */ + rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, + PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, + pcpu_cpu_distance, + pcpu_fc_alloc, pcpu_fc_free); + if (rc < 0) + panic("Failed to initialize percpu areas."); + + delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; + for_each_possible_cpu(cpu) + __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; +} +#endif + +/** + * numa_add_memblk() - Set node id to memblk + * @nid: NUMA node ID of the new memblk + * @start: Start address of the new memblk + * @end: End address of the new memblk + * + * RETURNS: + * 0 on success, -errno on failure. + */ +int __init numa_add_memblk(int nid, u64 start, u64 end) +{ + int ret; + + ret = memblock_set_node(start, (end - start), &memblock.memory, nid); + if (ret < 0) { + pr_err("memblock [0x%llx - 0x%llx] failed to add on node %d\n", + start, (end - 1), nid); + return ret; + } + + node_set(nid, numa_nodes_parsed); + return ret; +} + +/* + * Initialize NODE_DATA for a node on the local memory + */ +static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn) +{ + const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES); + u64 nd_pa; + void *nd; + int tnid; + + if (start_pfn >= end_pfn) + pr_info("Initmem setup node %d [<memory-less node>]\n", nid); + + nd_pa = memblock_phys_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid); + if (!nd_pa) + panic("Cannot allocate %zu bytes for node %d data\n", + nd_size, nid); + + nd = __va(nd_pa); + + /* report and initialize */ + pr_info("NODE_DATA [mem %#010Lx-%#010Lx]\n", + nd_pa, nd_pa + nd_size - 1); + tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT); + if (tnid != nid) + pr_info("NODE_DATA(%d) on node %d\n", nid, tnid); + + node_data[nid] = nd; + memset(NODE_DATA(nid), 0, sizeof(pg_data_t)); + NODE_DATA(nid)->node_id = nid; + NODE_DATA(nid)->node_start_pfn = start_pfn; + NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn; +} + +/* + * numa_free_distance + * + * The current table is freed. + */ +void __init numa_free_distance(void) +{ + size_t size; + + if (!numa_distance) + return; + + size = numa_distance_cnt * numa_distance_cnt * + sizeof(numa_distance[0]); + + memblock_free(__pa(numa_distance), size); + numa_distance_cnt = 0; + numa_distance = NULL; +} + +/* + * Create a new NUMA distance table. + */ +static int __init numa_alloc_distance(void) +{ + size_t size; + u64 phys; + int i, j; + + size = nr_node_ids * nr_node_ids * sizeof(numa_distance[0]); + phys = memblock_find_in_range(0, PFN_PHYS(max_pfn), + size, PAGE_SIZE); + if (WARN_ON(!phys)) + return -ENOMEM; + + memblock_reserve(phys, size); + + numa_distance = __va(phys); + numa_distance_cnt = nr_node_ids; + + /* fill with the default distances */ + for (i = 0; i < numa_distance_cnt; i++) + for (j = 0; j < numa_distance_cnt; j++) + numa_distance[i * numa_distance_cnt + j] = i == j ? + LOCAL_DISTANCE : REMOTE_DISTANCE; + + pr_debug("Initialized distance table, cnt=%d\n", numa_distance_cnt); + + return 0; +} + +/** + * numa_set_distance() - Set inter node NUMA distance from node to node. + * @from: the 'from' node to set distance + * @to: the 'to' node to set distance + * @distance: NUMA distance + * + * Set the distance from node @from to @to to @distance. + * If distance table doesn't exist, a warning is printed. + * + * If @from or @to is higher than the highest known node or lower than zero + * or @distance doesn't make sense, the call is ignored. + */ +void __init numa_set_distance(int from, int to, int distance) +{ + if (!numa_distance) { + pr_warn_once("Warning: distance table not allocated yet\n"); + return; + } + + if (from >= numa_distance_cnt || to >= numa_distance_cnt || + from < 0 || to < 0) { + pr_warn_once("Warning: node ids are out of bound, from=%d to=%d distance=%d\n", + from, to, distance); + return; + } + + if ((u8)distance != distance || + (from == to && distance != LOCAL_DISTANCE)) { + pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n", + from, to, distance); + return; + } + + numa_distance[from * numa_distance_cnt + to] = distance; +} + +/* + * Return NUMA distance @from to @to + */ +int __node_distance(int from, int to) +{ + if (from >= numa_distance_cnt || to >= numa_distance_cnt) + return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE; + return numa_distance[from * numa_distance_cnt + to]; +} +EXPORT_SYMBOL(__node_distance); + +static int __init numa_register_nodes(void) +{ + int nid; + struct memblock_region *mblk; + + /* Check that valid nid is set to memblks */ + for_each_mem_region(mblk) { + int mblk_nid = memblock_get_region_node(mblk); + phys_addr_t start = mblk->base; + phys_addr_t end = mblk->base + mblk->size - 1; + + if (mblk_nid == NUMA_NO_NODE || mblk_nid >= MAX_NUMNODES) { + pr_warn("Warning: invalid memblk node %d [mem %pap-%pap]\n", + mblk_nid, &start, &end); + return -EINVAL; + } + } + + /* Finally register nodes. */ + for_each_node_mask(nid, numa_nodes_parsed) { + unsigned long start_pfn, end_pfn; + + get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); + setup_node_data(nid, start_pfn, end_pfn); + node_set_online(nid); + } + + /* Setup online nodes to actual nodes*/ + node_possible_map = numa_nodes_parsed; + + return 0; +} + +static int __init numa_init(int (*init_func)(void)) +{ + int ret; + + nodes_clear(numa_nodes_parsed); + nodes_clear(node_possible_map); + nodes_clear(node_online_map); + + ret = numa_alloc_distance(); + if (ret < 0) + return ret; + + ret = init_func(); + if (ret < 0) + goto out_free_distance; + + if (nodes_empty(numa_nodes_parsed)) { + pr_info("No NUMA configuration found\n"); + ret = -EINVAL; + goto out_free_distance; + } + + ret = numa_register_nodes(); + if (ret < 0) + goto out_free_distance; + + setup_node_to_cpumask_map(); + + return 0; +out_free_distance: + numa_free_distance(); + return ret; +} + +/** + * dummy_numa_init() - Fallback dummy NUMA init + * + * Used if there's no underlying NUMA architecture, NUMA initialization + * fails, or NUMA is disabled on the command line. + * + * Must online at least one node (node 0) and add memory blocks that cover all + * allowed memory. It is unlikely that this function fails. + * + * Return: 0 on success, -errno on failure. + */ +static int __init dummy_numa_init(void) +{ + phys_addr_t start = memblock_start_of_DRAM(); + phys_addr_t end = memblock_end_of_DRAM() - 1; + int ret; + + if (numa_off) + pr_info("NUMA disabled\n"); /* Forced off on command line. */ + pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end); + + ret = numa_add_memblk(0, start, end + 1); + if (ret) { + pr_err("NUMA init failed\n"); + return ret; + } + + numa_off = true; + return 0; +} + +#ifdef CONFIG_ACPI_NUMA +static int __init arch_acpi_numa_init(void) +{ + int ret; + + ret = acpi_numa_init(); + if (ret) { + pr_info("Failed to initialise from firmware\n"); + return ret; + } + + return srat_disabled() ? -EINVAL : 0; +} +#else +static int __init arch_acpi_numa_init(void) +{ + return -EOPNOTSUPP; +} +#endif + +/** + * arch_numa_init() - Initialize NUMA + * + * Try each configured NUMA initialization method until one succeeds. The + * last fallback is dummy single node config encompassing whole memory. + */ +void __init arch_numa_init(void) +{ + if (!numa_off) { + if (!acpi_disabled && !numa_init(arch_acpi_numa_init)) + return; + if (acpi_disabled && !numa_init(of_numa_init)) + return; + } + + numa_init(dummy_numa_init); +} diff --git a/drivers/base/auxiliary.c b/drivers/base/auxiliary.c index 8336535f1e11..d8b314e7d0fd 100644 --- a/drivers/base/auxiliary.c +++ b/drivers/base/auxiliary.c @@ -15,6 +15,7 @@ #include <linux/pm_runtime.h> #include <linux/string.h> #include <linux/auxiliary_bus.h> +#include "base.h" static const struct auxiliary_device_id *auxiliary_match_id(const struct auxiliary_device_id *id, const struct auxiliary_device *auxdev) @@ -260,19 +261,11 @@ void auxiliary_driver_unregister(struct auxiliary_driver *auxdrv) } EXPORT_SYMBOL_GPL(auxiliary_driver_unregister); -static int __init auxiliary_bus_init(void) +void __init auxiliary_bus_init(void) { - return bus_register(&auxiliary_bus_type); + WARN_ON(bus_register(&auxiliary_bus_type)); } -static void __exit auxiliary_bus_exit(void) -{ - bus_unregister(&auxiliary_bus_type); -} - -module_init(auxiliary_bus_init); -module_exit(auxiliary_bus_exit); - MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Auxiliary Bus"); MODULE_AUTHOR("David Ertman <david.m.ertman@intel.com>"); diff --git a/drivers/base/base.h b/drivers/base/base.h index f5600a83124f..52b3d7b75c27 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -119,6 +119,11 @@ static inline int hypervisor_init(void) { return 0; } extern int platform_bus_init(void); extern void cpu_dev_init(void); extern void container_dev_init(void); +#ifdef CONFIG_AUXILIARY_BUS +extern void auxiliary_bus_init(void); +#else +static inline void auxiliary_bus_init(void) { } +#endif struct kobject *virtual_device_parent(struct device *dev); diff --git a/drivers/base/bus.c b/drivers/base/bus.c index a9c23ecebc7c..36d0c654ea61 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -633,7 +633,7 @@ int bus_add_driver(struct device_driver *drv) error = driver_add_groups(drv, bus->drv_groups); if (error) { /* How the hell do we get out of this pickle? Give up */ - printk(KERN_ERR "%s: driver_create_groups(%s) failed\n", + printk(KERN_ERR "%s: driver_add_groups(%s) failed\n", __func__, drv->name); } @@ -729,23 +729,6 @@ int device_reprobe(struct device *dev) } EXPORT_SYMBOL_GPL(device_reprobe); -/** - * find_bus - locate bus by name. - * @name: name of bus. - * - * Call kset_find_obj() to iterate over list of buses to - * find a bus by name. Return bus if found. - * - * Note that kset_find_obj increments bus' reference count. - */ -#if 0 -struct bus_type *find_bus(char *name) -{ - struct kobject *k = kset_find_obj(bus_kset, name); - return k ? to_bus(k) : NULL; -} -#endif /* 0 */ - static int bus_add_groups(struct bus_type *bus, const struct attribute_group **groups) { diff --git a/drivers/base/core.c b/drivers/base/core.c index 7c0406e675e9..f29839382f81 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -149,6 +149,21 @@ void fwnode_links_purge(struct fwnode_handle *fwnode) fwnode_links_purge_consumers(fwnode); } +static void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode) +{ + struct fwnode_handle *child; + + /* Don't purge consumer links of an added child */ + if (fwnode->dev) + return; + + fwnode->flags |= FWNODE_FLAG_NOT_DEVICE; + fwnode_links_purge_consumers(fwnode); + + fwnode_for_each_available_child_node(fwnode, child) + fw_devlink_purge_absent_suppliers(child); +} + #ifdef CONFIG_SRCU static DEFINE_MUTEX(device_links_lock); DEFINE_STATIC_SRCU(device_links_srcu); @@ -245,7 +260,8 @@ int device_is_dependent(struct device *dev, void *target) return ret; list_for_each_entry(link, &dev->links.consumers, s_node) { - if (link->flags == (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) + if ((link->flags & ~DL_FLAG_INFERRED) == + (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) continue; if (link->consumer == target) @@ -318,7 +334,8 @@ static int device_reorder_to_tail(struct device *dev, void *not_used) device_for_each_child(dev, NULL, device_reorder_to_tail); list_for_each_entry(link, &dev->links.consumers, s_node) { - if (link->flags == (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) + if ((link->flags & ~DL_FLAG_INFERRED) == + (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) continue; device_reorder_to_tail(link->consumer, NULL); } @@ -566,7 +583,8 @@ postcore_initcall(devlink_class_init); #define DL_MANAGED_LINK_FLAGS (DL_FLAG_AUTOREMOVE_CONSUMER | \ DL_FLAG_AUTOREMOVE_SUPPLIER | \ DL_FLAG_AUTOPROBE_CONSUMER | \ - DL_FLAG_SYNC_STATE_ONLY) + DL_FLAG_SYNC_STATE_ONLY | \ + DL_FLAG_INFERRED) #define DL_ADD_VALID_FLAGS (DL_MANAGED_LINK_FLAGS | DL_FLAG_STATELESS | \ DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE) @@ -635,7 +653,7 @@ struct device_link *device_link_add(struct device *consumer, if (!consumer || !supplier || flags & ~DL_ADD_VALID_FLAGS || (flags & DL_FLAG_STATELESS && flags & DL_MANAGED_LINK_FLAGS) || (flags & DL_FLAG_SYNC_STATE_ONLY && - flags != DL_FLAG_SYNC_STATE_ONLY) || + (flags & ~DL_FLAG_INFERRED) != DL_FLAG_SYNC_STATE_ONLY) || (flags & DL_FLAG_AUTOPROBE_CONSUMER && flags & (DL_FLAG_AUTOREMOVE_CONSUMER | DL_FLAG_AUTOREMOVE_SUPPLIER))) @@ -691,6 +709,10 @@ struct device_link *device_link_add(struct device *consumer, if (link->consumer != consumer) continue; + if (link->flags & DL_FLAG_INFERRED && + !(flags & DL_FLAG_INFERRED)) + link->flags &= ~DL_FLAG_INFERRED; + if (flags & DL_FLAG_PM_RUNTIME) { if (!(link->flags & DL_FLAG_PM_RUNTIME)) { pm_runtime_new_link(consumer); @@ -950,6 +972,10 @@ int device_links_check_suppliers(struct device *dev) mutex_lock(&fwnode_link_lock); if (dev->fwnode && !list_empty(&dev->fwnode->suppliers) && !fw_devlink_is_permissive()) { + dev_dbg(dev, "probe deferral - wait for supplier %pfwP\n", + list_first_entry(&dev->fwnode->suppliers, + struct fwnode_link, + c_hook)->supplier); mutex_unlock(&fwnode_link_lock); return -EPROBE_DEFER; } @@ -964,6 +990,8 @@ int device_links_check_suppliers(struct device *dev) if (link->status != DL_STATE_AVAILABLE && !(link->flags & DL_FLAG_SYNC_STATE_ONLY)) { device_links_missing_supplier(dev); + dev_dbg(dev, "probe deferral - supplier %s not ready\n", + dev_name(link->supplier)); ret = -EPROBE_DEFER; break; } @@ -1142,12 +1170,22 @@ void device_links_driver_bound(struct device *dev) LIST_HEAD(sync_list); /* - * If a device probes successfully, it's expected to have created all + * If a device binds successfully, it's expected to have created all * the device links it needs to or make new device links as it needs - * them. So, it no longer needs to wait on any suppliers. + * them. So, fw_devlink no longer needs to create device links to any + * of the device's suppliers. + * + * Also, if a child firmware node of this bound device is not added as + * a device by now, assume it is never going to be added and make sure + * other devices don't defer probe indefinitely by waiting for such a + * child device. */ - if (dev->fwnode && dev->fwnode->dev == dev) + if (dev->fwnode && dev->fwnode->dev == dev) { + struct fwnode_handle *child; fwnode_links_purge_suppliers(dev->fwnode); + fwnode_for_each_available_child_node(dev->fwnode, child) + fw_devlink_purge_absent_suppliers(child); + } device_remove_file(dev, &dev_attr_waiting_for_supplier); device_links_write_lock(); @@ -1458,7 +1496,14 @@ static void device_links_purge(struct device *dev) device_links_write_unlock(); } -static u32 fw_devlink_flags = DL_FLAG_SYNC_STATE_ONLY; +#define FW_DEVLINK_FLAGS_PERMISSIVE (DL_FLAG_INFERRED | \ + DL_FLAG_SYNC_STATE_ONLY) +#define FW_DEVLINK_FLAGS_ON (DL_FLAG_INFERRED | \ + DL_FLAG_AUTOPROBE_CONSUMER) +#define FW_DEVLINK_FLAGS_RPM (FW_DEVLINK_FLAGS_ON | \ + DL_FLAG_PM_RUNTIME) + +static u32 fw_devlink_flags = FW_DEVLINK_FLAGS_PERMISSIVE; static int __init fw_devlink_setup(char *arg) { if (!arg) @@ -1467,17 +1512,23 @@ static int __init fw_devlink_setup(char *arg) if (strcmp(arg, "off") == 0) { fw_devlink_flags = 0; } else if (strcmp(arg, "permissive") == 0) { - fw_devlink_flags = DL_FLAG_SYNC_STATE_ONLY; + fw_devlink_flags = FW_DEVLINK_FLAGS_PERMISSIVE; } else if (strcmp(arg, "on") == 0) { - fw_devlink_flags = DL_FLAG_AUTOPROBE_CONSUMER; + fw_devlink_flags = FW_DEVLINK_FLAGS_ON; } else if (strcmp(arg, "rpm") == 0) { - fw_devlink_flags = DL_FLAG_AUTOPROBE_CONSUMER | - DL_FLAG_PM_RUNTIME; + fw_devlink_flags = FW_DEVLINK_FLAGS_RPM; } return 0; } early_param("fw_devlink", fw_devlink_setup); +static bool fw_devlink_strict; +static int __init fw_devlink_strict_setup(char *arg) +{ + return strtobool(arg, &fw_devlink_strict); +} +early_param("fw_devlink.strict", fw_devlink_strict_setup); + u32 fw_devlink_get_flags(void) { return fw_devlink_flags; @@ -1485,7 +1536,12 @@ u32 fw_devlink_get_flags(void) static bool fw_devlink_is_permissive(void) { - return fw_devlink_flags == DL_FLAG_SYNC_STATE_ONLY; + return fw_devlink_flags == FW_DEVLINK_FLAGS_PERMISSIVE; +} + +bool fw_devlink_is_strict(void) +{ + return fw_devlink_strict && !fw_devlink_is_permissive(); } static void fw_devlink_parse_fwnode(struct fwnode_handle *fwnode) @@ -1508,6 +1564,53 @@ static void fw_devlink_parse_fwtree(struct fwnode_handle *fwnode) } /** + * fw_devlink_relax_cycle - Convert cyclic links to SYNC_STATE_ONLY links + * @con: Device to check dependencies for. + * @sup: Device to check against. + * + * Check if @sup depends on @con or any device dependent on it (its child or + * its consumer etc). When such a cyclic dependency is found, convert all + * device links created solely by fw_devlink into SYNC_STATE_ONLY device links. + * This is the equivalent of doing fw_devlink=permissive just between the + * devices in the cycle. We need to do this because, at this point, fw_devlink + * can't tell which of these dependencies is not a real dependency. + * + * Return 1 if a cycle is found. Otherwise, return 0. + */ +static int fw_devlink_relax_cycle(struct device *con, void *sup) +{ + struct device_link *link; + int ret; + + if (con == sup) + return 1; + + ret = device_for_each_child(con, sup, fw_devlink_relax_cycle); + if (ret) + return ret; + + list_for_each_entry(link, &con->links.consumers, s_node) { + if ((link->flags & ~DL_FLAG_INFERRED) == + (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) + continue; + + if (!fw_devlink_relax_cycle(link->consumer, sup)) + continue; + + ret = 1; + + if (!(link->flags & DL_FLAG_INFERRED)) + continue; + + pm_runtime_drop_link(link); + link->flags = DL_FLAG_MANAGED | FW_DEVLINK_FLAGS_PERMISSIVE; + dev_dbg(link->consumer, "Relaxing link with %s\n", + dev_name(link->supplier)); + } + return ret; +} + +/** * fw_devlink_create_devlink - Create a device link from a consumer to fwnode * @con - Consumer device for the device link * @sup_handle - fwnode handle of supplier @@ -1535,15 +1638,39 @@ static int fw_devlink_create_devlink(struct device *con, sup_dev = get_dev_from_fwnode(sup_handle); if (sup_dev) { /* + * If it's one of those drivers that don't actually bind to + * their device using driver core, then don't wait on this + * supplier device indefinitely. + */ + if (sup_dev->links.status == DL_DEV_NO_DRIVER && + sup_handle->flags & FWNODE_FLAG_INITIALIZED) { + ret = -EINVAL; + goto out; + } + + /* * If this fails, it is due to cycles in device links. Just * give up on this link and treat it as invalid. */ - if (!device_link_add(con, sup_dev, flags)) + if (!device_link_add(con, sup_dev, flags) && + !(flags & DL_FLAG_SYNC_STATE_ONLY)) { + dev_info(con, "Fixing up cyclic dependency with %s\n", + dev_name(sup_dev)); + device_links_write_lock(); + fw_devlink_relax_cycle(con, sup_dev); + device_links_write_unlock(); + device_link_add(con, sup_dev, + FW_DEVLINK_FLAGS_PERMISSIVE); ret = -EINVAL; + } goto out; } + /* Supplier that's already initialized without a struct device. */ + if (sup_handle->flags & FWNODE_FLAG_INITIALIZED) + return -EINVAL; + /* * DL_FLAG_SYNC_STATE_ONLY doesn't block probing and supports * cycles. So cycle detection isn't necessary and shouldn't be @@ -1632,7 +1759,7 @@ static void __fw_devlink_link_to_consumers(struct device *dev) con_dev = NULL; } else { own_link = false; - dl_flags = DL_FLAG_SYNC_STATE_ONLY; + dl_flags = FW_DEVLINK_FLAGS_PERMISSIVE; } } @@ -1687,7 +1814,7 @@ static void __fw_devlink_link_to_suppliers(struct device *dev, if (own_link) dl_flags = fw_devlink_get_flags(); else - dl_flags = DL_FLAG_SYNC_STATE_ONLY; + dl_flags = FW_DEVLINK_FLAGS_PERMISSIVE; list_for_each_entry_safe(link, tmp, &fwnode->suppliers, c_hook) { int ret; diff --git a/drivers/base/init.c b/drivers/base/init.c index 908e6520e804..a9f57c22fb9e 100644 --- a/drivers/base/init.c +++ b/drivers/base/init.c @@ -32,6 +32,7 @@ void __init driver_init(void) */ of_core_init(); platform_bus_init(); + auxiliary_bus_init(); cpu_dev_init(); memory_dev_init(); container_dev_init(); diff --git a/drivers/base/memory.c b/drivers/base/memory.c index eef4ffb6122c..f35298425575 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -35,7 +35,7 @@ static const char *const online_type_to_str[] = { [MMOP_ONLINE_MOVABLE] = "online_movable", }; -int memhp_online_type_from_str(const char *str) +int mhp_online_type_from_str(const char *str) { int i; @@ -253,7 +253,7 @@ static int memory_subsys_offline(struct device *dev) static ssize_t state_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - const int online_type = memhp_online_type_from_str(buf); + const int online_type = mhp_online_type_from_str(buf); struct memory_block *mem = to_memory_block(dev); int ret; @@ -290,20 +290,20 @@ static ssize_t state_store(struct device *dev, struct device_attribute *attr, } /* - * phys_device is a bad name for this. What I really want - * is a way to differentiate between memory ranges that - * are part of physical devices that constitute - * a complete removable unit or fru. - * i.e. do these ranges belong to the same physical device, - * s.t. if I offline all of these sections I can then - * remove the physical device? + * Legacy interface that we cannot remove: s390x exposes the storage increment + * covered by a memory block, allowing for identifying which memory blocks + * comprise a storage increment. Since a memory block spans complete + * storage increments nowadays, this interface is basically unused. Other + * archs never exposed != 0. */ static ssize_t phys_device_show(struct device *dev, struct device_attribute *attr, char *buf) { struct memory_block *mem = to_memory_block(dev); + unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); - return sysfs_emit(buf, "%d\n", mem->phys_device); + return sysfs_emit(buf, "%d\n", + arch_get_memory_phys_device(start_pfn)); } #ifdef CONFIG_MEMORY_HOTREMOVE @@ -387,19 +387,19 @@ static ssize_t auto_online_blocks_show(struct device *dev, struct device_attribute *attr, char *buf) { return sysfs_emit(buf, "%s\n", - online_type_to_str[memhp_default_online_type]); + online_type_to_str[mhp_default_online_type]); } static ssize_t auto_online_blocks_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - const int online_type = memhp_online_type_from_str(buf); + const int online_type = mhp_online_type_from_str(buf); if (online_type < 0) return -EINVAL; - memhp_default_online_type = online_type; + mhp_default_online_type = online_type; return count; } @@ -488,11 +488,7 @@ static DEVICE_ATTR_WO(soft_offline_page); static DEVICE_ATTR_WO(hard_offline_page); #endif -/* - * Note that phys_device is optional. It is here to allow for - * differentiation between which *physical* devices each - * section belongs to... - */ +/* See phys_device_show(). */ int __weak arch_get_memory_phys_device(unsigned long start_pfn) { return 0; @@ -574,7 +570,6 @@ int register_memory(struct memory_block *memory) static int init_memory_block(unsigned long block_id, unsigned long state) { struct memory_block *mem; - unsigned long start_pfn; int ret = 0; mem = find_memory_block_by_id(block_id); @@ -588,8 +583,6 @@ static int init_memory_block(unsigned long block_id, unsigned long state) mem->start_section_nr = block_id * sections_per_block; mem->state = state; - start_pfn = section_nr_to_pfn(mem->start_section_nr); - mem->phys_device = arch_get_memory_phys_device(start_pfn); mem->nid = NUMA_NO_NODE; ret = register_memory(mem); diff --git a/drivers/base/node.c b/drivers/base/node.c index 04f71c7bc3f8..f449dbb2c746 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -372,14 +372,19 @@ static ssize_t node_read_meminfo(struct device *dev, struct pglist_data *pgdat = NODE_DATA(nid); struct sysinfo i; unsigned long sreclaimable, sunreclaimable; + unsigned long swapcached = 0; si_meminfo_node(&i, nid); sreclaimable = node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B); sunreclaimable = node_page_state_pages(pgdat, NR_SLAB_UNRECLAIMABLE_B); +#ifdef CONFIG_SWAP + swapcached = node_page_state_pages(pgdat, NR_SWAPCACHE); +#endif len = sysfs_emit_at(buf, len, "Node %d MemTotal: %8lu kB\n" "Node %d MemFree: %8lu kB\n" "Node %d MemUsed: %8lu kB\n" + "Node %d SwapCached: %8lu kB\n" "Node %d Active: %8lu kB\n" "Node %d Inactive: %8lu kB\n" "Node %d Active(anon): %8lu kB\n" @@ -391,6 +396,7 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(i.totalram), nid, K(i.freeram), nid, K(i.totalram - i.freeram), + nid, K(swapcached), nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) + node_page_state(pgdat, NR_ACTIVE_FILE)), nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) + @@ -461,16 +467,11 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(sunreclaimable) #ifdef CONFIG_TRANSPARENT_HUGEPAGE , - nid, K(node_page_state(pgdat, NR_ANON_THPS) * - HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_SHMEM_THPS) * - HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * - HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_FILE_THPS) * - HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED) * - HPAGE_PMD_NR) + nid, K(node_page_state(pgdat, NR_ANON_THPS)), + nid, K(node_page_state(pgdat, NR_SHMEM_THPS)), + nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), + nid, K(node_page_state(pgdat, NR_FILE_THPS)), + nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED)) #endif ); len += hugetlb_report_node_meminfo(buf, len, nid); @@ -519,10 +520,14 @@ static ssize_t node_read_vmstat(struct device *dev, sum_zone_numa_state(nid, i)); #endif - for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) - len += sysfs_emit_at(buf, len, "%s %lu\n", - node_stat_name(i), - node_page_state_pages(pgdat, i)); + for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { + unsigned long pages = node_page_state_pages(pgdat, i); + + if (vmstat_item_print_in_thp(i)) + pages /= HPAGE_PMD_NR; + len += sysfs_emit_at(buf, len, "%s %lu\n", node_stat_name(i), + pages); + } return len; } diff --git a/drivers/base/platform.c b/drivers/base/platform.c index ac68328b0aca..6e1f8e0b661c 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -1463,13 +1463,16 @@ static int platform_remove(struct device *_dev) { struct platform_driver *drv = to_platform_driver(_dev->driver); struct platform_device *dev = to_platform_device(_dev); - int ret = 0; - if (drv->remove) - ret = drv->remove(dev); + if (drv->remove) { + int ret = drv->remove(dev); + + if (ret) + dev_warn(_dev, "remove callback returned a non-zero value. This will be ignored.\n"); + } dev_pm_domain_detach(_dev, true); - return ret; + return 0; } static void platform_shutdown(struct device *_dev) diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index aaf6c83b5cf6..78c310d3179d 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -2196,6 +2196,7 @@ static int genpd_add_provider(struct device_node *np, genpd_xlate_t xlate, cp->node = of_node_get(np); cp->data = data; cp->xlate = xlate; + fwnode_dev_initialized(&np->fwnode, true); mutex_lock(&of_genpd_mutex); list_add(&cp->link, &of_genpd_providers); @@ -2385,6 +2386,7 @@ void of_genpd_del_provider(struct device_node *np) } } + fwnode_dev_initialized(&cp->node->fwnode, false); list_del(&cp->link); of_node_put(cp->node); kfree(cp); diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index a46a7e30881b..18b82427d0cb 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -325,22 +325,22 @@ static void rpm_put_suppliers(struct device *dev) static int __rpm_callback(int (*cb)(struct device *), struct device *dev) __releases(&dev->power.lock) __acquires(&dev->power.lock) { - int retval, idx; bool use_links = dev->power.links_count > 0; + bool get = false; + int retval, idx; + bool put; if (dev->power.irq_safe) { spin_unlock(&dev->power.lock); + } else if (!use_links) { + spin_unlock_irq(&dev->power.lock); } else { + get = dev->power.runtime_status == RPM_RESUMING; + spin_unlock_irq(&dev->power.lock); - /* - * Resume suppliers if necessary. - * - * The device's runtime PM status cannot change until this - * routine returns, so it is safe to read the status outside of - * the lock. - */ - if (use_links && dev->power.runtime_status == RPM_RESUMING) { + /* Resume suppliers if necessary. */ + if (get) { idx = device_links_read_lock(); retval = rpm_get_suppliers(dev); @@ -355,24 +355,36 @@ static int __rpm_callback(int (*cb)(struct device *), struct device *dev) if (dev->power.irq_safe) { spin_lock(&dev->power.lock); - } else { - /* - * If the device is suspending and the callback has returned - * success, drop the usage counters of the suppliers that have - * been reference counted on its resume. - * - * Do that if resume fails too. - */ - if (use_links - && ((dev->power.runtime_status == RPM_SUSPENDING && !retval) - || (dev->power.runtime_status == RPM_RESUMING && retval))) { - idx = device_links_read_lock(); + return retval; + } - fail: - rpm_put_suppliers(dev); + spin_lock_irq(&dev->power.lock); - device_links_read_unlock(idx); - } + if (!use_links) + return retval; + + /* + * If the device is suspending and the callback has returned success, + * drop the usage counters of the suppliers that have been reference + * counted on its resume. + * + * Do that if the resume fails too. + */ + put = dev->power.runtime_status == RPM_SUSPENDING && !retval; + if (put) + __update_runtime_status(dev, RPM_SUSPENDED); + else + put = get && retval; + + if (put) { + spin_unlock_irq(&dev->power.lock); + + idx = device_links_read_lock(); + +fail: + rpm_put_suppliers(dev); + + device_links_read_unlock(idx); spin_lock_irq(&dev->power.lock); } diff --git a/drivers/base/regmap/regmap-sdw-mbq.c b/drivers/base/regmap/regmap-sdw-mbq.c index 8ce30650b97c..fe3ac26b66ad 100644 --- a/drivers/base/regmap/regmap-sdw-mbq.c +++ b/drivers/base/regmap/regmap-sdw-mbq.c @@ -15,11 +15,11 @@ static int regmap_sdw_mbq_write(void *context, unsigned int reg, unsigned int va struct sdw_slave *slave = dev_to_sdw_dev(dev); int ret; - ret = sdw_write(slave, SDW_SDCA_MBQ_CTL(reg), (val >> 8) & 0xff); + ret = sdw_write_no_pm(slave, SDW_SDCA_MBQ_CTL(reg), (val >> 8) & 0xff); if (ret < 0) return ret; - return sdw_write(slave, reg, val & 0xff); + return sdw_write_no_pm(slave, reg, val & 0xff); } static int regmap_sdw_mbq_read(void *context, unsigned int reg, unsigned int *val) @@ -29,11 +29,11 @@ static int regmap_sdw_mbq_read(void *context, unsigned int reg, unsigned int *va int read0; int read1; - read0 = sdw_read(slave, reg); + read0 = sdw_read_no_pm(slave, reg); if (read0 < 0) return read0; - read1 = sdw_read(slave, SDW_SDCA_MBQ_CTL(reg)); + read1 = sdw_read_no_pm(slave, SDW_SDCA_MBQ_CTL(reg)); if (read1 < 0) return read1; @@ -98,4 +98,4 @@ struct regmap *__devm_regmap_init_sdw_mbq(struct sdw_slave *sdw, EXPORT_SYMBOL_GPL(__devm_regmap_init_sdw_mbq); MODULE_DESCRIPTION("Regmap SoundWire MBQ Module"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/base/regmap/regmap-sdw.c b/drivers/base/regmap/regmap-sdw.c index c83be26434e7..966de8a136d9 100644 --- a/drivers/base/regmap/regmap-sdw.c +++ b/drivers/base/regmap/regmap-sdw.c @@ -13,7 +13,7 @@ static int regmap_sdw_write(void *context, unsigned int reg, unsigned int val) struct device *dev = context; struct sdw_slave *slave = dev_to_sdw_dev(dev); - return sdw_write(slave, reg, val); + return sdw_write_no_pm(slave, reg, val); } static int regmap_sdw_read(void *context, unsigned int reg, unsigned int *val) @@ -22,7 +22,7 @@ static int regmap_sdw_read(void *context, unsigned int reg, unsigned int *val) struct sdw_slave *slave = dev_to_sdw_dev(dev); int read; - read = sdw_read(slave, reg); + read = sdw_read_no_pm(slave, reg); if (read < 0) return read; diff --git a/drivers/base/test/Makefile b/drivers/base/test/Makefile index 3ca56367c84b..2f15fae8625f 100644 --- a/drivers/base/test/Makefile +++ b/drivers/base/test/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_TEST_ASYNC_DRIVER_PROBE) += test_async_driver_probe.o obj-$(CONFIG_KUNIT_DRIVER_PE_TEST) += property-entry-test.o +CFLAGS_REMOVE_property-entry-test.o += -fplugin-arg-structleak_plugin-byref -fplugin-arg-structleak_plugin-byref-all diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index 2779e85795a7..fd236158f32d 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -66,6 +66,12 @@ config AMIGA_Z2RAM To compile this driver as a module, choose M here: the module will be called z2ram. +config N64CART + bool "N64 cart support" + depends on MACH_NINTENDO64 + help + Support for the N64 cart. + config CDROM tristate select BLK_SCSI_REQUEST diff --git a/drivers/block/Makefile b/drivers/block/Makefile index b501b8728fb9..e3e3f1c79a82 100644 --- a/drivers/block/Makefile +++ b/drivers/block/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_PS3_DISK) += ps3disk.o obj-$(CONFIG_PS3_VRAM) += ps3vram.o obj-$(CONFIG_ATARI_FLOPPY) += ataflop.o obj-$(CONFIG_AMIGA_Z2RAM) += z2ram.o +obj-$(CONFIG_N64CART) += n64cart.o obj-$(CONFIG_BLK_DEV_RAM) += brd.o obj-$(CONFIG_BLK_DEV_LOOP) += loop.o obj-$(CONFIG_XILINX_SYSACE) += xsysace.o diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 578fc034db3f..a370cde3ddd4 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -663,7 +663,7 @@ static inline int is_loop_device(struct file *file) { struct inode *i = file->f_mapping->host; - return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR; + return i && S_ISBLK(i->i_mode) && imajor(i) == LOOP_MAJOR; } static int loop_validate_file(struct file *file, struct block_device *bdev) @@ -1212,6 +1212,9 @@ static int __loop_clr_fd(struct loop_device *lo, bool release) goto out_unlock; } + if (test_bit(QUEUE_FLAG_WC, &lo->lo_queue->queue_flags)) + blk_queue_write_cache(lo->lo_queue, false, false); + /* freeze request queue during the transition */ blk_mq_freeze_queue(lo->lo_queue); diff --git a/drivers/block/n64cart.c b/drivers/block/n64cart.c new file mode 100644 index 000000000000..47bdf324e962 --- /dev/null +++ b/drivers/block/n64cart.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for the N64 cart. + * + * Copyright (c) 2021 Lauri Kasanen + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <linux/bitops.h> +#include <linux/blkdev.h> +#include <linux/dma-mapping.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/platform_device.h> + +enum { + PI_DRAM_REG = 0, + PI_CART_REG, + PI_READ_REG, + PI_WRITE_REG, + PI_STATUS_REG, +}; + +#define PI_STATUS_DMA_BUSY (1 << 0) +#define PI_STATUS_IO_BUSY (1 << 1) + +#define CART_DOMAIN 0x10000000 +#define CART_MAX 0x1FFFFFFF + +#define MIN_ALIGNMENT 8 + +static u32 __iomem *reg_base; + +static unsigned int start; +module_param(start, uint, 0); +MODULE_PARM_DESC(start, "Start address of the cart block data"); + +static unsigned int size; +module_param(size, uint, 0); +MODULE_PARM_DESC(size, "Size of the cart block data, in bytes"); + +static void n64cart_write_reg(const u8 reg, const u32 value) +{ + writel(value, reg_base + reg); +} + +static u32 n64cart_read_reg(const u8 reg) +{ + return readl(reg_base + reg); +} + +static void n64cart_wait_dma(void) +{ + while (n64cart_read_reg(PI_STATUS_REG) & + (PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY)) + cpu_relax(); +} + +/* + * Process a single bvec of a bio. + */ +static bool n64cart_do_bvec(struct device *dev, struct bio_vec *bv, u32 pos) +{ + dma_addr_t dma_addr; + const u32 bstart = pos + start; + + /* Alignment check */ + WARN_ON_ONCE((bv->bv_offset & (MIN_ALIGNMENT - 1)) || + (bv->bv_len & (MIN_ALIGNMENT - 1))); + + dma_addr = dma_map_bvec(dev, bv, DMA_FROM_DEVICE, 0); + if (dma_mapping_error(dev, dma_addr)) + return false; + + n64cart_wait_dma(); + + n64cart_write_reg(PI_DRAM_REG, dma_addr + bv->bv_offset); + n64cart_write_reg(PI_CART_REG, (bstart | CART_DOMAIN) & CART_MAX); + n64cart_write_reg(PI_WRITE_REG, bv->bv_len - 1); + + n64cart_wait_dma(); + + dma_unmap_page(dev, dma_addr, bv->bv_len, DMA_FROM_DEVICE); + return true; +} + +static blk_qc_t n64cart_submit_bio(struct bio *bio) +{ + struct bio_vec bvec; + struct bvec_iter iter; + struct device *dev = bio->bi_disk->private_data; + u32 pos = bio->bi_iter.bi_sector << SECTOR_SHIFT; + + bio_for_each_segment(bvec, bio, iter) { + if (!n64cart_do_bvec(dev, &bvec, pos)) + goto io_error; + pos += bvec.bv_len; + } + + bio_endio(bio); + return BLK_QC_T_NONE; +io_error: + bio_io_error(bio); + return BLK_QC_T_NONE; +} + +static const struct block_device_operations n64cart_fops = { + .owner = THIS_MODULE, + .submit_bio = n64cart_submit_bio, +}; + +/* + * The target device is embedded and RAM-constrained. We save RAM + * by initializing in __init code that gets dropped late in boot. + * For the same reason there is no module or unloading support. + */ +static int __init n64cart_probe(struct platform_device *pdev) +{ + struct gendisk *disk; + + if (!start || !size) { + pr_err("start or size not specified\n"); + return -ENODEV; + } + + if (size & 4095) { + pr_err("size must be a multiple of 4K\n"); + return -ENODEV; + } + + reg_base = devm_platform_ioremap_resource(pdev, 0); + if (!reg_base) + return -EINVAL; + + disk = alloc_disk(0); + if (!disk) + return -ENOMEM; + + disk->queue = blk_alloc_queue(NUMA_NO_NODE); + if (!disk->queue) + return -ENOMEM; + + disk->first_minor = 0; + disk->flags = GENHD_FL_NO_PART_SCAN | GENHD_FL_EXT_DEVT; + disk->fops = &n64cart_fops; + disk->private_data = &pdev->dev; + strcpy(disk->disk_name, "n64cart"); + + set_capacity(disk, size >> SECTOR_SHIFT); + set_disk_ro(disk, 1); + + blk_queue_flag_set(QUEUE_FLAG_NONROT, disk->queue); + blk_queue_physical_block_size(disk->queue, 4096); + blk_queue_logical_block_size(disk->queue, 4096); + + add_disk(disk); + + pr_info("n64cart: %u kb disk\n", size / 1024); + + return 0; +} + +static struct platform_driver n64cart_driver = { + .driver = { + .name = "n64cart", + }, +}; + +static int __init n64cart_init(void) +{ + return platform_driver_probe(&n64cart_driver, n64cart_probe); +} + +module_init(n64cart_init); + +MODULE_AUTHOR("Lauri Kasanen <cand@gmx.com>"); +MODULE_DESCRIPTION("Driver for the N64 cart"); +MODULE_LICENSE("GPL"); diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 8b9622eb0a21..4ff71b579cfc 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -78,8 +78,7 @@ struct link_dead_args { #define NBD_RT_HAS_PID_FILE 3 #define NBD_RT_HAS_CONFIG_REF 4 #define NBD_RT_BOUND 5 -#define NBD_RT_DESTROY_ON_DISCONNECT 6 -#define NBD_RT_DISCONNECT_ON_CLOSE 7 +#define NBD_RT_DISCONNECT_ON_CLOSE 6 #define NBD_DESTROY_ON_DISCONNECT 0 #define NBD_DISCONNECT_REQUESTED 1 @@ -1904,12 +1903,21 @@ again: if (info->attrs[NBD_ATTR_CLIENT_FLAGS]) { u64 flags = nla_get_u64(info->attrs[NBD_ATTR_CLIENT_FLAGS]); if (flags & NBD_CFLAG_DESTROY_ON_DISCONNECT) { - set_bit(NBD_RT_DESTROY_ON_DISCONNECT, - &config->runtime_flags); - set_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags); - put_dev = true; + /* + * We have 1 ref to keep the device around, and then 1 + * ref for our current operation here, which will be + * inherited by the config. If we already have + * DESTROY_ON_DISCONNECT set then we know we don't have + * that extra ref already held so we don't need the + * put_dev. + */ + if (!test_and_set_bit(NBD_DESTROY_ON_DISCONNECT, + &nbd->flags)) + put_dev = true; } else { - clear_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags); + if (test_and_clear_bit(NBD_DESTROY_ON_DISCONNECT, + &nbd->flags)) + refcount_inc(&nbd->refs); } if (flags & NBD_CFLAG_DISCONNECT_ON_CLOSE) { set_bit(NBD_RT_DISCONNECT_ON_CLOSE, @@ -2080,15 +2088,13 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info) if (info->attrs[NBD_ATTR_CLIENT_FLAGS]) { u64 flags = nla_get_u64(info->attrs[NBD_ATTR_CLIENT_FLAGS]); if (flags & NBD_CFLAG_DESTROY_ON_DISCONNECT) { - if (!test_and_set_bit(NBD_RT_DESTROY_ON_DISCONNECT, - &config->runtime_flags)) + if (!test_and_set_bit(NBD_DESTROY_ON_DISCONNECT, + &nbd->flags)) put_dev = true; - set_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags); } else { - if (test_and_clear_bit(NBD_RT_DESTROY_ON_DISCONNECT, - &config->runtime_flags)) + if (test_and_clear_bit(NBD_DESTROY_ON_DISCONNECT, + &nbd->flags)) refcount_inc(&nbd->refs); - clear_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags); } if (flags & NBD_CFLAG_DISCONNECT_ON_CLOSE) { diff --git a/drivers/block/rsxx/core.c b/drivers/block/rsxx/core.c index 63f549889f87..5ac1881396af 100644 --- a/drivers/block/rsxx/core.c +++ b/drivers/block/rsxx/core.c @@ -165,15 +165,17 @@ static ssize_t rsxx_cram_read(struct file *fp, char __user *ubuf, { struct rsxx_cardinfo *card = file_inode(fp)->i_private; char *buf; - ssize_t st; + int st; buf = kzalloc(cnt, GFP_KERNEL); if (!buf) return -ENOMEM; st = rsxx_creg_read(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1); - if (!st) - st = copy_to_user(ubuf, buf, cnt); + if (!st) { + if (copy_to_user(ubuf, buf, cnt)) + st = -EFAULT; + } kfree(buf); if (st) return st; diff --git a/drivers/block/rsxx/rsxx_priv.h b/drivers/block/rsxx/rsxx_priv.h index 4861669e5786..6147977994ff 100644 --- a/drivers/block/rsxx/rsxx_priv.h +++ b/drivers/block/rsxx/rsxx_priv.h @@ -11,7 +11,6 @@ #ifndef __RSXX_PRIV_H__ #define __RSXX_PRIV_H__ -#include <linux/version.h> #include <linux/semaphore.h> #include <linux/fs.h> diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index b0285db7cf4f..b9fa3ef5b57c 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -705,6 +705,7 @@ static int virtblk_probe(struct virtio_device *vdev) u32 v, blk_size, max_size, sg_elems, opt_io_size; u16 min_io_size; u8 physical_block_exp, alignment_offset; + unsigned int queue_depth; if (!vdev->config->get) { dev_err(&vdev->dev, "%s failure: config access disabled\n", @@ -756,16 +757,18 @@ static int virtblk_probe(struct virtio_device *vdev) } /* Default queue sizing is to fill the ring. */ - if (!virtblk_queue_depth) { - virtblk_queue_depth = vblk->vqs[0].vq->num_free; + if (likely(!virtblk_queue_depth)) { + queue_depth = vblk->vqs[0].vq->num_free; /* ... but without indirect descs, we use 2 descs per req */ if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) - virtblk_queue_depth /= 2; + queue_depth /= 2; + } else { + queue_depth = virtblk_queue_depth; } memset(&vblk->tag_set, 0, sizeof(vblk->tag_set)); vblk->tag_set.ops = &virtio_mq_ops; - vblk->tag_set.queue_depth = virtblk_queue_depth; + vblk->tag_set.queue_depth = queue_depth; vblk->tag_set.numa_node = NUMA_NO_NODE; vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; vblk->tag_set.cmd_size = diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c index da16121140ca..1cdf09ff67b6 100644 --- a/drivers/block/xen-blkback/blkback.c +++ b/drivers/block/xen-blkback/blkback.c @@ -1326,9 +1326,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, pages[i]->page, seg[i].nsec << 9, seg[i].offset) == 0)) { - - int nr_iovecs = min_t(int, (nseg-i), BIO_MAX_PAGES); - bio = bio_alloc(GFP_KERNEL, nr_iovecs); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(nseg - i)); if (unlikely(bio == NULL)) goto fail_put_bio; diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index d7018543842e..a711a2e2a794 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1081,7 +1081,7 @@ static ssize_t mm_stat_show(struct device *dev, zram->limit_pages << PAGE_SHIFT, max_used << PAGE_SHIFT, (u64)atomic64_read(&zram->stats.same_pages), - pool_stats.pages_compacted, + atomic_long_read(&pool_stats.pages_compacted), (u64)atomic64_read(&zram->stats.huge_pages), (u64)atomic64_read(&zram->stats.huge_pages_since)); up_read(&zram->init_lock); diff --git a/drivers/bus/fsl-mc/Kconfig b/drivers/bus/fsl-mc/Kconfig index c23c77c9b705..b1fd55901c50 100644 --- a/drivers/bus/fsl-mc/Kconfig +++ b/drivers/bus/fsl-mc/Kconfig @@ -14,3 +14,10 @@ config FSL_MC_BUS architecture. The fsl-mc bus driver handles discovery of DPAA2 objects (which are represented as Linux devices) and binding objects to drivers. + +config FSL_MC_UAPI_SUPPORT + bool "Management Complex (MC) userspace support" + depends on FSL_MC_BUS + help + Provides userspace support for interrogating, creating, destroying or + configuring DPAA2 objects exported by the Management Complex. diff --git a/drivers/bus/fsl-mc/Makefile b/drivers/bus/fsl-mc/Makefile index 3c518c7e8374..4ae292a30e53 100644 --- a/drivers/bus/fsl-mc/Makefile +++ b/drivers/bus/fsl-mc/Makefile @@ -16,3 +16,6 @@ mc-bus-driver-objs := fsl-mc-bus.o \ fsl-mc-allocator.o \ fsl-mc-msi.o \ dpmcp.o + +# MC userspace support +obj-$(CONFIG_FSL_MC_UAPI_SUPPORT) += fsl-mc-uapi.o diff --git a/drivers/bus/fsl-mc/dprc-driver.c b/drivers/bus/fsl-mc/dprc-driver.c index 68488a7ad0d6..e3e2ae41c22b 100644 --- a/drivers/bus/fsl-mc/dprc-driver.c +++ b/drivers/bus/fsl-mc/dprc-driver.c @@ -237,8 +237,8 @@ static void dprc_add_new_devices(struct fsl_mc_device *mc_bus_dev, * populated before they can get allocation requests from probe callbacks * of the device drivers for the non-allocatable devices. */ -static int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev, - bool alloc_interrupts) +int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev, + bool alloc_interrupts) { int num_child_objects; int dprc_get_obj_failures; @@ -458,8 +458,9 @@ out: /* * Disable and clear interrupt for a given DPRC object */ -static int disable_dprc_irq(struct fsl_mc_device *mc_dev) +int disable_dprc_irq(struct fsl_mc_device *mc_dev) { + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); int error; struct fsl_mc_io *mc_io = mc_dev->mc_io; @@ -496,9 +497,18 @@ static int disable_dprc_irq(struct fsl_mc_device *mc_dev) return error; } + mc_bus->irq_enabled = 0; + return 0; } +int get_dprc_irq_state(struct fsl_mc_device *mc_dev) +{ + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); + + return mc_bus->irq_enabled; +} + static int register_dprc_irq_handler(struct fsl_mc_device *mc_dev) { int error; @@ -525,8 +535,9 @@ static int register_dprc_irq_handler(struct fsl_mc_device *mc_dev) return 0; } -static int enable_dprc_irq(struct fsl_mc_device *mc_dev) +int enable_dprc_irq(struct fsl_mc_device *mc_dev) { + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); int error; /* @@ -554,6 +565,8 @@ static int enable_dprc_irq(struct fsl_mc_device *mc_dev) return error; } + mc_bus->irq_enabled = 1; + return 0; } @@ -603,6 +616,7 @@ int dprc_setup(struct fsl_mc_device *mc_dev) struct irq_domain *mc_msi_domain; bool mc_io_created = false; bool msi_domain_set = false; + bool uapi_created = false; u16 major_ver, minor_ver; size_t region_size; int error; @@ -635,6 +649,11 @@ int dprc_setup(struct fsl_mc_device *mc_dev) return error; mc_io_created = true; + } else { + error = fsl_mc_uapi_create_device_file(mc_bus); + if (error < 0) + return -EPROBE_DEFER; + uapi_created = true; } mc_msi_domain = fsl_mc_find_msi_domain(&mc_dev->dev); @@ -692,6 +711,9 @@ error_cleanup_msi_domain: mc_dev->mc_io = NULL; } + if (uapi_created) + fsl_mc_uapi_remove_device_file(mc_bus); + return error; } EXPORT_SYMBOL_GPL(dprc_setup); @@ -763,6 +785,7 @@ static void dprc_teardown_irq(struct fsl_mc_device *mc_dev) int dprc_cleanup(struct fsl_mc_device *mc_dev) { + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); int error; /* this function should be called only for DPRCs, it @@ -793,6 +816,8 @@ int dprc_cleanup(struct fsl_mc_device *mc_dev) if (!fsl_mc_is_root_dprc(&mc_dev->dev)) { fsl_destroy_mc_io(mc_dev->mc_io); mc_dev->mc_io = NULL; + } else { + fsl_mc_uapi_remove_device_file(mc_bus); } return 0; diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index 8af978bd0000..380ad1fdb745 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -41,7 +41,7 @@ struct fsl_mc { struct fsl_mc_device *root_mc_bus_dev; u8 num_translation_ranges; struct fsl_mc_addr_translation_range *translation_ranges; - void *fsl_mc_regs; + void __iomem *fsl_mc_regs; }; /** @@ -208,12 +208,108 @@ static struct attribute *fsl_mc_dev_attrs[] = { ATTRIBUTE_GROUPS(fsl_mc_dev); +static int scan_fsl_mc_bus(struct device *dev, void *data) +{ + struct fsl_mc_device *root_mc_dev; + struct fsl_mc_bus *root_mc_bus; + + if (!fsl_mc_is_root_dprc(dev)) + goto exit; + + root_mc_dev = to_fsl_mc_device(dev); + root_mc_bus = to_fsl_mc_bus(root_mc_dev); + mutex_lock(&root_mc_bus->scan_mutex); + dprc_scan_objects(root_mc_dev, NULL); + mutex_unlock(&root_mc_bus->scan_mutex); + +exit: + return 0; +} + +static ssize_t rescan_store(struct bus_type *bus, + const char *buf, size_t count) +{ + unsigned long val; + + if (kstrtoul(buf, 0, &val) < 0) + return -EINVAL; + + if (val) + bus_for_each_dev(bus, NULL, NULL, scan_fsl_mc_bus); + + return count; +} +static BUS_ATTR_WO(rescan); + +static int fsl_mc_bus_set_autorescan(struct device *dev, void *data) +{ + struct fsl_mc_device *root_mc_dev; + unsigned long val; + char *buf = data; + + if (!fsl_mc_is_root_dprc(dev)) + goto exit; + + root_mc_dev = to_fsl_mc_device(dev); + + if (kstrtoul(buf, 0, &val) < 0) + return -EINVAL; + + if (val) + enable_dprc_irq(root_mc_dev); + else + disable_dprc_irq(root_mc_dev); + +exit: + return 0; +} + +static int fsl_mc_bus_get_autorescan(struct device *dev, void *data) +{ + struct fsl_mc_device *root_mc_dev; + char *buf = data; + + if (!fsl_mc_is_root_dprc(dev)) + goto exit; + + root_mc_dev = to_fsl_mc_device(dev); + + sprintf(buf, "%d\n", get_dprc_irq_state(root_mc_dev)); +exit: + return 0; +} + +static ssize_t autorescan_store(struct bus_type *bus, + const char *buf, size_t count) +{ + bus_for_each_dev(bus, NULL, (void *)buf, fsl_mc_bus_set_autorescan); + + return count; +} + +static ssize_t autorescan_show(struct bus_type *bus, char *buf) +{ + bus_for_each_dev(bus, NULL, (void *)buf, fsl_mc_bus_get_autorescan); + return strlen(buf); +} + +static BUS_ATTR_RW(autorescan); + +static struct attribute *fsl_mc_bus_attrs[] = { + &bus_attr_rescan.attr, + &bus_attr_autorescan.attr, + NULL, +}; + +ATTRIBUTE_GROUPS(fsl_mc_bus); + struct bus_type fsl_mc_bus_type = { .name = "fsl-mc", .match = fsl_mc_bus_match, .uevent = fsl_mc_bus_uevent, .dma_configure = fsl_mc_dma_configure, .dev_groups = fsl_mc_dev_groups, + .bus_groups = fsl_mc_bus_groups, }; EXPORT_SYMBOL_GPL(fsl_mc_bus_type); @@ -292,6 +388,11 @@ struct device_type fsl_mc_bus_dpdmai_type = { }; EXPORT_SYMBOL_GPL(fsl_mc_bus_dpdmai_type); +struct device_type fsl_mc_bus_dpdbg_type = { + .name = "fsl_mc_bus_dpdbg" +}; +EXPORT_SYMBOL_GPL(fsl_mc_bus_dpdbg_type); + static struct device_type *fsl_mc_get_device_type(const char *type) { static const struct { @@ -313,6 +414,7 @@ static struct device_type *fsl_mc_get_device_type(const char *type) { &fsl_mc_bus_dpaiop_type, "dpaiop" }, { &fsl_mc_bus_dpci_type, "dpci" }, { &fsl_mc_bus_dpdmai_type, "dpdmai" }, + { &fsl_mc_bus_dpdbg_type, "dpdbg" }, { NULL, NULL } }; int i; diff --git a/drivers/bus/fsl-mc/fsl-mc-private.h b/drivers/bus/fsl-mc/fsl-mc-private.h index c932387641fa..1958fa065360 100644 --- a/drivers/bus/fsl-mc/fsl-mc-private.h +++ b/drivers/bus/fsl-mc/fsl-mc-private.h @@ -10,6 +10,8 @@ #include <linux/fsl/mc.h> #include <linux/mutex.h> +#include <linux/ioctl.h> +#include <linux/miscdevice.h> /* * Data Path Management Complex (DPMNG) General API @@ -543,6 +545,22 @@ struct fsl_mc_resource_pool { }; /** + * struct fsl_mc_uapi - information associated with a device file + * @misc: struct miscdevice linked to the root dprc + * @device: newly created device in /dev + * @mutex: mutex lock to serialize the open/release operations + * @local_instance_in_use: local MC I/O instance in use or not + * @static_mc_io: pointer to the static MC I/O object + */ +struct fsl_mc_uapi { + struct miscdevice misc; + struct device *device; + struct mutex mutex; /* serialize open/release operations */ + u32 local_instance_in_use; + struct fsl_mc_io *static_mc_io; +}; + +/** * struct fsl_mc_bus - logical bus that corresponds to a physical DPRC * @mc_dev: fsl-mc device for the bus device itself. * @resource_pools: array of resource pools (one pool per resource type) @@ -551,6 +569,7 @@ struct fsl_mc_resource_pool { * @irq_resources: Pointer to array of IRQ objects for the IRQ pool * @scan_mutex: Serializes bus scanning * @dprc_attr: DPRC attributes + * @uapi_misc: struct that abstracts the interaction with userspace */ struct fsl_mc_bus { struct fsl_mc_device mc_dev; @@ -558,6 +577,8 @@ struct fsl_mc_bus { struct fsl_mc_device_irq *irq_resources; struct mutex scan_mutex; /* serializes bus scanning */ struct dprc_attributes dprc_attr; + struct fsl_mc_uapi uapi_misc; + int irq_enabled; }; #define to_fsl_mc_bus(_mc_dev) \ @@ -574,6 +595,9 @@ int __init dprc_driver_init(void); void dprc_driver_exit(void); +int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev, + bool alloc_interrupts); + int __init fsl_mc_allocator_driver_init(void); void fsl_mc_allocator_driver_exit(void); @@ -612,4 +636,29 @@ void fsl_mc_get_root_dprc(struct device *dev, struct fsl_mc_device *fsl_mc_device_lookup(struct fsl_mc_obj_desc *obj_desc, struct fsl_mc_device *mc_bus_dev); +u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd); + +#ifdef CONFIG_FSL_MC_UAPI_SUPPORT + +int fsl_mc_uapi_create_device_file(struct fsl_mc_bus *mc_bus); + +void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus); + +#else + +static inline int fsl_mc_uapi_create_device_file(struct fsl_mc_bus *mc_bus) +{ + return 0; +} + +static inline void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus) +{ +} + +#endif + +int disable_dprc_irq(struct fsl_mc_device *mc_dev); +int enable_dprc_irq(struct fsl_mc_device *mc_dev); +int get_dprc_irq_state(struct fsl_mc_device *mc_dev); + #endif /* _FSL_MC_PRIVATE_H_ */ diff --git a/drivers/bus/fsl-mc/fsl-mc-uapi.c b/drivers/bus/fsl-mc/fsl-mc-uapi.c new file mode 100644 index 000000000000..9c4c1395fcdb --- /dev/null +++ b/drivers/bus/fsl-mc/fsl-mc-uapi.c @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Management Complex (MC) userspace support + * + * Copyright 2021 NXP + * + */ + +#include <linux/slab.h> +#include <linux/fs.h> +#include <linux/uaccess.h> +#include <linux/miscdevice.h> + +#include "fsl-mc-private.h" + +struct uapi_priv_data { + struct fsl_mc_uapi *uapi; + struct fsl_mc_io *mc_io; +}; + +struct fsl_mc_cmd_desc { + u16 cmdid_value; + u16 cmdid_mask; + int size; + bool token; + int flags; +}; + +#define FSL_MC_CHECK_MODULE_ID BIT(0) +#define FSL_MC_CAP_NET_ADMIN_NEEDED BIT(1) + +enum fsl_mc_cmd_index { + DPDBG_DUMP = 0, + DPDBG_SET, + DPRC_GET_CONTAINER_ID, + DPRC_CREATE_CONT, + DPRC_DESTROY_CONT, + DPRC_ASSIGN, + DPRC_UNASSIGN, + DPRC_GET_OBJ_COUNT, + DPRC_GET_OBJ, + DPRC_GET_RES_COUNT, + DPRC_GET_RES_IDS, + DPRC_SET_OBJ_LABEL, + DPRC_SET_LOCKED, + DPRC_CONNECT, + DPRC_DISCONNECT, + DPRC_GET_POOL, + DPRC_GET_POOL_COUNT, + DPRC_GET_CONNECTION, + DPCI_GET_LINK_STATE, + DPCI_GET_PEER_ATTR, + DPAIOP_GET_SL_VERSION, + DPAIOP_GET_STATE, + DPMNG_GET_VERSION, + DPSECI_GET_TX_QUEUE, + DPMAC_GET_COUNTER, + DPMAC_GET_MAC_ADDR, + DPNI_SET_PRIM_MAC, + DPNI_GET_PRIM_MAC, + DPNI_GET_STATISTICS, + DPNI_GET_LINK_STATE, + DPNI_GET_MAX_FRAME_LENGTH, + DPSW_GET_TAILDROP, + DPSW_SET_TAILDROP, + DPSW_IF_GET_COUNTER, + DPSW_IF_GET_MAX_FRAME_LENGTH, + DPDMUX_GET_COUNTER, + DPDMUX_IF_GET_MAX_FRAME_LENGTH, + GET_ATTR, + GET_IRQ_MASK, + GET_IRQ_STATUS, + CLOSE, + OPEN, + GET_API_VERSION, + DESTROY, + CREATE, +}; + +static struct fsl_mc_cmd_desc fsl_mc_accepted_cmds[] = { + [DPDBG_DUMP] = { + .cmdid_value = 0x1300, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 28, + }, + [DPDBG_SET] = { + .cmdid_value = 0x1400, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 28, + }, + [DPRC_GET_CONTAINER_ID] = { + .cmdid_value = 0x8300, + .cmdid_mask = 0xFFF0, + .token = false, + .size = 8, + }, + [DPRC_CREATE_CONT] = { + .cmdid_value = 0x1510, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_DESTROY_CONT] = { + .cmdid_value = 0x1520, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 12, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_ASSIGN] = { + .cmdid_value = 0x1570, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_UNASSIGN] = { + .cmdid_value = 0x1580, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_GET_OBJ_COUNT] = { + .cmdid_value = 0x1590, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 16, + }, + [DPRC_GET_OBJ] = { + .cmdid_value = 0x15A0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 12, + }, + [DPRC_GET_RES_COUNT] = { + .cmdid_value = 0x15B0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 32, + }, + [DPRC_GET_RES_IDS] = { + .cmdid_value = 0x15C0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + }, + [DPRC_SET_OBJ_LABEL] = { + .cmdid_value = 0x1610, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 48, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_SET_LOCKED] = { + .cmdid_value = 0x16B0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 16, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_CONNECT] = { + .cmdid_value = 0x1670, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 56, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_DISCONNECT] = { + .cmdid_value = 0x1680, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 32, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_GET_POOL] = { + .cmdid_value = 0x1690, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 12, + }, + [DPRC_GET_POOL_COUNT] = { + .cmdid_value = 0x16A0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPRC_GET_CONNECTION] = { + .cmdid_value = 0x16C0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 32, + }, + + [DPCI_GET_LINK_STATE] = { + .cmdid_value = 0x0E10, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPCI_GET_PEER_ATTR] = { + .cmdid_value = 0x0E20, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPAIOP_GET_SL_VERSION] = { + .cmdid_value = 0x2820, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPAIOP_GET_STATE] = { + .cmdid_value = 0x2830, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPMNG_GET_VERSION] = { + .cmdid_value = 0x8310, + .cmdid_mask = 0xFFF0, + .token = false, + .size = 8, + }, + [DPSECI_GET_TX_QUEUE] = { + .cmdid_value = 0x1970, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 14, + }, + [DPMAC_GET_COUNTER] = { + .cmdid_value = 0x0c40, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 9, + }, + [DPMAC_GET_MAC_ADDR] = { + .cmdid_value = 0x0c50, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPNI_SET_PRIM_MAC] = { + .cmdid_value = 0x2240, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 16, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPNI_GET_PRIM_MAC] = { + .cmdid_value = 0x2250, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPNI_GET_STATISTICS] = { + .cmdid_value = 0x25D0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 10, + }, + [DPNI_GET_LINK_STATE] = { + .cmdid_value = 0x2150, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPNI_GET_MAX_FRAME_LENGTH] = { + .cmdid_value = 0x2170, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPSW_GET_TAILDROP] = { + .cmdid_value = 0x0A80, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 14, + }, + [DPSW_SET_TAILDROP] = { + .cmdid_value = 0x0A90, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 24, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPSW_IF_GET_COUNTER] = { + .cmdid_value = 0x0340, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 11, + }, + [DPSW_IF_GET_MAX_FRAME_LENGTH] = { + .cmdid_value = 0x0450, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 10, + }, + [DPDMUX_GET_COUNTER] = { + .cmdid_value = 0x0b20, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 11, + }, + [DPDMUX_IF_GET_MAX_FRAME_LENGTH] = { + .cmdid_value = 0x0a20, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 10, + }, + [GET_ATTR] = { + .cmdid_value = 0x0040, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [GET_IRQ_MASK] = { + .cmdid_value = 0x0150, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 13, + }, + [GET_IRQ_STATUS] = { + .cmdid_value = 0x0160, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 13, + }, + [CLOSE] = { + .cmdid_value = 0x8000, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + + /* Common commands amongst all types of objects. Must be checked last. */ + [OPEN] = { + .cmdid_value = 0x8000, + .cmdid_mask = 0xFC00, + .token = false, + .size = 12, + .flags = FSL_MC_CHECK_MODULE_ID, + }, + [GET_API_VERSION] = { + .cmdid_value = 0xA000, + .cmdid_mask = 0xFC00, + .token = false, + .size = 8, + .flags = FSL_MC_CHECK_MODULE_ID, + }, + [DESTROY] = { + .cmdid_value = 0x9800, + .cmdid_mask = 0xFC00, + .token = true, + .size = 12, + .flags = FSL_MC_CHECK_MODULE_ID | FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [CREATE] = { + .cmdid_value = 0x9000, + .cmdid_mask = 0xFC00, + .token = true, + .size = 64, + .flags = FSL_MC_CHECK_MODULE_ID | FSL_MC_CAP_NET_ADMIN_NEEDED, + }, +}; + +#define FSL_MC_NUM_ACCEPTED_CMDS ARRAY_SIZE(fsl_mc_accepted_cmds) + +#define FSL_MC_MAX_MODULE_ID 0x10 + +static int fsl_mc_command_check(struct fsl_mc_device *mc_dev, + struct fsl_mc_command *mc_cmd) +{ + struct fsl_mc_cmd_desc *desc = NULL; + int mc_cmd_max_size, i; + bool token_provided; + u16 cmdid, module_id; + char *mc_cmd_end; + char sum = 0; + + /* Check if this is an accepted MC command */ + cmdid = mc_cmd_hdr_read_cmdid(mc_cmd); + for (i = 0; i < FSL_MC_NUM_ACCEPTED_CMDS; i++) { + desc = &fsl_mc_accepted_cmds[i]; + if ((cmdid & desc->cmdid_mask) == desc->cmdid_value) + break; + } + if (i == FSL_MC_NUM_ACCEPTED_CMDS) { + dev_err(&mc_dev->dev, "MC command 0x%04x: cmdid not accepted\n", cmdid); + return -EACCES; + } + + /* Check if the size of the command is honored. Anything beyond the + * last valid byte of the command should be zeroed. + */ + mc_cmd_max_size = sizeof(*mc_cmd); + mc_cmd_end = ((char *)mc_cmd) + desc->size; + for (i = desc->size; i < mc_cmd_max_size; i++) + sum |= *mc_cmd_end++; + if (sum) { + dev_err(&mc_dev->dev, "MC command 0x%04x: garbage beyond max size of %d bytes!\n", + cmdid, desc->size); + return -EACCES; + } + + /* Some MC commands request a token to be passed so that object + * identification is possible. Check if the token passed in the command + * is as expected. + */ + token_provided = mc_cmd_hdr_read_token(mc_cmd) ? true : false; + if (token_provided != desc->token) { + dev_err(&mc_dev->dev, "MC command 0x%04x: token 0x%04x is invalid!\n", + cmdid, mc_cmd_hdr_read_token(mc_cmd)); + return -EACCES; + } + + /* If needed, check if the module ID passed is valid */ + if (desc->flags & FSL_MC_CHECK_MODULE_ID) { + /* The module ID is represented by bits [4:9] from the cmdid */ + module_id = (cmdid & GENMASK(9, 4)) >> 4; + if (module_id == 0 || module_id > FSL_MC_MAX_MODULE_ID) { + dev_err(&mc_dev->dev, "MC command 0x%04x: unknown module ID 0x%x\n", + cmdid, module_id); + return -EACCES; + } + } + + /* Some commands alter how hardware resources are managed. For these + * commands, check for CAP_NET_ADMIN. + */ + if (desc->flags & FSL_MC_CAP_NET_ADMIN_NEEDED) { + if (!capable(CAP_NET_ADMIN)) { + dev_err(&mc_dev->dev, "MC command 0x%04x: needs CAP_NET_ADMIN!\n", + cmdid); + return -EPERM; + } + } + + return 0; +} + +static int fsl_mc_uapi_send_command(struct fsl_mc_device *mc_dev, unsigned long arg, + struct fsl_mc_io *mc_io) +{ + struct fsl_mc_command mc_cmd; + int error; + + error = copy_from_user(&mc_cmd, (void __user *)arg, sizeof(mc_cmd)); + if (error) + return -EFAULT; + + error = fsl_mc_command_check(mc_dev, &mc_cmd); + if (error) + return error; + + error = mc_send_command(mc_io, &mc_cmd); + if (error) + return error; + + error = copy_to_user((void __user *)arg, &mc_cmd, sizeof(mc_cmd)); + if (error) + return -EFAULT; + + return 0; +} + +static int fsl_mc_uapi_dev_open(struct inode *inode, struct file *filep) +{ + struct fsl_mc_device *root_mc_device; + struct uapi_priv_data *priv_data; + struct fsl_mc_io *dynamic_mc_io; + struct fsl_mc_uapi *mc_uapi; + struct fsl_mc_bus *mc_bus; + int error; + + priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL); + if (!priv_data) + return -ENOMEM; + + mc_uapi = container_of(filep->private_data, struct fsl_mc_uapi, misc); + mc_bus = container_of(mc_uapi, struct fsl_mc_bus, uapi_misc); + root_mc_device = &mc_bus->mc_dev; + + mutex_lock(&mc_uapi->mutex); + + if (!mc_uapi->local_instance_in_use) { + priv_data->mc_io = mc_uapi->static_mc_io; + mc_uapi->local_instance_in_use = 1; + } else { + error = fsl_mc_portal_allocate(root_mc_device, 0, + &dynamic_mc_io); + if (error) { + dev_dbg(&root_mc_device->dev, + "Could not allocate MC portal\n"); + goto error_portal_allocate; + } + + priv_data->mc_io = dynamic_mc_io; + } + priv_data->uapi = mc_uapi; + filep->private_data = priv_data; + + mutex_unlock(&mc_uapi->mutex); + + return 0; + +error_portal_allocate: + mutex_unlock(&mc_uapi->mutex); + kfree(priv_data); + + return error; +} + +static int fsl_mc_uapi_dev_release(struct inode *inode, struct file *filep) +{ + struct uapi_priv_data *priv_data; + struct fsl_mc_uapi *mc_uapi; + struct fsl_mc_io *mc_io; + + priv_data = filep->private_data; + mc_uapi = priv_data->uapi; + mc_io = priv_data->mc_io; + + mutex_lock(&mc_uapi->mutex); + + if (mc_io == mc_uapi->static_mc_io) + mc_uapi->local_instance_in_use = 0; + else + fsl_mc_portal_free(mc_io); + + kfree(filep->private_data); + filep->private_data = NULL; + + mutex_unlock(&mc_uapi->mutex); + + return 0; +} + +static long fsl_mc_uapi_dev_ioctl(struct file *file, + unsigned int cmd, + unsigned long arg) +{ + struct uapi_priv_data *priv_data = file->private_data; + struct fsl_mc_device *root_mc_device; + struct fsl_mc_bus *mc_bus; + int error; + + mc_bus = container_of(priv_data->uapi, struct fsl_mc_bus, uapi_misc); + root_mc_device = &mc_bus->mc_dev; + + switch (cmd) { + case FSL_MC_SEND_MC_COMMAND: + error = fsl_mc_uapi_send_command(root_mc_device, arg, priv_data->mc_io); + break; + default: + dev_dbg(&root_mc_device->dev, "unexpected ioctl call number\n"); + error = -EINVAL; + } + + return error; +} + +static const struct file_operations fsl_mc_uapi_dev_fops = { + .owner = THIS_MODULE, + .open = fsl_mc_uapi_dev_open, + .release = fsl_mc_uapi_dev_release, + .unlocked_ioctl = fsl_mc_uapi_dev_ioctl, +}; + +int fsl_mc_uapi_create_device_file(struct fsl_mc_bus *mc_bus) +{ + struct fsl_mc_device *mc_dev = &mc_bus->mc_dev; + struct fsl_mc_uapi *mc_uapi = &mc_bus->uapi_misc; + int error; + + mc_uapi->misc.minor = MISC_DYNAMIC_MINOR; + mc_uapi->misc.name = dev_name(&mc_dev->dev); + mc_uapi->misc.fops = &fsl_mc_uapi_dev_fops; + + error = misc_register(&mc_uapi->misc); + if (error) + return error; + + mc_uapi->static_mc_io = mc_bus->mc_dev.mc_io; + + mutex_init(&mc_uapi->mutex); + + return 0; +} + +void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus) +{ + misc_deregister(&mc_bus->uapi_misc.misc); +} diff --git a/drivers/bus/fsl-mc/mc-sys.c b/drivers/bus/fsl-mc/mc-sys.c index 85a0225db522..b291b35e3884 100644 --- a/drivers/bus/fsl-mc/mc-sys.c +++ b/drivers/bus/fsl-mc/mc-sys.c @@ -35,7 +35,7 @@ static enum mc_cmd_status mc_cmd_hdr_read_status(struct fsl_mc_command *cmd) return (enum mc_cmd_status)hdr->status; } -static u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd) +u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd) { struct mc_cmd_header *hdr = (struct mc_cmd_header *)&cmd->header; u16 cmd_id = le16_to_cpu(hdr->cmd_id); diff --git a/drivers/bus/mhi/core/init.c b/drivers/bus/mhi/core/init.c index f0697f433c2f..be4eebb0971b 100644 --- a/drivers/bus/mhi/core/init.c +++ b/drivers/bus/mhi/core/init.c @@ -151,12 +151,17 @@ int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl) { struct mhi_event *mhi_event = mhi_cntrl->mhi_event; struct device *dev = &mhi_cntrl->mhi_dev->dev; + unsigned long irq_flags = IRQF_SHARED | IRQF_NO_SUSPEND; int i, ret; + /* if controller driver has set irq_flags, use it */ + if (mhi_cntrl->irq_flags) + irq_flags = mhi_cntrl->irq_flags; + /* Setup BHI_INTVEC IRQ */ ret = request_threaded_irq(mhi_cntrl->irq[0], mhi_intvec_handler, mhi_intvec_threaded_handler, - IRQF_SHARED | IRQF_NO_SUSPEND, + irq_flags, "bhi", mhi_cntrl); if (ret) return ret; @@ -174,7 +179,7 @@ int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl) ret = request_irq(mhi_cntrl->irq[mhi_event->irq], mhi_irq_handler, - IRQF_SHARED | IRQF_NO_SUSPEND, + irq_flags, "mhi", mhi_event); if (ret) { dev_err(dev, "Error requesting irq:%d for ev:%d\n", @@ -552,6 +557,9 @@ void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl, tre_ring = &mhi_chan->tre_ring; chan_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[mhi_chan->chan]; + if (!chan_ctxt->rbase) /* Already uninitialized */ + return; + mhi_free_coherent(mhi_cntrl, tre_ring->alloc_size, tre_ring->pre_aligned, tre_ring->dma_handle); vfree(buf_ring->base); diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c index 1202433ecf98..4e0131b94056 100644 --- a/drivers/bus/mhi/core/main.c +++ b/drivers/bus/mhi/core/main.c @@ -111,7 +111,14 @@ void mhi_ring_chan_db(struct mhi_controller *mhi_cntrl, dma_addr_t db; db = ring->iommu_base + (ring->wp - ring->base); + + /* + * Writes to the new ring element must be visible to the hardware + * before letting h/w know there is new element to fetch. + */ + dma_wmb(); *ring->ctxt_wp = db; + mhi_chan->db_cfg.process_db(mhi_cntrl, &mhi_chan->db_cfg, ring->db_addr, db); } @@ -135,6 +142,19 @@ enum mhi_state mhi_get_mhi_state(struct mhi_controller *mhi_cntrl) } EXPORT_SYMBOL_GPL(mhi_get_mhi_state); +void mhi_soc_reset(struct mhi_controller *mhi_cntrl) +{ + if (mhi_cntrl->reset) { + mhi_cntrl->reset(mhi_cntrl); + return; + } + + /* Generic MHI SoC reset */ + mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, + MHI_SOC_RESET_REQ); +} +EXPORT_SYMBOL_GPL(mhi_soc_reset); + int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl, struct mhi_buf_info *buf_info) { @@ -959,118 +979,88 @@ static bool mhi_is_ring_full(struct mhi_controller *mhi_cntrl, return (tmp == ring->rp); } -int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir, - struct sk_buff *skb, size_t len, enum mhi_flags mflags) +static int mhi_queue(struct mhi_device *mhi_dev, struct mhi_buf_info *buf_info, + enum dma_data_direction dir, enum mhi_flags mflags) { struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : mhi_dev->dl_chan; struct mhi_ring *tre_ring = &mhi_chan->tre_ring; - struct mhi_buf_info buf_info = { }; + unsigned long flags; int ret; - /* If MHI host pre-allocates buffers then client drivers cannot queue */ - if (mhi_chan->pre_alloc) - return -EINVAL; + if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) + return -EIO; - if (mhi_is_ring_full(mhi_cntrl, tre_ring)) - return -ENOMEM; + read_lock_irqsave(&mhi_cntrl->pm_lock, flags); - read_lock_bh(&mhi_cntrl->pm_lock); - if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) { - read_unlock_bh(&mhi_cntrl->pm_lock); - return -EIO; + ret = mhi_is_ring_full(mhi_cntrl, tre_ring); + if (unlikely(ret)) { + ret = -ENOMEM; + goto exit_unlock; } - /* we're in M3 or transitioning to M3 */ + ret = mhi_gen_tre(mhi_cntrl, mhi_chan, buf_info, mflags); + if (unlikely(ret)) + goto exit_unlock; + + /* trigger M3 exit if necessary */ if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) mhi_trigger_resume(mhi_cntrl); - /* Toggle wake to exit out of M2 */ + /* Assert dev_wake (to exit/prevent M1/M2)*/ mhi_cntrl->wake_toggle(mhi_cntrl); - buf_info.v_addr = skb->data; - buf_info.cb_buf = skb; - buf_info.len = len; - - ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); - if (unlikely(ret)) { - read_unlock_bh(&mhi_cntrl->pm_lock); - return ret; - } - if (mhi_chan->dir == DMA_TO_DEVICE) atomic_inc(&mhi_cntrl->pending_pkts); - if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { - read_lock_bh(&mhi_chan->lock); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); - read_unlock_bh(&mhi_chan->lock); + if (unlikely(!MHI_DB_ACCESS_VALID(mhi_cntrl))) { + ret = -EIO; + goto exit_unlock; } - read_unlock_bh(&mhi_cntrl->pm_lock); + mhi_ring_chan_db(mhi_cntrl, mhi_chan); - return 0; +exit_unlock: + read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags); + + return ret; } -EXPORT_SYMBOL_GPL(mhi_queue_skb); -int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir, - struct mhi_buf *mhi_buf, size_t len, enum mhi_flags mflags) +int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir, + struct sk_buff *skb, size_t len, enum mhi_flags mflags) { - struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : mhi_dev->dl_chan; - struct device *dev = &mhi_cntrl->mhi_dev->dev; - struct mhi_ring *tre_ring = &mhi_chan->tre_ring; struct mhi_buf_info buf_info = { }; - int ret; - - /* If MHI host pre-allocates buffers then client drivers cannot queue */ - if (mhi_chan->pre_alloc) - return -EINVAL; - if (mhi_is_ring_full(mhi_cntrl, tre_ring)) - return -ENOMEM; - - read_lock_bh(&mhi_cntrl->pm_lock); - if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) { - dev_err(dev, "MHI is not in activate state, PM state: %s\n", - to_mhi_pm_state_str(mhi_cntrl->pm_state)); - read_unlock_bh(&mhi_cntrl->pm_lock); + buf_info.v_addr = skb->data; + buf_info.cb_buf = skb; + buf_info.len = len; - return -EIO; - } + if (unlikely(mhi_chan->pre_alloc)) + return -EINVAL; - /* we're in M3 or transitioning to M3 */ - if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) - mhi_trigger_resume(mhi_cntrl); + return mhi_queue(mhi_dev, &buf_info, dir, mflags); +} +EXPORT_SYMBOL_GPL(mhi_queue_skb); - /* Toggle wake to exit out of M2 */ - mhi_cntrl->wake_toggle(mhi_cntrl); +int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir, + struct mhi_buf *mhi_buf, size_t len, enum mhi_flags mflags) +{ + struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : + mhi_dev->dl_chan; + struct mhi_buf_info buf_info = { }; buf_info.p_addr = mhi_buf->dma_addr; buf_info.cb_buf = mhi_buf; buf_info.pre_mapped = true; buf_info.len = len; - ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); - if (unlikely(ret)) { - read_unlock_bh(&mhi_cntrl->pm_lock); - return ret; - } - - if (mhi_chan->dir == DMA_TO_DEVICE) - atomic_inc(&mhi_cntrl->pending_pkts); - - if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { - read_lock_bh(&mhi_chan->lock); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); - read_unlock_bh(&mhi_chan->lock); - } - - read_unlock_bh(&mhi_cntrl->pm_lock); + if (unlikely(mhi_chan->pre_alloc)) + return -EINVAL; - return 0; + return mhi_queue(mhi_dev, &buf_info, dir, mflags); } EXPORT_SYMBOL_GPL(mhi_queue_dma); @@ -1124,57 +1114,13 @@ int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan, int mhi_queue_buf(struct mhi_device *mhi_dev, enum dma_data_direction dir, void *buf, size_t len, enum mhi_flags mflags) { - struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; - struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : - mhi_dev->dl_chan; - struct mhi_ring *tre_ring; struct mhi_buf_info buf_info = { }; - unsigned long flags; - int ret; - - /* - * this check here only as a guard, it's always - * possible mhi can enter error while executing rest of function, - * which is not fatal so we do not need to hold pm_lock - */ - if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) - return -EIO; - - tre_ring = &mhi_chan->tre_ring; - if (mhi_is_ring_full(mhi_cntrl, tre_ring)) - return -ENOMEM; buf_info.v_addr = buf; buf_info.cb_buf = buf; buf_info.len = len; - ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); - if (unlikely(ret)) - return ret; - - read_lock_irqsave(&mhi_cntrl->pm_lock, flags); - - /* we're in M3 or transitioning to M3 */ - if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) - mhi_trigger_resume(mhi_cntrl); - - /* Toggle wake to exit out of M2 */ - mhi_cntrl->wake_toggle(mhi_cntrl); - - if (mhi_chan->dir == DMA_TO_DEVICE) - atomic_inc(&mhi_cntrl->pending_pkts); - - if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { - unsigned long flags; - - read_lock_irqsave(&mhi_chan->lock, flags); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); - read_unlock_irqrestore(&mhi_chan->lock, flags); - } - - read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags); - - return 0; + return mhi_queue(mhi_dev, &buf_info, dir, mflags); } EXPORT_SYMBOL_GPL(mhi_queue_buf); diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index f5bee76ea061..20673a4b4a3c 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -8,13 +8,21 @@ * Copyright (C) 2020 Linaro Ltd <loic.poulain@linaro.org> */ +#include <linux/aer.h> +#include <linux/delay.h> #include <linux/device.h> #include <linux/mhi.h> #include <linux/module.h> #include <linux/pci.h> +#include <linux/timer.h> +#include <linux/workqueue.h> #define MHI_PCI_DEFAULT_BAR_NUM 0 +#define MHI_POST_RESET_DELAY_MS 500 + +#define HEALTH_CHECK_PERIOD (HZ * 2) + /** * struct mhi_pci_dev_info - MHI PCI device specific information * @config: MHI controller configuration @@ -76,6 +84,36 @@ struct mhi_pci_dev_info { .offload_channel = false, \ } +#define MHI_CHANNEL_CONFIG_HW_UL(ch_num, ch_name, el_count, ev_ring) \ + { \ + .num = ch_num, \ + .name = ch_name, \ + .num_elements = el_count, \ + .event_ring = ev_ring, \ + .dir = DMA_TO_DEVICE, \ + .ee_mask = BIT(MHI_EE_AMSS), \ + .pollcfg = 0, \ + .doorbell = MHI_DB_BRST_ENABLE, \ + .lpm_notify = false, \ + .offload_channel = false, \ + .doorbell_mode_switch = true, \ + } \ + +#define MHI_CHANNEL_CONFIG_HW_DL(ch_num, ch_name, el_count, ev_ring) \ + { \ + .num = ch_num, \ + .name = ch_name, \ + .num_elements = el_count, \ + .event_ring = ev_ring, \ + .dir = DMA_FROM_DEVICE, \ + .ee_mask = BIT(MHI_EE_AMSS), \ + .pollcfg = 0, \ + .doorbell = MHI_DB_BRST_ENABLE, \ + .lpm_notify = false, \ + .offload_channel = false, \ + .doorbell_mode_switch = true, \ + } + #define MHI_EVENT_CONFIG_DATA(ev_ring) \ { \ .num_elements = 128, \ @@ -91,8 +129,8 @@ struct mhi_pci_dev_info { #define MHI_EVENT_CONFIG_HW_DATA(ev_ring, ch_num) \ { \ - .num_elements = 128, \ - .irq_moderation_ms = 5, \ + .num_elements = 2048, \ + .irq_moderation_ms = 1, \ .irq = (ev_ring) + 1, \ .priority = 1, \ .mode = MHI_DB_BRST_DISABLE, \ @@ -104,27 +142,31 @@ struct mhi_pci_dev_info { } static const struct mhi_channel_config modem_qcom_v1_mhi_channels[] = { + MHI_CHANNEL_CONFIG_UL(4, "DIAG", 16, 1), + MHI_CHANNEL_CONFIG_DL(5, "DIAG", 16, 1), MHI_CHANNEL_CONFIG_UL(12, "MBIM", 4, 0), MHI_CHANNEL_CONFIG_DL(13, "MBIM", 4, 0), MHI_CHANNEL_CONFIG_UL(14, "QMI", 4, 0), MHI_CHANNEL_CONFIG_DL(15, "QMI", 4, 0), MHI_CHANNEL_CONFIG_UL(20, "IPCR", 8, 0), MHI_CHANNEL_CONFIG_DL(21, "IPCR", 8, 0), - MHI_CHANNEL_CONFIG_UL(100, "IP_HW0", 128, 1), - MHI_CHANNEL_CONFIG_DL(101, "IP_HW0", 128, 2), + MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 2), + MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 3), }; -static const struct mhi_event_config modem_qcom_v1_mhi_events[] = { +static struct mhi_event_config modem_qcom_v1_mhi_events[] = { /* first ring is control+data ring */ MHI_EVENT_CONFIG_CTRL(0), + /* DIAG dedicated event ring */ + MHI_EVENT_CONFIG_DATA(1), /* Hardware channels request dedicated hardware event rings */ - MHI_EVENT_CONFIG_HW_DATA(1, 100), - MHI_EVENT_CONFIG_HW_DATA(2, 101) + MHI_EVENT_CONFIG_HW_DATA(2, 100), + MHI_EVENT_CONFIG_HW_DATA(3, 101) }; -static const struct mhi_controller_config modem_qcom_v1_mhiv_config = { +static struct mhi_controller_config modem_qcom_v1_mhiv_config = { .max_channels = 128, - .timeout_ms = 5000, + .timeout_ms = 8000, .num_channels = ARRAY_SIZE(modem_qcom_v1_mhi_channels), .ch_cfg = modem_qcom_v1_mhi_channels, .num_events = ARRAY_SIZE(modem_qcom_v1_mhi_events), @@ -147,6 +189,18 @@ static const struct pci_device_id mhi_pci_id_table[] = { }; MODULE_DEVICE_TABLE(pci, mhi_pci_id_table); +enum mhi_pci_device_status { + MHI_PCI_DEV_STARTED, +}; + +struct mhi_pci_device { + struct mhi_controller mhi_cntrl; + struct pci_saved_state *pci_state; + struct work_struct recovery_work; + struct timer_list health_check_timer; + unsigned long status; +}; + static int mhi_pci_read_reg(struct mhi_controller *mhi_cntrl, void __iomem *addr, u32 *out) { @@ -163,7 +217,31 @@ static void mhi_pci_write_reg(struct mhi_controller *mhi_cntrl, static void mhi_pci_status_cb(struct mhi_controller *mhi_cntrl, enum mhi_callback cb) { + struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); + /* Nothing to do for now */ + switch (cb) { + case MHI_CB_FATAL_ERROR: + case MHI_CB_SYS_ERROR: + dev_warn(&pdev->dev, "firmware crashed (%u)\n", cb); + break; + default: + break; + } +} + +static bool mhi_pci_is_alive(struct mhi_controller *mhi_cntrl) +{ + struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); + u16 vendor = 0; + + if (pci_read_config_word(pdev, PCI_VENDOR_ID, &vendor)) + return false; + + if (vendor == (u16) ~0 || vendor == 0) + return false; + + return true; } static int mhi_pci_claim(struct mhi_controller *mhi_cntrl, @@ -227,8 +305,12 @@ static int mhi_pci_get_irqs(struct mhi_controller *mhi_cntrl, } if (nr_vectors < mhi_cntrl->nr_irqs) { - dev_warn(&pdev->dev, "Not enough MSI vectors (%d/%d), use shared MSI\n", - nr_vectors, mhi_cntrl_config->num_events); + dev_warn(&pdev->dev, "using shared MSI\n"); + + /* Patch msi vectors, use only one (shared) */ + for (i = 0; i < mhi_cntrl_config->num_events; i++) + mhi_cntrl_config->event_cfg[i].irq = 0; + mhi_cntrl->nr_irqs = 1; } irq = devm_kcalloc(&pdev->dev, mhi_cntrl->nr_irqs, sizeof(int), GFP_KERNEL); @@ -257,20 +339,89 @@ static void mhi_pci_runtime_put(struct mhi_controller *mhi_cntrl) /* no PM for now */ } +static void mhi_pci_recovery_work(struct work_struct *work) +{ + struct mhi_pci_device *mhi_pdev = container_of(work, struct mhi_pci_device, + recovery_work); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); + int err; + + dev_warn(&pdev->dev, "device recovery started\n"); + + del_timer(&mhi_pdev->health_check_timer); + + /* Clean up MHI state */ + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, false); + mhi_unprepare_after_power_down(mhi_cntrl); + } + + /* Check if we can recover without full reset */ + pci_set_power_state(pdev, PCI_D0); + pci_load_saved_state(pdev, mhi_pdev->pci_state); + pci_restore_state(pdev); + + if (!mhi_pci_is_alive(mhi_cntrl)) + goto err_try_reset; + + err = mhi_prepare_for_power_up(mhi_cntrl); + if (err) + goto err_try_reset; + + err = mhi_sync_power_up(mhi_cntrl); + if (err) + goto err_unprepare; + + dev_dbg(&pdev->dev, "Recovery completed\n"); + + set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); + return; + +err_unprepare: + mhi_unprepare_after_power_down(mhi_cntrl); +err_try_reset: + if (pci_reset_function(pdev)) + dev_err(&pdev->dev, "Recovery failed\n"); +} + +static void health_check(struct timer_list *t) +{ + struct mhi_pci_device *mhi_pdev = from_timer(mhi_pdev, t, health_check_timer); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + if (!mhi_pci_is_alive(mhi_cntrl)) { + dev_err(mhi_cntrl->cntrl_dev, "Device died\n"); + queue_work(system_long_wq, &mhi_pdev->recovery_work); + return; + } + + /* reschedule in two seconds */ + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); +} + static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { const struct mhi_pci_dev_info *info = (struct mhi_pci_dev_info *) id->driver_data; const struct mhi_controller_config *mhi_cntrl_config; + struct mhi_pci_device *mhi_pdev; struct mhi_controller *mhi_cntrl; int err; dev_dbg(&pdev->dev, "MHI PCI device found: %s\n", info->name); - mhi_cntrl = mhi_alloc_controller(); - if (!mhi_cntrl) + /* mhi_pdev.mhi_cntrl must be zero-initialized */ + mhi_pdev = devm_kzalloc(&pdev->dev, sizeof(*mhi_pdev), GFP_KERNEL); + if (!mhi_pdev) return -ENOMEM; + INIT_WORK(&mhi_pdev->recovery_work, mhi_pci_recovery_work); + timer_setup(&mhi_pdev->health_check_timer, health_check, 0); + mhi_cntrl_config = info->config; + mhi_cntrl = &mhi_pdev->mhi_cntrl; + mhi_cntrl->cntrl_dev = &pdev->dev; mhi_cntrl->iova_start = 0; mhi_cntrl->iova_stop = (dma_addr_t)DMA_BIT_MASK(info->dma_data_width); @@ -285,17 +436,23 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) err = mhi_pci_claim(mhi_cntrl, info->bar_num, DMA_BIT_MASK(info->dma_data_width)); if (err) - goto err_release; + return err; err = mhi_pci_get_irqs(mhi_cntrl, mhi_cntrl_config); if (err) - goto err_release; + return err; + + pci_set_drvdata(pdev, mhi_pdev); + + /* Have stored pci confspace at hand for restore in sudden PCI error */ + pci_save_state(pdev); + mhi_pdev->pci_state = pci_store_saved_state(pdev); - pci_set_drvdata(pdev, mhi_cntrl); + pci_enable_pcie_error_reporting(pdev); err = mhi_register_controller(mhi_cntrl, mhi_cntrl_config); if (err) - goto err_release; + return err; /* MHI bus does not power up the controller by default */ err = mhi_prepare_for_power_up(mhi_cntrl); @@ -310,33 +467,209 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_unprepare; } + set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + + /* start health check */ + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); + return 0; err_unprepare: mhi_unprepare_after_power_down(mhi_cntrl); err_unregister: mhi_unregister_controller(mhi_cntrl); -err_release: - mhi_free_controller(mhi_cntrl); return err; } static void mhi_pci_remove(struct pci_dev *pdev) { - struct mhi_controller *mhi_cntrl = pci_get_drvdata(pdev); + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + del_timer(&mhi_pdev->health_check_timer); + cancel_work_sync(&mhi_pdev->recovery_work); + + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, true); + mhi_unprepare_after_power_down(mhi_cntrl); + } - mhi_power_down(mhi_cntrl, true); - mhi_unprepare_after_power_down(mhi_cntrl); mhi_unregister_controller(mhi_cntrl); - mhi_free_controller(mhi_cntrl); } +static void mhi_pci_reset_prepare(struct pci_dev *pdev) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + dev_info(&pdev->dev, "reset\n"); + + del_timer(&mhi_pdev->health_check_timer); + + /* Clean up MHI state */ + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, false); + mhi_unprepare_after_power_down(mhi_cntrl); + } + + /* cause internal device reset */ + mhi_soc_reset(mhi_cntrl); + + /* Be sure device reset has been executed */ + msleep(MHI_POST_RESET_DELAY_MS); +} + +static void mhi_pci_reset_done(struct pci_dev *pdev) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + int err; + + /* Restore initial known working PCI state */ + pci_load_saved_state(pdev, mhi_pdev->pci_state); + pci_restore_state(pdev); + + /* Is device status available ? */ + if (!mhi_pci_is_alive(mhi_cntrl)) { + dev_err(&pdev->dev, "reset failed\n"); + return; + } + + err = mhi_prepare_for_power_up(mhi_cntrl); + if (err) { + dev_err(&pdev->dev, "failed to prepare MHI controller\n"); + return; + } + + err = mhi_sync_power_up(mhi_cntrl); + if (err) { + dev_err(&pdev->dev, "failed to power up MHI controller\n"); + mhi_unprepare_after_power_down(mhi_cntrl); + return; + } + + set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); +} + +static pci_ers_result_t mhi_pci_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + dev_err(&pdev->dev, "PCI error detected, state = %u\n", state); + + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + /* Clean up MHI state */ + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, false); + mhi_unprepare_after_power_down(mhi_cntrl); + } else { + /* Nothing to do */ + return PCI_ERS_RESULT_RECOVERED; + } + + pci_disable_device(pdev); + + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t mhi_pci_slot_reset(struct pci_dev *pdev) +{ + if (pci_enable_device(pdev)) { + dev_err(&pdev->dev, "Cannot re-enable PCI device after reset.\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + return PCI_ERS_RESULT_RECOVERED; +} + +static void mhi_pci_io_resume(struct pci_dev *pdev) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + + dev_err(&pdev->dev, "PCI slot reset done\n"); + + queue_work(system_long_wq, &mhi_pdev->recovery_work); +} + +static const struct pci_error_handlers mhi_pci_err_handler = { + .error_detected = mhi_pci_error_detected, + .slot_reset = mhi_pci_slot_reset, + .resume = mhi_pci_io_resume, + .reset_prepare = mhi_pci_reset_prepare, + .reset_done = mhi_pci_reset_done, +}; + +static int __maybe_unused mhi_pci_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + del_timer(&mhi_pdev->health_check_timer); + cancel_work_sync(&mhi_pdev->recovery_work); + + /* Transition to M3 state */ + mhi_pm_suspend(mhi_cntrl); + + pci_save_state(pdev); + pci_disable_device(pdev); + pci_wake_from_d3(pdev, true); + pci_set_power_state(pdev, PCI_D3hot); + + return 0; +} + +static int __maybe_unused mhi_pci_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + int err; + + pci_set_power_state(pdev, PCI_D0); + pci_restore_state(pdev); + pci_set_master(pdev); + + err = pci_enable_device(pdev); + if (err) + goto err_recovery; + + /* Exit M3, transition to M0 state */ + err = mhi_pm_resume(mhi_cntrl); + if (err) { + dev_err(&pdev->dev, "failed to resume device: %d\n", err); + goto err_recovery; + } + + /* Resume health check */ + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); + + return 0; + +err_recovery: + /* The device may have loose power or crashed, try recovering it */ + queue_work(system_long_wq, &mhi_pdev->recovery_work); + + return err; +} + +static const struct dev_pm_ops mhi_pci_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(mhi_pci_suspend, mhi_pci_resume) +}; + static struct pci_driver mhi_pci_driver = { .name = "mhi-pci-generic", .id_table = mhi_pci_id_table, .probe = mhi_pci_probe, - .remove = mhi_pci_remove + .remove = mhi_pci_remove, + .err_handler = &mhi_pci_err_handler, + .driver.pm = &mhi_pci_pm_ops }; module_pci_driver(mhi_pci_driver); diff --git a/drivers/char/random.c b/drivers/char/random.c index 84e24986a97a..0fe9e200e4c8 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1959,7 +1959,7 @@ static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg) return -EPERM; if (crng_init < 2) return -ENODATA; - crng_reseed(&primary_crng, NULL); + crng_reseed(&primary_crng, &input_pool); crng_global_init_time = jiffies - 1; return 0; default: diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c index 19e23fcc6bc8..ddaeceb7e109 100644 --- a/drivers/char/tpm/tpm-chip.c +++ b/drivers/char/tpm/tpm-chip.c @@ -278,8 +278,6 @@ static void tpm_devs_release(struct device *dev) { struct tpm_chip *chip = container_of(dev, struct tpm_chip, devs); - dump_stack(); - /* release the master device reference */ put_device(&chip->dev); } diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index 431919d5f48a..a2e0395cbe61 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -707,12 +707,22 @@ static int tpm_tis_gen_interrupt(struct tpm_chip *chip) const char *desc = "attempting to generate an interrupt"; u32 cap2; cap_t cap; + int ret; + /* TPM 2.0 */ if (chip->flags & TPM_CHIP_FLAG_TPM2) return tpm2_get_tpm_pt(chip, 0x100, &cap2, desc); - else - return tpm1_getcap(chip, TPM_CAP_PROP_TIS_TIMEOUT, &cap, desc, - 0); + + /* TPM 1.2 */ + ret = request_locality(chip, 0); + if (ret < 0) + return ret; + + ret = tpm1_getcap(chip, TPM_CAP_PROP_TIS_TIMEOUT, &cap, desc, 0); + + release_locality(chip, 0); + + return ret; } /* Register the IRQ and issue a command that will cause an interrupt. If an @@ -1019,11 +1029,21 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq, init_waitqueue_head(&priv->read_queue); init_waitqueue_head(&priv->int_queue); if (irq != -1) { - /* Before doing irq testing issue a command to the TPM in polling mode + /* + * Before doing irq testing issue a command to the TPM in polling mode * to make sure it works. May as well use that command to set the * proper timeouts for the driver. */ - if (tpm_get_timeouts(chip)) { + + rc = request_locality(chip, 0); + if (rc < 0) + goto out_err; + + rc = tpm_get_timeouts(chip); + + release_locality(chip, 0); + + if (rc) { dev_err(dev, "Could not get TPM timeouts and durations\n"); rc = -ENODEV; goto out_err; diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 4b47170dbe2e..a588d56502d4 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -369,6 +369,13 @@ config COMMON_CLK_FIXED_MMIO help Support for Memory Mapped IO Fixed clocks +config COMMON_CLK_K210 + bool "Clock driver for the Canaan Kendryte K210 SoC" + depends on OF && RISCV && SOC_CANAAN + default SOC_CANAAN + help + Support for the Canaan Kendryte K210 RISC-V SoC clocks. + source "drivers/clk/actions/Kconfig" source "drivers/clk/analogbits/Kconfig" source "drivers/clk/baikal-t1/Kconfig" diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index 71c1fa24b5f0..b22ae4f81e0b 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -36,6 +36,7 @@ obj-$(CONFIG_COMMON_CLK_ASPEED) += clk-aspeed.o obj-$(CONFIG_MACH_ASPEED_G6) += clk-ast2600.o obj-$(CONFIG_ARCH_HIGHBANK) += clk-highbank.o obj-$(CONFIG_CLK_HSDK) += clk-hsdk-pll.o +obj-$(CONFIG_COMMON_CLK_K210) += clk-k210.o obj-$(CONFIG_COMMON_CLK_LOCHNAGAR) += clk-lochnagar.o obj-$(CONFIG_COMMON_CLK_MAX77686) += clk-max77686.o obj-$(CONFIG_COMMON_CLK_MAX9485) += clk-max9485.o diff --git a/drivers/clk/clk-k210.c b/drivers/clk/clk-k210.c new file mode 100644 index 000000000000..6c84abf5b2e3 --- /dev/null +++ b/drivers/clk/clk-k210.c @@ -0,0 +1,1007 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2019-20 Sean Anderson <seanga2@gmail.com> + * Copyright (c) 2019 Western Digital Corporation or its affiliates. + */ +#define pr_fmt(fmt) "k210-clk: " fmt + +#include <linux/io.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/platform_device.h> +#include <linux/of.h> +#include <linux/of_clk.h> +#include <linux/of_platform.h> +#include <linux/of_address.h> +#include <linux/clk-provider.h> +#include <linux/bitfield.h> +#include <linux/delay.h> +#include <soc/canaan/k210-sysctl.h> + +#include <dt-bindings/clock/k210-clk.h> + +struct k210_sysclk; + +struct k210_clk { + int id; + struct k210_sysclk *ksc; + struct clk_hw hw; +}; + +struct k210_clk_cfg { + const char *name; + u8 gate_reg; + u8 gate_bit; + u8 div_reg; + u8 div_shift; + u8 div_width; + u8 div_type; + u8 mux_reg; + u8 mux_bit; +}; + +enum k210_clk_div_type { + K210_DIV_NONE, + K210_DIV_ONE_BASED, + K210_DIV_DOUBLE_ONE_BASED, + K210_DIV_POWER_OF_TWO, +}; + +#define K210_GATE(_reg, _bit) \ + .gate_reg = (_reg), \ + .gate_bit = (_bit) + +#define K210_DIV(_reg, _shift, _width, _type) \ + .div_reg = (_reg), \ + .div_shift = (_shift), \ + .div_width = (_width), \ + .div_type = (_type) + +#define K210_MUX(_reg, _bit) \ + .mux_reg = (_reg), \ + .mux_bit = (_bit) + +static struct k210_clk_cfg k210_clk_cfgs[K210_NUM_CLKS] = { + /* Gated clocks, no mux, no divider */ + [K210_CLK_CPU] = { + .name = "cpu", + K210_GATE(K210_SYSCTL_EN_CENT, 0) + }, + [K210_CLK_DMA] = { + .name = "dma", + K210_GATE(K210_SYSCTL_EN_PERI, 1) + }, + [K210_CLK_FFT] = { + .name = "fft", + K210_GATE(K210_SYSCTL_EN_PERI, 4) + }, + [K210_CLK_GPIO] = { + .name = "gpio", + K210_GATE(K210_SYSCTL_EN_PERI, 5) + }, + [K210_CLK_UART1] = { + .name = "uart1", + K210_GATE(K210_SYSCTL_EN_PERI, 16) + }, + [K210_CLK_UART2] = { + .name = "uart2", + K210_GATE(K210_SYSCTL_EN_PERI, 17) + }, + [K210_CLK_UART3] = { + .name = "uart3", + K210_GATE(K210_SYSCTL_EN_PERI, 18) + }, + [K210_CLK_FPIOA] = { + .name = "fpioa", + K210_GATE(K210_SYSCTL_EN_PERI, 20) + }, + [K210_CLK_SHA] = { + .name = "sha", + K210_GATE(K210_SYSCTL_EN_PERI, 26) + }, + [K210_CLK_AES] = { + .name = "aes", + K210_GATE(K210_SYSCTL_EN_PERI, 19) + }, + [K210_CLK_OTP] = { + .name = "otp", + K210_GATE(K210_SYSCTL_EN_PERI, 27) + }, + [K210_CLK_RTC] = { + .name = "rtc", + K210_GATE(K210_SYSCTL_EN_PERI, 29) + }, + + /* Gated divider clocks */ + [K210_CLK_SRAM0] = { + .name = "sram0", + K210_GATE(K210_SYSCTL_EN_CENT, 1), + K210_DIV(K210_SYSCTL_THR0, 0, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_SRAM1] = { + .name = "sram1", + K210_GATE(K210_SYSCTL_EN_CENT, 2), + K210_DIV(K210_SYSCTL_THR0, 4, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_ROM] = { + .name = "rom", + K210_GATE(K210_SYSCTL_EN_PERI, 0), + K210_DIV(K210_SYSCTL_THR0, 16, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_DVP] = { + .name = "dvp", + K210_GATE(K210_SYSCTL_EN_PERI, 3), + K210_DIV(K210_SYSCTL_THR0, 12, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_APB0] = { + .name = "apb0", + K210_GATE(K210_SYSCTL_EN_CENT, 3), + K210_DIV(K210_SYSCTL_SEL0, 3, 3, K210_DIV_ONE_BASED) + }, + [K210_CLK_APB1] = { + .name = "apb1", + K210_GATE(K210_SYSCTL_EN_CENT, 4), + K210_DIV(K210_SYSCTL_SEL0, 6, 3, K210_DIV_ONE_BASED) + }, + [K210_CLK_APB2] = { + .name = "apb2", + K210_GATE(K210_SYSCTL_EN_CENT, 5), + K210_DIV(K210_SYSCTL_SEL0, 9, 3, K210_DIV_ONE_BASED) + }, + [K210_CLK_AI] = { + .name = "ai", + K210_GATE(K210_SYSCTL_EN_PERI, 2), + K210_DIV(K210_SYSCTL_THR0, 8, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_SPI0] = { + .name = "spi0", + K210_GATE(K210_SYSCTL_EN_PERI, 6), + K210_DIV(K210_SYSCTL_THR1, 0, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_SPI1] = { + .name = "spi1", + K210_GATE(K210_SYSCTL_EN_PERI, 7), + K210_DIV(K210_SYSCTL_THR1, 8, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_SPI2] = { + .name = "spi2", + K210_GATE(K210_SYSCTL_EN_PERI, 8), + K210_DIV(K210_SYSCTL_THR1, 16, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2C0] = { + .name = "i2c0", + K210_GATE(K210_SYSCTL_EN_PERI, 13), + K210_DIV(K210_SYSCTL_THR5, 8, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2C1] = { + .name = "i2c1", + K210_GATE(K210_SYSCTL_EN_PERI, 14), + K210_DIV(K210_SYSCTL_THR5, 16, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2C2] = { + .name = "i2c2", + K210_GATE(K210_SYSCTL_EN_PERI, 15), + K210_DIV(K210_SYSCTL_THR5, 24, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_WDT0] = { + .name = "wdt0", + K210_GATE(K210_SYSCTL_EN_PERI, 24), + K210_DIV(K210_SYSCTL_THR6, 0, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_WDT1] = { + .name = "wdt1", + K210_GATE(K210_SYSCTL_EN_PERI, 25), + K210_DIV(K210_SYSCTL_THR6, 8, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S0] = { + .name = "i2s0", + K210_GATE(K210_SYSCTL_EN_PERI, 10), + K210_DIV(K210_SYSCTL_THR3, 0, 16, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S1] = { + .name = "i2s1", + K210_GATE(K210_SYSCTL_EN_PERI, 11), + K210_DIV(K210_SYSCTL_THR3, 16, 16, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S2] = { + .name = "i2s2", + K210_GATE(K210_SYSCTL_EN_PERI, 12), + K210_DIV(K210_SYSCTL_THR4, 0, 16, K210_DIV_DOUBLE_ONE_BASED) + }, + + /* Divider clocks, no gate, no mux */ + [K210_CLK_I2S0_M] = { + .name = "i2s0_m", + K210_DIV(K210_SYSCTL_THR4, 16, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S1_M] = { + .name = "i2s1_m", + K210_DIV(K210_SYSCTL_THR4, 24, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S2_M] = { + .name = "i2s2_m", + K210_DIV(K210_SYSCTL_THR4, 0, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + + /* Muxed gated divider clocks */ + [K210_CLK_SPI3] = { + .name = "spi3", + K210_GATE(K210_SYSCTL_EN_PERI, 9), + K210_DIV(K210_SYSCTL_THR1, 24, 8, K210_DIV_DOUBLE_ONE_BASED), + K210_MUX(K210_SYSCTL_SEL0, 12) + }, + [K210_CLK_TIMER0] = { + .name = "timer0", + K210_GATE(K210_SYSCTL_EN_PERI, 21), + K210_DIV(K210_SYSCTL_THR2, 0, 8, K210_DIV_DOUBLE_ONE_BASED), + K210_MUX(K210_SYSCTL_SEL0, 13) + }, + [K210_CLK_TIMER1] = { + .name = "timer1", + K210_GATE(K210_SYSCTL_EN_PERI, 22), + K210_DIV(K210_SYSCTL_THR2, 8, 8, K210_DIV_DOUBLE_ONE_BASED), + K210_MUX(K210_SYSCTL_SEL0, 14) + }, + [K210_CLK_TIMER2] = { + .name = "timer2", + K210_GATE(K210_SYSCTL_EN_PERI, 23), + K210_DIV(K210_SYSCTL_THR2, 16, 8, K210_DIV_DOUBLE_ONE_BASED), + K210_MUX(K210_SYSCTL_SEL0, 15) + }, +}; + +/* + * PLL control register bits. + */ +#define K210_PLL_CLKR GENMASK(3, 0) +#define K210_PLL_CLKF GENMASK(9, 4) +#define K210_PLL_CLKOD GENMASK(13, 10) +#define K210_PLL_BWADJ GENMASK(19, 14) +#define K210_PLL_RESET (1 << 20) +#define K210_PLL_PWRD (1 << 21) +#define K210_PLL_INTFB (1 << 22) +#define K210_PLL_BYPASS (1 << 23) +#define K210_PLL_TEST (1 << 24) +#define K210_PLL_EN (1 << 25) +#define K210_PLL_SEL GENMASK(27, 26) /* PLL2 only */ + +/* + * PLL lock register bits. + */ +#define K210_PLL_LOCK 0 +#define K210_PLL_CLEAR_SLIP 2 +#define K210_PLL_TEST_OUT 3 + +/* + * Clock selector register bits. + */ +#define K210_ACLK_SEL BIT(0) +#define K210_ACLK_DIV GENMASK(2, 1) + +/* + * PLLs. + */ +enum k210_pll_id { + K210_PLL0, K210_PLL1, K210_PLL2, K210_PLL_NUM +}; + +struct k210_pll { + enum k210_pll_id id; + struct k210_sysclk *ksc; + void __iomem *base; + void __iomem *reg; + void __iomem *lock; + u8 lock_shift; + u8 lock_width; + struct clk_hw hw; +}; +#define to_k210_pll(_hw) container_of(_hw, struct k210_pll, hw) + +/* + * PLLs configuration: by default PLL0 runs at 780 MHz and PLL1 at 299 MHz. + * The first 2 SRAM banks depend on ACLK/CPU clock which is by default PLL0 + * rate divided by 2. Set PLL1 to 390 MHz so that the third SRAM bank has the + * same clock as the first 2. + */ +struct k210_pll_cfg { + u32 reg; + u8 lock_shift; + u8 lock_width; + u32 r; + u32 f; + u32 od; + u32 bwadj; +}; + +static struct k210_pll_cfg k210_plls_cfg[] = { + { K210_SYSCTL_PLL0, 0, 2, 0, 59, 1, 59 }, /* 780 MHz */ + { K210_SYSCTL_PLL1, 8, 1, 0, 59, 3, 59 }, /* 390 MHz */ + { K210_SYSCTL_PLL2, 16, 1, 0, 22, 1, 22 }, /* 299 MHz */ +}; + +/** + * struct k210_sysclk - sysclk driver data + * @regs: system controller registers start address + * @clk_lock: clock setting spinlock + * @plls: SoC PLLs descriptors + * @aclk: ACLK clock + * @clks: All other clocks + */ +struct k210_sysclk { + void __iomem *regs; + spinlock_t clk_lock; + struct k210_pll plls[K210_PLL_NUM]; + struct clk_hw aclk; + struct k210_clk clks[K210_NUM_CLKS]; +}; + +#define to_k210_sysclk(_hw) container_of(_hw, struct k210_sysclk, aclk) + +/* + * Set ACLK parent selector: 0 for IN0, 1 for PLL0. + */ +static void k210_aclk_set_selector(void __iomem *regs, u8 sel) +{ + u32 reg = readl(regs + K210_SYSCTL_SEL0); + + if (sel) + reg |= K210_ACLK_SEL; + else + reg &= K210_ACLK_SEL; + writel(reg, regs + K210_SYSCTL_SEL0); +} + +static void k210_init_pll(void __iomem *regs, enum k210_pll_id pllid, + struct k210_pll *pll) +{ + pll->id = pllid; + pll->reg = regs + k210_plls_cfg[pllid].reg; + pll->lock = regs + K210_SYSCTL_PLL_LOCK; + pll->lock_shift = k210_plls_cfg[pllid].lock_shift; + pll->lock_width = k210_plls_cfg[pllid].lock_width; +} + +static void k210_pll_wait_for_lock(struct k210_pll *pll) +{ + u32 reg, mask = GENMASK(pll->lock_shift + pll->lock_width - 1, + pll->lock_shift); + + while (true) { + reg = readl(pll->lock); + if ((reg & mask) == mask) + break; + + reg |= BIT(pll->lock_shift + K210_PLL_CLEAR_SLIP); + writel(reg, pll->lock); + } +} + +static bool k210_pll_hw_is_enabled(struct k210_pll *pll) +{ + u32 reg = readl(pll->reg); + u32 mask = K210_PLL_PWRD | K210_PLL_EN; + + if (reg & K210_PLL_RESET) + return false; + + return (reg & mask) == mask; +} + +static void k210_pll_enable_hw(void __iomem *regs, struct k210_pll *pll) +{ + struct k210_pll_cfg *pll_cfg = &k210_plls_cfg[pll->id]; + u32 reg; + + if (k210_pll_hw_is_enabled(pll)) + return; + + /* + * For PLL0, we need to re-parent ACLK to IN0 to keep the CPU cores and + * SRAM running. + */ + if (pll->id == K210_PLL0) + k210_aclk_set_selector(regs, 0); + + /* Set PLL factors */ + reg = readl(pll->reg); + reg &= ~GENMASK(19, 0); + reg |= FIELD_PREP(K210_PLL_CLKR, pll_cfg->r); + reg |= FIELD_PREP(K210_PLL_CLKF, pll_cfg->f); + reg |= FIELD_PREP(K210_PLL_CLKOD, pll_cfg->od); + reg |= FIELD_PREP(K210_PLL_BWADJ, pll_cfg->bwadj); + reg |= K210_PLL_PWRD; + writel(reg, pll->reg); + + /* + * Reset the PLL: ensure reset is low before asserting it. + * The magic NOPs come from the Kendryte reference SDK. + */ + reg &= ~K210_PLL_RESET; + writel(reg, pll->reg); + reg |= K210_PLL_RESET; + writel(reg, pll->reg); + nop(); + nop(); + reg &= ~K210_PLL_RESET; + writel(reg, pll->reg); + + k210_pll_wait_for_lock(pll); + + reg &= ~K210_PLL_BYPASS; + reg |= K210_PLL_EN; + writel(reg, pll->reg); + + if (pll->id == K210_PLL0) + k210_aclk_set_selector(regs, 1); +} + +static int k210_pll_enable(struct clk_hw *hw) +{ + struct k210_pll *pll = to_k210_pll(hw); + struct k210_sysclk *ksc = pll->ksc; + unsigned long flags; + + spin_lock_irqsave(&ksc->clk_lock, flags); + + k210_pll_enable_hw(ksc->regs, pll); + + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static void k210_pll_disable(struct clk_hw *hw) +{ + struct k210_pll *pll = to_k210_pll(hw); + struct k210_sysclk *ksc = pll->ksc; + unsigned long flags; + u32 reg; + + /* + * Bypassing before powering off is important so child clocks do not + * stop working. This is especially important for pll0, the indirect + * parent of the cpu clock. + */ + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(pll->reg); + reg |= K210_PLL_BYPASS; + writel(reg, pll->reg); + + reg &= ~K210_PLL_PWRD; + reg &= ~K210_PLL_EN; + writel(reg, pll->reg); + spin_unlock_irqrestore(&ksc->clk_lock, flags); +} + +static int k210_pll_is_enabled(struct clk_hw *hw) +{ + return k210_pll_hw_is_enabled(to_k210_pll(hw)); +} + +static unsigned long k210_pll_get_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct k210_pll *pll = to_k210_pll(hw); + u32 reg = readl(pll->reg); + u32 r, f, od; + + if (reg & K210_PLL_BYPASS) + return parent_rate; + + if (!(reg & K210_PLL_PWRD)) + return 0; + + r = FIELD_GET(K210_PLL_CLKR, reg) + 1; + f = FIELD_GET(K210_PLL_CLKF, reg) + 1; + od = FIELD_GET(K210_PLL_CLKOD, reg) + 1; + + return (u64)parent_rate * f / (r * od); +} + +static const struct clk_ops k210_pll_ops = { + .enable = k210_pll_enable, + .disable = k210_pll_disable, + .is_enabled = k210_pll_is_enabled, + .recalc_rate = k210_pll_get_rate, +}; + +static int k210_pll2_set_parent(struct clk_hw *hw, u8 index) +{ + struct k210_pll *pll = to_k210_pll(hw); + struct k210_sysclk *ksc = pll->ksc; + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&ksc->clk_lock, flags); + + reg = readl(pll->reg); + reg &= ~K210_PLL_SEL; + reg |= FIELD_PREP(K210_PLL_SEL, index); + writel(reg, pll->reg); + + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static u8 k210_pll2_get_parent(struct clk_hw *hw) +{ + struct k210_pll *pll = to_k210_pll(hw); + u32 reg = readl(pll->reg); + + return FIELD_GET(K210_PLL_SEL, reg); +} + +static const struct clk_ops k210_pll2_ops = { + .enable = k210_pll_enable, + .disable = k210_pll_disable, + .is_enabled = k210_pll_is_enabled, + .recalc_rate = k210_pll_get_rate, + .set_parent = k210_pll2_set_parent, + .get_parent = k210_pll2_get_parent, +}; + +static int __init k210_register_pll(struct device_node *np, + struct k210_sysclk *ksc, + enum k210_pll_id pllid, const char *name, + int num_parents, const struct clk_ops *ops) +{ + struct k210_pll *pll = &ksc->plls[pllid]; + struct clk_init_data init = {}; + const struct clk_parent_data parent_data[] = { + { /* .index = 0 for in0 */ }, + { .hw = &ksc->plls[K210_PLL0].hw }, + { .hw = &ksc->plls[K210_PLL1].hw }, + }; + + init.name = name; + init.parent_data = parent_data; + init.num_parents = num_parents; + init.ops = ops; + + pll->hw.init = &init; + pll->ksc = ksc; + + return of_clk_hw_register(np, &pll->hw); +} + +static int __init k210_register_plls(struct device_node *np, + struct k210_sysclk *ksc) +{ + int i, ret; + + for (i = 0; i < K210_PLL_NUM; i++) + k210_init_pll(ksc->regs, i, &ksc->plls[i]); + + /* PLL0 and PLL1 only have IN0 as parent */ + ret = k210_register_pll(np, ksc, K210_PLL0, "pll0", 1, &k210_pll_ops); + if (ret) { + pr_err("%pOFP: register PLL0 failed\n", np); + return ret; + } + ret = k210_register_pll(np, ksc, K210_PLL1, "pll1", 1, &k210_pll_ops); + if (ret) { + pr_err("%pOFP: register PLL1 failed\n", np); + return ret; + } + + /* PLL2 has IN0, PLL0 and PLL1 as parents */ + ret = k210_register_pll(np, ksc, K210_PLL2, "pll2", 3, &k210_pll2_ops); + if (ret) { + pr_err("%pOFP: register PLL2 failed\n", np); + return ret; + } + + return 0; +} + +static int k210_aclk_set_parent(struct clk_hw *hw, u8 index) +{ + struct k210_sysclk *ksc = to_k210_sysclk(hw); + unsigned long flags; + + spin_lock_irqsave(&ksc->clk_lock, flags); + + k210_aclk_set_selector(ksc->regs, index); + + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static u8 k210_aclk_get_parent(struct clk_hw *hw) +{ + struct k210_sysclk *ksc = to_k210_sysclk(hw); + u32 sel; + + sel = readl(ksc->regs + K210_SYSCTL_SEL0) & K210_ACLK_SEL; + + return sel ? 1 : 0; +} + +static unsigned long k210_aclk_get_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct k210_sysclk *ksc = to_k210_sysclk(hw); + u32 reg = readl(ksc->regs + K210_SYSCTL_SEL0); + unsigned int shift; + + if (!(reg & 0x1)) + return parent_rate; + + shift = FIELD_GET(K210_ACLK_DIV, reg); + + return parent_rate / (2UL << shift); +} + +static const struct clk_ops k210_aclk_ops = { + .set_parent = k210_aclk_set_parent, + .get_parent = k210_aclk_get_parent, + .recalc_rate = k210_aclk_get_rate, +}; + +/* + * ACLK has IN0 and PLL0 as parents. + */ +static int __init k210_register_aclk(struct device_node *np, + struct k210_sysclk *ksc) +{ + struct clk_init_data init = {}; + const struct clk_parent_data parent_data[] = { + { /* .index = 0 for in0 */ }, + { .hw = &ksc->plls[K210_PLL0].hw }, + }; + int ret; + + init.name = "aclk"; + init.parent_data = parent_data; + init.num_parents = 2; + init.ops = &k210_aclk_ops; + ksc->aclk.init = &init; + + ret = of_clk_hw_register(np, &ksc->aclk); + if (ret) { + pr_err("%pOFP: register aclk failed\n", np); + return ret; + } + + return 0; +} + +#define to_k210_clk(_hw) container_of(_hw, struct k210_clk, hw) + +static int k210_clk_enable(struct clk_hw *hw) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + unsigned long flags; + u32 reg; + + if (!cfg->gate_reg) + return 0; + + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(ksc->regs + cfg->gate_reg); + reg |= BIT(cfg->gate_bit); + writel(reg, ksc->regs + cfg->gate_reg); + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static void k210_clk_disable(struct clk_hw *hw) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + unsigned long flags; + u32 reg; + + if (!cfg->gate_reg) + return; + + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(ksc->regs + cfg->gate_reg); + reg &= ~BIT(cfg->gate_bit); + writel(reg, ksc->regs + cfg->gate_reg); + spin_unlock_irqrestore(&ksc->clk_lock, flags); +} + +static int k210_clk_set_parent(struct clk_hw *hw, u8 index) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(ksc->regs + cfg->mux_reg); + if (index) + reg |= BIT(cfg->mux_bit); + else + reg &= ~BIT(cfg->mux_bit); + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static u8 k210_clk_get_parent(struct clk_hw *hw) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + unsigned long flags; + u32 reg, idx; + + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(ksc->regs + cfg->mux_reg); + idx = (reg & BIT(cfg->mux_bit)) ? 1 : 0; + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return idx; +} + +static unsigned long k210_clk_get_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + u32 reg, div_val; + + if (!cfg->div_reg) + return parent_rate; + + reg = readl(ksc->regs + cfg->div_reg); + div_val = (reg >> cfg->div_shift) & GENMASK(cfg->div_width - 1, 0); + + switch (cfg->div_type) { + case K210_DIV_ONE_BASED: + return parent_rate / (div_val + 1); + case K210_DIV_DOUBLE_ONE_BASED: + return parent_rate / ((div_val + 1) * 2); + case K210_DIV_POWER_OF_TWO: + return parent_rate / (2UL << div_val); + case K210_DIV_NONE: + default: + return 0; + } +} + +static const struct clk_ops k210_clk_mux_ops = { + .enable = k210_clk_enable, + .disable = k210_clk_disable, + .set_parent = k210_clk_set_parent, + .get_parent = k210_clk_get_parent, + .recalc_rate = k210_clk_get_rate, +}; + +static const struct clk_ops k210_clk_ops = { + .enable = k210_clk_enable, + .disable = k210_clk_disable, + .recalc_rate = k210_clk_get_rate, +}; + +static void __init k210_register_clk(struct device_node *np, + struct k210_sysclk *ksc, int id, + const struct clk_parent_data *parent_data, + int num_parents, unsigned long flags) +{ + struct k210_clk *kclk = &ksc->clks[id]; + struct clk_init_data init = {}; + int ret; + + init.name = k210_clk_cfgs[id].name; + init.flags = flags; + init.parent_data = parent_data; + init.num_parents = num_parents; + if (num_parents > 1) + init.ops = &k210_clk_mux_ops; + else + init.ops = &k210_clk_ops; + + kclk->id = id; + kclk->ksc = ksc; + kclk->hw.init = &init; + + ret = of_clk_hw_register(np, &kclk->hw); + if (ret) { + pr_err("%pOFP: register clock %s failed\n", + np, k210_clk_cfgs[id].name); + kclk->id = -1; + } +} + +/* + * All muxed clocks have IN0 and PLL0 as parents. + */ +static inline void __init k210_register_mux_clk(struct device_node *np, + struct k210_sysclk *ksc, int id) +{ + const struct clk_parent_data parent_data[2] = { + { /* .index = 0 for in0 */ }, + { .hw = &ksc->plls[K210_PLL0].hw } + }; + + k210_register_clk(np, ksc, id, parent_data, 2, 0); +} + +static inline void __init k210_register_in0_child(struct device_node *np, + struct k210_sysclk *ksc, int id) +{ + const struct clk_parent_data parent_data = { + /* .index = 0 for in0 */ + }; + + k210_register_clk(np, ksc, id, &parent_data, 1, 0); +} + +static inline void __init k210_register_pll_child(struct device_node *np, + struct k210_sysclk *ksc, int id, + enum k210_pll_id pllid, + unsigned long flags) +{ + const struct clk_parent_data parent_data = { + .hw = &ksc->plls[pllid].hw, + }; + + k210_register_clk(np, ksc, id, &parent_data, 1, flags); +} + +static inline void __init k210_register_aclk_child(struct device_node *np, + struct k210_sysclk *ksc, int id, + unsigned long flags) +{ + const struct clk_parent_data parent_data = { + .hw = &ksc->aclk, + }; + + k210_register_clk(np, ksc, id, &parent_data, 1, flags); +} + +static inline void __init k210_register_clk_child(struct device_node *np, + struct k210_sysclk *ksc, int id, + int parent_id) +{ + const struct clk_parent_data parent_data = { + .hw = &ksc->clks[parent_id].hw, + }; + + k210_register_clk(np, ksc, id, &parent_data, 1, 0); +} + +static struct clk_hw *k210_clk_hw_onecell_get(struct of_phandle_args *clkspec, + void *data) +{ + struct k210_sysclk *ksc = data; + unsigned int idx = clkspec->args[0]; + + if (idx >= K210_NUM_CLKS) + return ERR_PTR(-EINVAL); + + return &ksc->clks[idx].hw; +} + +static void __init k210_clk_init(struct device_node *np) +{ + struct device_node *sysctl_np; + struct k210_sysclk *ksc; + int i, ret; + + ksc = kzalloc(sizeof(*ksc), GFP_KERNEL); + if (!ksc) + return; + + spin_lock_init(&ksc->clk_lock); + sysctl_np = of_get_parent(np); + ksc->regs = of_iomap(sysctl_np, 0); + of_node_put(sysctl_np); + if (!ksc->regs) { + pr_err("%pOFP: failed to map registers\n", np); + return; + } + + ret = k210_register_plls(np, ksc); + if (ret) + return; + + ret = k210_register_aclk(np, ksc); + if (ret) + return; + + /* + * Critical clocks: there are no consumers of the SRAM clocks, + * including the AI clock for the third SRAM bank. The CPU clock + * is only referenced by the uarths serial device and so would be + * disabled if the serial console is disabled to switch to another + * console. Mark all these clocks as critical so that they are never + * disabled by the core clock management. + */ + k210_register_aclk_child(np, ksc, K210_CLK_CPU, CLK_IS_CRITICAL); + k210_register_aclk_child(np, ksc, K210_CLK_SRAM0, CLK_IS_CRITICAL); + k210_register_aclk_child(np, ksc, K210_CLK_SRAM1, CLK_IS_CRITICAL); + k210_register_pll_child(np, ksc, K210_CLK_AI, K210_PLL1, + CLK_IS_CRITICAL); + + /* Clocks with aclk as source */ + k210_register_aclk_child(np, ksc, K210_CLK_DMA, 0); + k210_register_aclk_child(np, ksc, K210_CLK_FFT, 0); + k210_register_aclk_child(np, ksc, K210_CLK_ROM, 0); + k210_register_aclk_child(np, ksc, K210_CLK_DVP, 0); + k210_register_aclk_child(np, ksc, K210_CLK_APB0, 0); + k210_register_aclk_child(np, ksc, K210_CLK_APB1, 0); + k210_register_aclk_child(np, ksc, K210_CLK_APB2, 0); + + /* Clocks with PLL0 as source */ + k210_register_pll_child(np, ksc, K210_CLK_SPI0, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_SPI1, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_SPI2, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2C0, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2C1, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2C2, K210_PLL0, 0); + + /* Clocks with PLL2 as source */ + k210_register_pll_child(np, ksc, K210_CLK_I2S0, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S1, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S2, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S0_M, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S1_M, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S2_M, K210_PLL2, 0); + + /* Clocks with IN0 as source */ + k210_register_in0_child(np, ksc, K210_CLK_WDT0); + k210_register_in0_child(np, ksc, K210_CLK_WDT1); + k210_register_in0_child(np, ksc, K210_CLK_RTC); + + /* Clocks with APB0 as source */ + k210_register_clk_child(np, ksc, K210_CLK_GPIO, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_UART1, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_UART2, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_UART3, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_FPIOA, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_SHA, K210_CLK_APB0); + + /* Clocks with APB1 as source */ + k210_register_clk_child(np, ksc, K210_CLK_AES, K210_CLK_APB1); + k210_register_clk_child(np, ksc, K210_CLK_OTP, K210_CLK_APB1); + + /* Mux clocks with in0 or pll0 as source */ + k210_register_mux_clk(np, ksc, K210_CLK_SPI3); + k210_register_mux_clk(np, ksc, K210_CLK_TIMER0); + k210_register_mux_clk(np, ksc, K210_CLK_TIMER1); + k210_register_mux_clk(np, ksc, K210_CLK_TIMER2); + + /* Check for registration errors */ + for (i = 0; i < K210_NUM_CLKS; i++) { + if (ksc->clks[i].id != i) + return; + } + + ret = of_clk_add_hw_provider(np, k210_clk_hw_onecell_get, ksc); + if (ret) { + pr_err("%pOFP: add clock provider failed %d\n", np, ret); + return; + } + + pr_info("%pOFP: CPU running at %lu MHz\n", + np, clk_hw_get_rate(&ksc->clks[K210_CLK_CPU].hw) / 1000000); +} + +CLK_OF_DECLARE(k210_clk, "canaan,k210-clk", k210_clk_init); + +/* + * Enable PLL1 to be able to use the AI SRAM. + */ +void __init k210_clk_early_init(void __iomem *regs) +{ + struct k210_pll pll1; + + /* Make sure ACLK selector is set to PLL0 */ + k210_aclk_set_selector(regs, 1); + + /* Startup PLL1 to enable the aisram bank for general memory use */ + k210_init_pll(regs, K210_PLL1, &pll1); + k210_pll_enable_hw(regs, &pll1); +} diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 3d751ae5bc70..5052541a0986 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -4576,6 +4576,8 @@ int of_clk_add_provider(struct device_node *np, if (ret < 0) of_clk_del_provider(np); + fwnode_dev_initialized(&np->fwnode, true); + return ret; } EXPORT_SYMBOL_GPL(of_clk_add_provider); @@ -4693,6 +4695,7 @@ void of_clk_del_provider(struct device_node *np) list_for_each_entry(cp, &of_clk_providers, link) { if (cp->node == np) { list_del(&cp->link); + fwnode_dev_initialized(&np->fwnode, false); of_node_put(cp->node); kfree(cp); break; diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index 399526289320..92701a18bdd9 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -62,16 +62,6 @@ config X86_ACPI_CPUFREQ_CPB By enabling this option the acpi_cpufreq driver provides the old entry in addition to the new boost ones, for compatibility reasons. -config X86_SFI_CPUFREQ - tristate "SFI Performance-States driver" - depends on X86_INTEL_MID && SFI - help - This adds a CPUFreq driver for some Silvermont based Intel Atom - architectures like Z34xx and Z35xx which enumerate processor - performance states through SFI. - - If in doubt, say N. - config ELAN_CPUFREQ tristate "AMD Elan SC400 and SC410" depends on MELAN diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 1ab9b1536304..27d3bd7ea9d4 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -43,7 +43,6 @@ obj-$(CONFIG_X86_P4_CLOCKMOD) += p4-clockmod.o obj-$(CONFIG_X86_CPUFREQ_NFORCE2) += cpufreq-nforce2.o obj-$(CONFIG_X86_INTEL_PSTATE) += intel_pstate.o obj-$(CONFIG_X86_AMD_FREQ_SENSITIVITY) += amd_freq_sensitivity.o -obj-$(CONFIG_X86_SFI_CPUFREQ) += sfi-cpufreq.o ################################################################################## # ARM SoC drivers diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index d3e5a6fceb61..d1bbc16fba4b 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -54,7 +54,6 @@ struct acpi_cpufreq_data { unsigned int resume; unsigned int cpu_feature; unsigned int acpi_perf_cpu; - unsigned int first_perf_state; cpumask_var_t freqdomain_cpus; void (*cpu_freq_write)(struct acpi_pct_register *reg, u32 val); u32 (*cpu_freq_read)(struct acpi_pct_register *reg); @@ -223,10 +222,10 @@ static unsigned extract_msr(struct cpufreq_policy *policy, u32 msr) perf = to_perf_data(data); - cpufreq_for_each_entry(pos, policy->freq_table + data->first_perf_state) + cpufreq_for_each_entry(pos, policy->freq_table) if (msr == perf->states[pos->driver_data].status) return pos->frequency; - return policy->freq_table[data->first_perf_state].frequency; + return policy->freq_table[0].frequency; } static unsigned extract_freq(struct cpufreq_policy *policy, u32 val) @@ -365,7 +364,6 @@ static unsigned int get_cur_freq_on_cpu(unsigned int cpu) struct cpufreq_policy *policy; unsigned int freq; unsigned int cached_freq; - unsigned int state; pr_debug("%s (%d)\n", __func__, cpu); @@ -377,11 +375,7 @@ static unsigned int get_cur_freq_on_cpu(unsigned int cpu) if (unlikely(!data || !policy->freq_table)) return 0; - state = to_perf_data(data)->state; - if (state < data->first_perf_state) - state = data->first_perf_state; - - cached_freq = policy->freq_table[state].frequency; + cached_freq = policy->freq_table[to_perf_data(data)->state].frequency; freq = extract_freq(policy, get_cur_val(cpumask_of(cpu), data)); if (freq != cached_freq) { /* @@ -680,7 +674,6 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) struct cpuinfo_x86 *c = &cpu_data(cpu); unsigned int valid_states = 0; unsigned int result = 0; - unsigned int state_count; u64 max_boost_ratio; unsigned int i; #ifdef CONFIG_SMP @@ -795,28 +788,8 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) goto err_unreg; } - state_count = perf->state_count + 1; - - max_boost_ratio = get_max_boost_ratio(cpu); - if (max_boost_ratio) { - /* - * Make a room for one more entry to represent the highest - * available "boost" frequency. - */ - state_count++; - valid_states++; - data->first_perf_state = valid_states; - } else { - /* - * If the maximum "boost" frequency is unknown, ask the arch - * scale-invariance code to use the "nominal" performance for - * CPU utilization scaling so as to prevent the schedutil - * governor from selecting inadequate CPU frequencies. - */ - arch_set_max_freq_ratio(true); - } - - freq_table = kcalloc(state_count, sizeof(*freq_table), GFP_KERNEL); + freq_table = kcalloc(perf->state_count + 1, sizeof(*freq_table), + GFP_KERNEL); if (!freq_table) { result = -ENOMEM; goto err_unreg; @@ -851,27 +824,25 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) } freq_table[valid_states].frequency = CPUFREQ_TABLE_END; + max_boost_ratio = get_max_boost_ratio(cpu); if (max_boost_ratio) { - unsigned int state = data->first_perf_state; - unsigned int freq = freq_table[state].frequency; + unsigned int freq = freq_table[0].frequency; /* * Because the loop above sorts the freq_table entries in the * descending order, freq is the maximum frequency in the table. * Assume that it corresponds to the CPPC nominal frequency and - * use it to populate the frequency field of the extra "boost" - * frequency entry. + * use it to set cpuinfo.max_freq. */ - freq_table[0].frequency = freq * max_boost_ratio >> SCHED_CAPACITY_SHIFT; + policy->cpuinfo.max_freq = freq * max_boost_ratio >> SCHED_CAPACITY_SHIFT; + } else { /* - * The purpose of the extra "boost" frequency entry is to make - * the rest of cpufreq aware of the real maximum frequency, but - * the way to request it is the same as for the first_perf_state - * entry that is expected to cover the entire range of "boost" - * frequencies of the CPU, so copy the driver_data value from - * that entry. + * If the maximum "boost" frequency is unknown, ask the arch + * scale-invariance code to use the "nominal" performance for + * CPU utilization scaling so as to prevent the schedutil + * governor from selecting inadequate CPU frequencies. */ - freq_table[0].driver_data = freq_table[state].driver_data; + arch_set_max_freq_ratio(true); } policy->freq_table = freq_table; @@ -947,8 +918,7 @@ static void acpi_cpufreq_cpu_ready(struct cpufreq_policy *policy) { struct acpi_processor_performance *perf = per_cpu_ptr(acpi_perf_data, policy->cpu); - struct acpi_cpufreq_data *data = policy->driver_data; - unsigned int freq = policy->freq_table[data->first_perf_state].frequency; + unsigned int freq = policy->freq_table[0].frequency; if (perf->states[0].core_frequency * 1000 != freq) pr_warn(FW_WARN "P-state 0 is not max freq\n"); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 7d0ae968def7..1d1b563cea4b 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2101,7 +2101,7 @@ EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go. * @cpu: Target CPU. * @min_perf: Minimum (required) performance level (units of @capacity). - * @target_perf: Terget (desired) performance level (units of @capacity). + * @target_perf: Target (desired) performance level (units of @capacity). * @capacity: Capacity of the target CPU. * * Carry out a fast performance level switch of @cpu without sleeping. diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c index f839dc9852c0..d3f756f7b5a0 100644 --- a/drivers/cpufreq/freq_table.c +++ b/drivers/cpufreq/freq_table.c @@ -52,7 +52,13 @@ int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy, } policy->min = policy->cpuinfo.min_freq = min_freq; - policy->max = policy->cpuinfo.max_freq = max_freq; + policy->max = max_freq; + /* + * If the driver has set its own cpuinfo.max_freq above max_freq, leave + * it as is. + */ + if (policy->cpuinfo.max_freq < max_freq) + policy->max = policy->cpuinfo.max_freq = max_freq; if (policy->min == ~0) return -EINVAL; diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 356244510b18..d3c23447b892 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -32,6 +32,7 @@ struct qcom_cpufreq_soc_data { struct qcom_cpufreq_data { void __iomem *base; + struct resource *res; const struct qcom_cpufreq_soc_data *soc_data; }; @@ -280,6 +281,7 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) struct of_phandle_args args; struct device_node *cpu_np; struct device *cpu_dev; + struct resource *res; void __iomem *base; struct qcom_cpufreq_data *data; int ret, index; @@ -303,18 +305,33 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) index = args.args[0]; - base = devm_platform_ioremap_resource(pdev, index); - if (IS_ERR(base)) - return PTR_ERR(base); + res = platform_get_resource(pdev, IORESOURCE_MEM, index); + if (!res) { + dev_err(dev, "failed to get mem resource %d\n", index); + return -ENODEV; + } + + if (!request_mem_region(res->start, resource_size(res), res->name)) { + dev_err(dev, "failed to request resource %pR\n", res); + return -EBUSY; + } - data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + base = ioremap(res->start, resource_size(res)); + if (IS_ERR(base)) { + dev_err(dev, "failed to map resource %pR\n", res); + ret = PTR_ERR(base); + goto release_region; + } + + data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) { ret = -ENOMEM; - goto error; + goto unmap_base; } data->soc_data = of_device_get_match_data(&pdev->dev); data->base = base; + data->res = res; /* HW should be in enabled state to proceed */ if (!(readl_relaxed(base + data->soc_data->reg_enable) & 0x1)) { @@ -355,7 +372,11 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) return 0; error: - devm_iounmap(dev, base); + kfree(data); +unmap_base: + iounmap(data->base); +release_region: + release_mem_region(res->start, resource_size(res)); return ret; } @@ -363,12 +384,15 @@ static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) { struct device *cpu_dev = get_cpu_device(policy->cpu); struct qcom_cpufreq_data *data = policy->driver_data; - struct platform_device *pdev = cpufreq_get_driver_data(); + struct resource *res = data->res; + void __iomem *base = data->base; dev_pm_opp_remove_all_dynamic(cpu_dev); dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); kfree(policy->freq_table); - devm_iounmap(&pdev->dev, data->base); + kfree(data); + iounmap(base); + release_mem_region(res->start, resource_size(res)); return 0; } diff --git a/drivers/cpufreq/sfi-cpufreq.c b/drivers/cpufreq/sfi-cpufreq.c deleted file mode 100644 index 45cfdf67cf03..000000000000 --- a/drivers/cpufreq/sfi-cpufreq.c +++ /dev/null @@ -1,127 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * SFI Performance States Driver - * - * Author: Vishwesh M Rudramuni <vishwesh.m.rudramuni@intel.com> - * Author: Srinidhi Kasagar <srinidhi.kasagar@intel.com> - */ - -#include <linux/cpufreq.h> -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/sfi.h> -#include <linux/slab.h> -#include <linux/smp.h> - -#include <asm/msr.h> - -static struct cpufreq_frequency_table *freq_table; -static struct sfi_freq_table_entry *sfi_cpufreq_array; -static int num_freq_table_entries; - -static int sfi_parse_freq(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_freq_table_entry *pentry; - int totallen; - - sb = (struct sfi_table_simple *)table; - num_freq_table_entries = SFI_GET_NUM_ENTRIES(sb, - struct sfi_freq_table_entry); - if (num_freq_table_entries <= 1) { - pr_err("No p-states discovered\n"); - return -ENODEV; - } - - pentry = (struct sfi_freq_table_entry *)sb->pentry; - totallen = num_freq_table_entries * sizeof(*pentry); - - sfi_cpufreq_array = kmemdup(pentry, totallen, GFP_KERNEL); - if (!sfi_cpufreq_array) - return -ENOMEM; - - return 0; -} - -static int sfi_cpufreq_target(struct cpufreq_policy *policy, unsigned int index) -{ - unsigned int next_perf_state = 0; /* Index into perf table */ - u32 lo, hi; - - next_perf_state = policy->freq_table[index].driver_data; - - rdmsr_on_cpu(policy->cpu, MSR_IA32_PERF_CTL, &lo, &hi); - lo = (lo & ~INTEL_PERF_CTL_MASK) | - ((u32) sfi_cpufreq_array[next_perf_state].ctrl_val & - INTEL_PERF_CTL_MASK); - wrmsr_on_cpu(policy->cpu, MSR_IA32_PERF_CTL, lo, hi); - - return 0; -} - -static int sfi_cpufreq_cpu_init(struct cpufreq_policy *policy) -{ - policy->shared_type = CPUFREQ_SHARED_TYPE_HW; - policy->cpuinfo.transition_latency = 100000; /* 100us */ - policy->freq_table = freq_table; - - return 0; -} - -static struct cpufreq_driver sfi_cpufreq_driver = { - .flags = CPUFREQ_CONST_LOOPS, - .verify = cpufreq_generic_frequency_table_verify, - .target_index = sfi_cpufreq_target, - .init = sfi_cpufreq_cpu_init, - .name = "sfi-cpufreq", - .attr = cpufreq_generic_attr, -}; - -static int __init sfi_cpufreq_init(void) -{ - int ret, i; - - /* parse the freq table from SFI */ - ret = sfi_table_parse(SFI_SIG_FREQ, NULL, NULL, sfi_parse_freq); - if (ret) - return ret; - - freq_table = kcalloc(num_freq_table_entries + 1, sizeof(*freq_table), - GFP_KERNEL); - if (!freq_table) { - ret = -ENOMEM; - goto err_free_array; - } - - for (i = 0; i < num_freq_table_entries; i++) { - freq_table[i].driver_data = i; - freq_table[i].frequency = sfi_cpufreq_array[i].freq_mhz * 1000; - } - freq_table[i].frequency = CPUFREQ_TABLE_END; - - ret = cpufreq_register_driver(&sfi_cpufreq_driver); - if (ret) - goto err_free_tbl; - - return ret; - -err_free_tbl: - kfree(freq_table); -err_free_array: - kfree(sfi_cpufreq_array); - return ret; -} -late_initcall(sfi_cpufreq_init); - -static void __exit sfi_cpufreq_exit(void) -{ - cpufreq_unregister_driver(&sfi_cpufreq_driver); - kfree(freq_table); - kfree(sfi_cpufreq_array); -} -module_exit(sfi_cpufreq_exit); - -MODULE_AUTHOR("Vishwesh M Rudramuni <vishwesh.m.rudramuni@intel.com>"); -MODULE_DESCRIPTION("SFI Performance-States Driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig new file mode 100644 index 000000000000..97dc4d751651 --- /dev/null +++ b/drivers/cxl/Kconfig @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: GPL-2.0-only +menuconfig CXL_BUS + tristate "CXL (Compute Express Link) Devices Support" + depends on PCI + help + CXL is a bus that is electrically compatible with PCI Express, but + layers three protocols on that signalling (CXL.io, CXL.cache, and + CXL.mem). The CXL.cache protocol allows devices to hold cachelines + locally, the CXL.mem protocol allows devices to be fully coherent + memory targets, the CXL.io protocol is equivalent to PCI Express. + Say 'y' to enable support for the configuration and management of + devices supporting these protocols. + +if CXL_BUS + +config CXL_MEM + tristate "CXL.mem: Memory Devices" + help + The CXL.mem protocol allows a device to act as a provider of + "System RAM" and/or "Persistent Memory" that is fully coherent + as if the memory was attached to the typical CPU memory + controller. + + Say 'y/m' to enable a driver (named "cxl_mem.ko" when built as + a module) that will attach to CXL.mem devices for + configuration, provisioning, and health monitoring. This + driver is required for dynamic provisioning of CXL.mem + attached memory which is a prerequisite for persistent memory + support. Typically volatile memory is mapped by platform + firmware and included in the platform memory map, but in some + cases the OS is responsible for mapping that memory. See + Chapter 2.3 Type 3 CXL Device in the CXL 2.0 specification. + + If unsure say 'm'. + +config CXL_MEM_RAW_COMMANDS + bool "RAW Command Interface for Memory Devices" + depends on CXL_MEM + help + Enable CXL RAW command interface. + + The CXL driver ioctl interface may assign a kernel ioctl command + number for each specification defined opcode. At any given point in + time the number of opcodes that the specification defines and a device + may implement may exceed the kernel's set of associated ioctl function + numbers. The mismatch is either by omission, specification is too new, + or by design. When prototyping new hardware, or developing / debugging + the driver it is useful to be able to submit any possible command to + the hardware, even commands that may crash the kernel due to their + potential impact to memory currently in use by the kernel. + + If developing CXL hardware or the driver say Y, otherwise say N. +endif diff --git a/drivers/cxl/Makefile b/drivers/cxl/Makefile new file mode 100644 index 000000000000..a314a1891f4d --- /dev/null +++ b/drivers/cxl/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CXL_BUS) += cxl_bus.o +obj-$(CONFIG_CXL_MEM) += cxl_mem.o + +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=CXL +cxl_bus-y := bus.o +cxl_mem-y := mem.o diff --git a/drivers/cxl/bus.c b/drivers/cxl/bus.c new file mode 100644 index 000000000000..58f74796d525 --- /dev/null +++ b/drivers/cxl/bus.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#include <linux/device.h> +#include <linux/module.h> + +/** + * DOC: cxl bus + * + * The CXL bus provides namespace for control devices and a rendezvous + * point for cross-device interleave coordination. + */ +struct bus_type cxl_bus_type = { + .name = "cxl", +}; +EXPORT_SYMBOL_GPL(cxl_bus_type); + +static __init int cxl_bus_init(void) +{ + return bus_register(&cxl_bus_type); +} + +static void cxl_bus_exit(void) +{ + bus_unregister(&cxl_bus_type); +} + +module_init(cxl_bus_init); +module_exit(cxl_bus_exit); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h new file mode 100644 index 000000000000..6f14838c2d25 --- /dev/null +++ b/drivers/cxl/cxl.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2020 Intel Corporation. */ + +#ifndef __CXL_H__ +#define __CXL_H__ + +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/io.h> + +/* CXL 2.0 8.2.8.1 Device Capabilities Array Register */ +#define CXLDEV_CAP_ARRAY_OFFSET 0x0 +#define CXLDEV_CAP_ARRAY_CAP_ID 0 +#define CXLDEV_CAP_ARRAY_ID_MASK GENMASK_ULL(15, 0) +#define CXLDEV_CAP_ARRAY_COUNT_MASK GENMASK_ULL(47, 32) +/* CXL 2.0 8.2.8.2 CXL Device Capability Header Register */ +#define CXLDEV_CAP_HDR_CAP_ID_MASK GENMASK(15, 0) +/* CXL 2.0 8.2.8.2.1 CXL Device Capabilities */ +#define CXLDEV_CAP_CAP_ID_DEVICE_STATUS 0x1 +#define CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX 0x2 +#define CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX 0x3 +#define CXLDEV_CAP_CAP_ID_MEMDEV 0x4000 + +/* CXL 2.0 8.2.8.4 Mailbox Registers */ +#define CXLDEV_MBOX_CAPS_OFFSET 0x00 +#define CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK GENMASK(4, 0) +#define CXLDEV_MBOX_CTRL_OFFSET 0x04 +#define CXLDEV_MBOX_CTRL_DOORBELL BIT(0) +#define CXLDEV_MBOX_CMD_OFFSET 0x08 +#define CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK GENMASK_ULL(15, 0) +#define CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK GENMASK_ULL(36, 16) +#define CXLDEV_MBOX_STATUS_OFFSET 0x10 +#define CXLDEV_MBOX_STATUS_RET_CODE_MASK GENMASK_ULL(47, 32) +#define CXLDEV_MBOX_BG_CMD_STATUS_OFFSET 0x18 +#define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20 + +/* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ +#define CXLMDEV_STATUS_OFFSET 0x0 +#define CXLMDEV_DEV_FATAL BIT(0) +#define CXLMDEV_FW_HALT BIT(1) +#define CXLMDEV_STATUS_MEDIA_STATUS_MASK GENMASK(3, 2) +#define CXLMDEV_MS_NOT_READY 0 +#define CXLMDEV_MS_READY 1 +#define CXLMDEV_MS_ERROR 2 +#define CXLMDEV_MS_DISABLED 3 +#define CXLMDEV_READY(status) \ + (FIELD_GET(CXLMDEV_STATUS_MEDIA_STATUS_MASK, status) == \ + CXLMDEV_MS_READY) +#define CXLMDEV_MBOX_IF_READY BIT(4) +#define CXLMDEV_RESET_NEEDED_MASK GENMASK(7, 5) +#define CXLMDEV_RESET_NEEDED_NOT 0 +#define CXLMDEV_RESET_NEEDED_COLD 1 +#define CXLMDEV_RESET_NEEDED_WARM 2 +#define CXLMDEV_RESET_NEEDED_HOT 3 +#define CXLMDEV_RESET_NEEDED_CXL 4 +#define CXLMDEV_RESET_NEEDED(status) \ + (FIELD_GET(CXLMDEV_RESET_NEEDED_MASK, status) != \ + CXLMDEV_RESET_NEEDED_NOT) + +struct cxl_memdev; +/** + * struct cxl_mem - A CXL memory device + * @pdev: The PCI device associated with this CXL device. + * @regs: IO mappings to the device's MMIO + * @status_regs: CXL 2.0 8.2.8.3 Device Status Registers + * @mbox_regs: CXL 2.0 8.2.8.4 Mailbox Registers + * @memdev_regs: CXL 2.0 8.2.8.5 Memory Device Registers + * @payload_size: Size of space for payload + * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register) + * @mbox_mutex: Mutex to synchronize mailbox access. + * @firmware_version: Firmware version for the memory device. + * @enabled_commands: Hardware commands found enabled in CEL. + * @pmem_range: Persistent memory capacity information. + * @ram_range: Volatile memory capacity information. + */ +struct cxl_mem { + struct pci_dev *pdev; + void __iomem *regs; + struct cxl_memdev *cxlmd; + + void __iomem *status_regs; + void __iomem *mbox_regs; + void __iomem *memdev_regs; + + size_t payload_size; + struct mutex mbox_mutex; /* Protects device mailbox and firmware */ + char firmware_version[0x10]; + unsigned long *enabled_cmds; + + struct range pmem_range; + struct range ram_range; +}; + +extern struct bus_type cxl_bus_type; +#endif /* __CXL_H__ */ diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c new file mode 100644 index 000000000000..244cb7d89678 --- /dev/null +++ b/drivers/cxl/mem.c @@ -0,0 +1,1552 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#include <uapi/linux/cxl_mem.h> +#include <linux/security.h> +#include <linux/debugfs.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/cdev.h> +#include <linux/idr.h> +#include <linux/pci.h> +#include <linux/io.h> +#include <linux/io-64-nonatomic-lo-hi.h> +#include "pci.h" +#include "cxl.h" + +/** + * DOC: cxl mem + * + * This implements a CXL memory device ("type-3") as it is defined by the + * Compute Express Link specification. + * + * The driver has several responsibilities, mainly: + * - Create the memX device and register on the CXL bus. + * - Enumerate device's register interface and map them. + * - Probe the device attributes to establish sysfs interface. + * - Provide an IOCTL interface to userspace to communicate with the device for + * things like firmware update. + * - Support management of interleave sets. + * - Handle and manage error conditions. + */ + +/* + * An entire PCI topology full of devices should be enough for any + * config + */ +#define CXL_MEM_MAX_DEVS 65536 + +#define cxl_doorbell_busy(cxlm) \ + (readl((cxlm)->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET) & \ + CXLDEV_MBOX_CTRL_DOORBELL) + +/* CXL 2.0 - 8.2.8.4 */ +#define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) + +enum opcode { + CXL_MBOX_OP_INVALID = 0x0000, + CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID, + CXL_MBOX_OP_GET_FW_INFO = 0x0200, + CXL_MBOX_OP_ACTIVATE_FW = 0x0202, + CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400, + CXL_MBOX_OP_GET_LOG = 0x0401, + CXL_MBOX_OP_IDENTIFY = 0x4000, + CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100, + CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101, + CXL_MBOX_OP_GET_LSA = 0x4102, + CXL_MBOX_OP_SET_LSA = 0x4103, + CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200, + CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204, + CXL_MBOX_OP_SCAN_MEDIA = 0x4304, + CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305, + CXL_MBOX_OP_MAX = 0x10000 +}; + +/** + * struct mbox_cmd - A command to be submitted to hardware. + * @opcode: (input) The command set and command submitted to hardware. + * @payload_in: (input) Pointer to the input payload. + * @payload_out: (output) Pointer to the output payload. Must be allocated by + * the caller. + * @size_in: (input) Number of bytes to load from @payload_in. + * @size_out: (input) Max number of bytes loaded into @payload_out. + * (output) Number of bytes generated by the device. For fixed size + * outputs commands this is always expected to be deterministic. For + * variable sized output commands, it tells the exact number of bytes + * written. + * @return_code: (output) Error code returned from hardware. + * + * This is the primary mechanism used to send commands to the hardware. + * All the fields except @payload_* correspond exactly to the fields described in + * Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and + * @payload_out are written to, and read from the Command Payload Registers + * defined in CXL 2.0 8.2.8.4.8. + */ +struct mbox_cmd { + u16 opcode; + void *payload_in; + void *payload_out; + size_t size_in; + size_t size_out; + u16 return_code; +#define CXL_MBOX_SUCCESS 0 +}; + +/** + * struct cxl_memdev - CXL bus object representing a Type-3 Memory Device + * @dev: driver core device object + * @cdev: char dev core object for ioctl operations + * @cxlm: pointer to the parent device driver data + * @ops_active: active user of @cxlm in ops handlers + * @ops_dead: completion when all @cxlm ops users have exited + * @id: id number of this memdev instance. + */ +struct cxl_memdev { + struct device dev; + struct cdev cdev; + struct cxl_mem *cxlm; + struct percpu_ref ops_active; + struct completion ops_dead; + int id; +}; + +static int cxl_mem_major; +static DEFINE_IDA(cxl_memdev_ida); +static struct dentry *cxl_debugfs; +static bool cxl_raw_allow_all; + +enum { + CEL_UUID, + VENDOR_DEBUG_UUID, +}; + +/* See CXL 2.0 Table 170. Get Log Input Payload */ +static const uuid_t log_uuid[] = { + [CEL_UUID] = UUID_INIT(0xda9c0b5, 0xbf41, 0x4b78, 0x8f, 0x79, 0x96, + 0xb1, 0x62, 0x3b, 0x3f, 0x17), + [VENDOR_DEBUG_UUID] = UUID_INIT(0xe1819d9, 0x11a9, 0x400c, 0x81, 0x1f, + 0xd6, 0x07, 0x19, 0x40, 0x3d, 0x86), +}; + +/** + * struct cxl_mem_command - Driver representation of a memory device command + * @info: Command information as it exists for the UAPI + * @opcode: The actual bits used for the mailbox protocol + * @flags: Set of flags effecting driver behavior. + * + * * %CXL_CMD_FLAG_FORCE_ENABLE: In cases of error, commands with this flag + * will be enabled by the driver regardless of what hardware may have + * advertised. + * + * The cxl_mem_command is the driver's internal representation of commands that + * are supported by the driver. Some of these commands may not be supported by + * the hardware. The driver will use @info to validate the fields passed in by + * the user then submit the @opcode to the hardware. + * + * See struct cxl_command_info. + */ +struct cxl_mem_command { + struct cxl_command_info info; + enum opcode opcode; + u32 flags; +#define CXL_CMD_FLAG_NONE 0 +#define CXL_CMD_FLAG_FORCE_ENABLE BIT(0) +}; + +#define CXL_CMD(_id, sin, sout, _flags) \ + [CXL_MEM_COMMAND_ID_##_id] = { \ + .info = { \ + .id = CXL_MEM_COMMAND_ID_##_id, \ + .size_in = sin, \ + .size_out = sout, \ + }, \ + .opcode = CXL_MBOX_OP_##_id, \ + .flags = _flags, \ + } + +/* + * This table defines the supported mailbox commands for the driver. This table + * is made up of a UAPI structure. Non-negative values as parameters in the + * table will be validated against the user's input. For example, if size_in is + * 0, and the user passed in 1, it is an error. + */ +static struct cxl_mem_command mem_commands[] = { + CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE), +#ifdef CONFIG_CXL_MEM_RAW_COMMANDS + CXL_CMD(RAW, ~0, ~0, 0), +#endif + CXL_CMD(GET_SUPPORTED_LOGS, 0, ~0, CXL_CMD_FLAG_FORCE_ENABLE), + CXL_CMD(GET_FW_INFO, 0, 0x50, 0), + CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0), + CXL_CMD(GET_LSA, 0x8, ~0, 0), + CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0), + CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE), +}; + +/* + * Commands that RAW doesn't permit. The rationale for each: + * + * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment / + * coordination of transaction timeout values at the root bridge level. + * + * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live + * and needs to be coordinated with HDM updates. + * + * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the + * driver and any writes from userspace invalidates those contents. + * + * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes + * to the device after it is marked clean, userspace can not make that + * assertion. + * + * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that + * is kept up to date with patrol notifications and error management. + */ +static u16 cxl_disabled_raw_commands[] = { + CXL_MBOX_OP_ACTIVATE_FW, + CXL_MBOX_OP_SET_PARTITION_INFO, + CXL_MBOX_OP_SET_LSA, + CXL_MBOX_OP_SET_SHUTDOWN_STATE, + CXL_MBOX_OP_SCAN_MEDIA, + CXL_MBOX_OP_GET_SCAN_MEDIA, +}; + +/* + * Command sets that RAW doesn't permit. All opcodes in this set are + * disabled because they pass plain text security payloads over the + * user/kernel boundary. This functionality is intended to be wrapped + * behind the keys ABI which allows for encrypted payloads in the UAPI + */ +static u8 security_command_sets[] = { + 0x44, /* Sanitize */ + 0x45, /* Persistent Memory Data-at-rest Security */ + 0x46, /* Security Passthrough */ +}; + +#define cxl_for_each_cmd(cmd) \ + for ((cmd) = &mem_commands[0]; \ + ((cmd) - mem_commands) < ARRAY_SIZE(mem_commands); (cmd)++) + +#define cxl_cmd_count ARRAY_SIZE(mem_commands) + +static int cxl_mem_wait_for_doorbell(struct cxl_mem *cxlm) +{ + const unsigned long start = jiffies; + unsigned long end = start; + + while (cxl_doorbell_busy(cxlm)) { + end = jiffies; + + if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { + /* Check again in case preempted before timeout test */ + if (!cxl_doorbell_busy(cxlm)) + break; + return -ETIMEDOUT; + } + cpu_relax(); + } + + dev_dbg(&cxlm->pdev->dev, "Doorbell wait took %dms", + jiffies_to_msecs(end) - jiffies_to_msecs(start)); + return 0; +} + +static bool cxl_is_security_command(u16 opcode) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(security_command_sets); i++) + if (security_command_sets[i] == (opcode >> 8)) + return true; + return false; +} + +static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm, + struct mbox_cmd *mbox_cmd) +{ + struct device *dev = &cxlm->pdev->dev; + + dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n", + mbox_cmd->opcode, mbox_cmd->size_in); +} + +/** + * __cxl_mem_mbox_send_cmd() - Execute a mailbox command + * @cxlm: The CXL memory device to communicate with. + * @mbox_cmd: Command to send to the memory device. + * + * Context: Any context. Expects mbox_mutex to be held. + * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. + * Caller should check the return code in @mbox_cmd to make sure it + * succeeded. + * + * This is a generic form of the CXL mailbox send command thus only using the + * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory + * devices, and perhaps other types of CXL devices may have further information + * available upon error conditions. Driver facilities wishing to send mailbox + * commands should use the wrapper command. + * + * The CXL spec allows for up to two mailboxes. The intention is for the primary + * mailbox to be OS controlled and the secondary mailbox to be used by system + * firmware. This allows the OS and firmware to communicate with the device and + * not need to coordinate with each other. The driver only uses the primary + * mailbox. + */ +static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, + struct mbox_cmd *mbox_cmd) +{ + void __iomem *payload = cxlm->mbox_regs + CXLDEV_MBOX_PAYLOAD_OFFSET; + u64 cmd_reg, status_reg; + size_t out_len; + int rc; + + lockdep_assert_held(&cxlm->mbox_mutex); + + /* + * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. + * 1. Caller reads MB Control Register to verify doorbell is clear + * 2. Caller writes Command Register + * 3. Caller writes Command Payload Registers if input payload is non-empty + * 4. Caller writes MB Control Register to set doorbell + * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured + * 6. Caller reads MB Status Register to fetch Return code + * 7. If command successful, Caller reads Command Register to get Payload Length + * 8. If output payload is non-empty, host reads Command Payload Registers + * + * Hardware is free to do whatever it wants before the doorbell is rung, + * and isn't allowed to change anything after it clears the doorbell. As + * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can + * also happen in any order (though some orders might not make sense). + */ + + /* #1 */ + if (cxl_doorbell_busy(cxlm)) { + dev_err_ratelimited(&cxlm->pdev->dev, + "Mailbox re-busy after acquiring\n"); + return -EBUSY; + } + + cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, + mbox_cmd->opcode); + if (mbox_cmd->size_in) { + if (WARN_ON(!mbox_cmd->payload_in)) + return -EINVAL; + + cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, + mbox_cmd->size_in); + memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); + } + + /* #2, #3 */ + writeq(cmd_reg, cxlm->mbox_regs + CXLDEV_MBOX_CMD_OFFSET); + + /* #4 */ + dev_dbg(&cxlm->pdev->dev, "Sending command\n"); + writel(CXLDEV_MBOX_CTRL_DOORBELL, + cxlm->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET); + + /* #5 */ + rc = cxl_mem_wait_for_doorbell(cxlm); + if (rc == -ETIMEDOUT) { + cxl_mem_mbox_timeout(cxlm, mbox_cmd); + return rc; + } + + /* #6 */ + status_reg = readq(cxlm->mbox_regs + CXLDEV_MBOX_STATUS_OFFSET); + mbox_cmd->return_code = + FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); + + if (mbox_cmd->return_code != 0) { + dev_dbg(&cxlm->pdev->dev, "Mailbox operation had an error\n"); + return 0; + } + + /* #7 */ + cmd_reg = readq(cxlm->mbox_regs + CXLDEV_MBOX_CMD_OFFSET); + out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); + + /* #8 */ + if (out_len && mbox_cmd->payload_out) { + /* + * Sanitize the copy. If hardware misbehaves, out_len per the + * spec can actually be greater than the max allowed size (21 + * bits available but spec defined 1M max). The caller also may + * have requested less data than the hardware supplied even + * within spec. + */ + size_t n = min3(mbox_cmd->size_out, cxlm->payload_size, out_len); + + memcpy_fromio(mbox_cmd->payload_out, payload, n); + mbox_cmd->size_out = n; + } else { + mbox_cmd->size_out = 0; + } + + return 0; +} + +/** + * cxl_mem_mbox_get() - Acquire exclusive access to the mailbox. + * @cxlm: The memory device to gain access to. + * + * Context: Any context. Takes the mbox_mutex. + * Return: 0 if exclusive access was acquired. + */ +static int cxl_mem_mbox_get(struct cxl_mem *cxlm) +{ + struct device *dev = &cxlm->pdev->dev; + u64 md_status; + int rc; + + mutex_lock_io(&cxlm->mbox_mutex); + + /* + * XXX: There is some amount of ambiguity in the 2.0 version of the spec + * around the mailbox interface ready (8.2.8.5.1.1). The purpose of the + * bit is to allow firmware running on the device to notify the driver + * that it's ready to receive commands. It is unclear if the bit needs + * to be read for each transaction mailbox, ie. the firmware can switch + * it on and off as needed. Second, there is no defined timeout for + * mailbox ready, like there is for the doorbell interface. + * + * Assumptions: + * 1. The firmware might toggle the Mailbox Interface Ready bit, check + * it for every command. + * + * 2. If the doorbell is clear, the firmware should have first set the + * Mailbox Interface Ready bit. Therefore, waiting for the doorbell + * to be ready is sufficient. + */ + rc = cxl_mem_wait_for_doorbell(cxlm); + if (rc) { + dev_warn(dev, "Mailbox interface not ready\n"); + goto out; + } + + md_status = readq(cxlm->memdev_regs + CXLMDEV_STATUS_OFFSET); + if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) { + dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n"); + rc = -EBUSY; + goto out; + } + + /* + * Hardware shouldn't allow a ready status but also have failure bits + * set. Spit out an error, this should be a bug report + */ + rc = -EFAULT; + if (md_status & CXLMDEV_DEV_FATAL) { + dev_err(dev, "mbox: reported ready, but fatal\n"); + goto out; + } + if (md_status & CXLMDEV_FW_HALT) { + dev_err(dev, "mbox: reported ready, but halted\n"); + goto out; + } + if (CXLMDEV_RESET_NEEDED(md_status)) { + dev_err(dev, "mbox: reported ready, but reset needed\n"); + goto out; + } + + /* with lock held */ + return 0; + +out: + mutex_unlock(&cxlm->mbox_mutex); + return rc; +} + +/** + * cxl_mem_mbox_put() - Release exclusive access to the mailbox. + * @cxlm: The CXL memory device to communicate with. + * + * Context: Any context. Expects mbox_mutex to be held. + */ +static void cxl_mem_mbox_put(struct cxl_mem *cxlm) +{ + mutex_unlock(&cxlm->mbox_mutex); +} + +/** + * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace. + * @cxlm: The CXL memory device to communicate with. + * @cmd: The validated command. + * @in_payload: Pointer to userspace's input payload. + * @out_payload: Pointer to userspace's output payload. + * @size_out: (Input) Max payload size to copy out. + * (Output) Payload size hardware generated. + * @retval: Hardware generated return code from the operation. + * + * Return: + * * %0 - Mailbox transaction succeeded. This implies the mailbox + * protocol completed successfully not that the operation itself + * was successful. + * * %-ENOMEM - Couldn't allocate a bounce buffer. + * * %-EFAULT - Something happened with copy_to/from_user. + * * %-EINTR - Mailbox acquisition interrupted. + * * %-EXXX - Transaction level failures. + * + * Creates the appropriate mailbox command and dispatches it on behalf of a + * userspace request. The input and output payloads are copied between + * userspace. + * + * See cxl_send_cmd(). + */ +static int handle_mailbox_cmd_from_user(struct cxl_mem *cxlm, + const struct cxl_mem_command *cmd, + u64 in_payload, u64 out_payload, + s32 *size_out, u32 *retval) +{ + struct device *dev = &cxlm->pdev->dev; + struct mbox_cmd mbox_cmd = { + .opcode = cmd->opcode, + .size_in = cmd->info.size_in, + .size_out = cmd->info.size_out, + }; + int rc; + + if (cmd->info.size_out) { + mbox_cmd.payload_out = kvzalloc(cmd->info.size_out, GFP_KERNEL); + if (!mbox_cmd.payload_out) + return -ENOMEM; + } + + if (cmd->info.size_in) { + mbox_cmd.payload_in = vmemdup_user(u64_to_user_ptr(in_payload), + cmd->info.size_in); + if (IS_ERR(mbox_cmd.payload_in)) { + kvfree(mbox_cmd.payload_out); + return PTR_ERR(mbox_cmd.payload_in); + } + } + + rc = cxl_mem_mbox_get(cxlm); + if (rc) + goto out; + + dev_dbg(dev, + "Submitting %s command for user\n" + "\topcode: %x\n" + "\tsize: %ub\n", + cxl_command_names[cmd->info.id].name, mbox_cmd.opcode, + cmd->info.size_in); + + dev_WARN_ONCE(dev, cmd->info.id == CXL_MEM_COMMAND_ID_RAW, + "raw command path used\n"); + + rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); + cxl_mem_mbox_put(cxlm); + if (rc) + goto out; + + /* + * @size_out contains the max size that's allowed to be written back out + * to userspace. While the payload may have written more output than + * this it will have to be ignored. + */ + if (mbox_cmd.size_out) { + dev_WARN_ONCE(dev, mbox_cmd.size_out > *size_out, + "Invalid return size\n"); + if (copy_to_user(u64_to_user_ptr(out_payload), + mbox_cmd.payload_out, mbox_cmd.size_out)) { + rc = -EFAULT; + goto out; + } + } + + *size_out = mbox_cmd.size_out; + *retval = mbox_cmd.return_code; + +out: + kvfree(mbox_cmd.payload_in); + kvfree(mbox_cmd.payload_out); + return rc; +} + +static bool cxl_mem_raw_command_allowed(u16 opcode) +{ + int i; + + if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS)) + return false; + + if (security_locked_down(LOCKDOWN_NONE)) + return false; + + if (cxl_raw_allow_all) + return true; + + if (cxl_is_security_command(opcode)) + return false; + + for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++) + if (cxl_disabled_raw_commands[i] == opcode) + return false; + + return true; +} + +/** + * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND. + * @cxlm: &struct cxl_mem device whose mailbox will be used. + * @send_cmd: &struct cxl_send_command copied in from userspace. + * @out_cmd: Sanitized and populated &struct cxl_mem_command. + * + * Return: + * * %0 - @out_cmd is ready to send. + * * %-ENOTTY - Invalid command specified. + * * %-EINVAL - Reserved fields or invalid values were used. + * * %-ENOMEM - Input or output buffer wasn't sized properly. + * * %-EPERM - Attempted to use a protected command. + * + * The result of this command is a fully validated command in @out_cmd that is + * safe to send to the hardware. + * + * See handle_mailbox_cmd_from_user() + */ +static int cxl_validate_cmd_from_user(struct cxl_mem *cxlm, + const struct cxl_send_command *send_cmd, + struct cxl_mem_command *out_cmd) +{ + const struct cxl_command_info *info; + struct cxl_mem_command *c; + + if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX) + return -ENOTTY; + + /* + * The user can never specify an input payload larger than what hardware + * supports, but output can be arbitrarily large (simply write out as + * much data as the hardware provides). + */ + if (send_cmd->in.size > cxlm->payload_size) + return -EINVAL; + + /* + * Checks are bypassed for raw commands but a WARN/taint will occur + * later in the callchain + */ + if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW) { + const struct cxl_mem_command temp = { + .info = { + .id = CXL_MEM_COMMAND_ID_RAW, + .flags = 0, + .size_in = send_cmd->in.size, + .size_out = send_cmd->out.size, + }, + .opcode = send_cmd->raw.opcode + }; + + if (send_cmd->raw.rsvd) + return -EINVAL; + + /* + * Unlike supported commands, the output size of RAW commands + * gets passed along without further checking, so it must be + * validated here. + */ + if (send_cmd->out.size > cxlm->payload_size) + return -EINVAL; + + if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode)) + return -EPERM; + + memcpy(out_cmd, &temp, sizeof(temp)); + + return 0; + } + + if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK) + return -EINVAL; + + if (send_cmd->rsvd) + return -EINVAL; + + if (send_cmd->in.rsvd || send_cmd->out.rsvd) + return -EINVAL; + + /* Convert user's command into the internal representation */ + c = &mem_commands[send_cmd->id]; + info = &c->info; + + /* Check that the command is enabled for hardware */ + if (!test_bit(info->id, cxlm->enabled_cmds)) + return -ENOTTY; + + /* Check the input buffer is the expected size */ + if (info->size_in >= 0 && info->size_in != send_cmd->in.size) + return -ENOMEM; + + /* Check the output buffer is at least large enough */ + if (info->size_out >= 0 && send_cmd->out.size < info->size_out) + return -ENOMEM; + + memcpy(out_cmd, c, sizeof(*c)); + out_cmd->info.size_in = send_cmd->in.size; + /* + * XXX: out_cmd->info.size_out will be controlled by the driver, and the + * specified number of bytes @send_cmd->out.size will be copied back out + * to userspace. + */ + + return 0; +} + +static int cxl_query_cmd(struct cxl_memdev *cxlmd, + struct cxl_mem_query_commands __user *q) +{ + struct device *dev = &cxlmd->dev; + struct cxl_mem_command *cmd; + u32 n_commands; + int j = 0; + + dev_dbg(dev, "Query IOCTL\n"); + + if (get_user(n_commands, &q->n_commands)) + return -EFAULT; + + /* returns the total number if 0 elements are requested. */ + if (n_commands == 0) + return put_user(cxl_cmd_count, &q->n_commands); + + /* + * otherwise, return max(n_commands, total commands) cxl_command_info + * structures. + */ + cxl_for_each_cmd(cmd) { + const struct cxl_command_info *info = &cmd->info; + + if (copy_to_user(&q->commands[j++], info, sizeof(*info))) + return -EFAULT; + + if (j == n_commands) + break; + } + + return 0; +} + +static int cxl_send_cmd(struct cxl_memdev *cxlmd, + struct cxl_send_command __user *s) +{ + struct cxl_mem *cxlm = cxlmd->cxlm; + struct device *dev = &cxlmd->dev; + struct cxl_send_command send; + struct cxl_mem_command c; + int rc; + + dev_dbg(dev, "Send IOCTL\n"); + + if (copy_from_user(&send, s, sizeof(send))) + return -EFAULT; + + rc = cxl_validate_cmd_from_user(cxlmd->cxlm, &send, &c); + if (rc) + return rc; + + /* Prepare to handle a full payload for variable sized output */ + if (c.info.size_out < 0) + c.info.size_out = cxlm->payload_size; + + rc = handle_mailbox_cmd_from_user(cxlm, &c, send.in.payload, + send.out.payload, &send.out.size, + &send.retval); + if (rc) + return rc; + + if (copy_to_user(s, &send, sizeof(send))) + return -EFAULT; + + return 0; +} + +static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, + unsigned long arg) +{ + switch (cmd) { + case CXL_MEM_QUERY_COMMANDS: + return cxl_query_cmd(cxlmd, (void __user *)arg); + case CXL_MEM_SEND_COMMAND: + return cxl_send_cmd(cxlmd, (void __user *)arg); + default: + return -ENOTTY; + } +} + +static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct cxl_memdev *cxlmd; + struct inode *inode; + int rc = -ENOTTY; + + inode = file_inode(file); + cxlmd = container_of(inode->i_cdev, typeof(*cxlmd), cdev); + + if (!percpu_ref_tryget_live(&cxlmd->ops_active)) + return -ENXIO; + + rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); + + percpu_ref_put(&cxlmd->ops_active); + + return rc; +} + +static const struct file_operations cxl_memdev_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = cxl_memdev_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .llseek = noop_llseek, +}; + +static inline struct cxl_mem_command *cxl_mem_find_command(u16 opcode) +{ + struct cxl_mem_command *c; + + cxl_for_each_cmd(c) + if (c->opcode == opcode) + return c; + + return NULL; +} + +/** + * cxl_mem_mbox_send_cmd() - Send a mailbox command to a memory device. + * @cxlm: The CXL memory device to communicate with. + * @opcode: Opcode for the mailbox command. + * @in: The input payload for the mailbox command. + * @in_size: The length of the input payload + * @out: Caller allocated buffer for the output. + * @out_size: Expected size of output. + * + * Context: Any context. Will acquire and release mbox_mutex. + * Return: + * * %>=0 - Number of bytes returned in @out. + * * %-E2BIG - Payload is too large for hardware. + * * %-EBUSY - Couldn't acquire exclusive mailbox access. + * * %-EFAULT - Hardware error occurred. + * * %-ENXIO - Command completed, but device reported an error. + * * %-EIO - Unexpected output size. + * + * Mailbox commands may execute successfully yet the device itself reported an + * error. While this distinction can be useful for commands from userspace, the + * kernel will only be able to use results when both are successful. + * + * See __cxl_mem_mbox_send_cmd() + */ +static int cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, u16 opcode, + void *in, size_t in_size, + void *out, size_t out_size) +{ + const struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); + struct mbox_cmd mbox_cmd = { + .opcode = opcode, + .payload_in = in, + .size_in = in_size, + .size_out = out_size, + .payload_out = out, + }; + int rc; + + if (out_size > cxlm->payload_size) + return -E2BIG; + + rc = cxl_mem_mbox_get(cxlm); + if (rc) + return rc; + + rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); + cxl_mem_mbox_put(cxlm); + if (rc) + return rc; + + /* TODO: Map return code to proper kernel style errno */ + if (mbox_cmd.return_code != CXL_MBOX_SUCCESS) + return -ENXIO; + + /* + * Variable sized commands can't be validated and so it's up to the + * caller to do that if they wish. + */ + if (cmd->info.size_out >= 0 && mbox_cmd.size_out != out_size) + return -EIO; + + return 0; +} + +/** + * cxl_mem_setup_regs() - Setup necessary MMIO. + * @cxlm: The CXL memory device to communicate with. + * + * Return: 0 if all necessary registers mapped. + * + * A memory device is required by spec to implement a certain set of MMIO + * regions. The purpose of this function is to enumerate and map those + * registers. + */ +static int cxl_mem_setup_regs(struct cxl_mem *cxlm) +{ + struct device *dev = &cxlm->pdev->dev; + int cap, cap_count; + u64 cap_array; + + cap_array = readq(cxlm->regs + CXLDEV_CAP_ARRAY_OFFSET); + if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) != + CXLDEV_CAP_ARRAY_CAP_ID) + return -ENODEV; + + cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array); + + for (cap = 1; cap <= cap_count; cap++) { + void __iomem *register_block; + u32 offset; + u16 cap_id; + + cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK, + readl(cxlm->regs + cap * 0x10)); + offset = readl(cxlm->regs + cap * 0x10 + 0x4); + register_block = cxlm->regs + offset; + + switch (cap_id) { + case CXLDEV_CAP_CAP_ID_DEVICE_STATUS: + dev_dbg(dev, "found Status capability (0x%x)\n", offset); + cxlm->status_regs = register_block; + break; + case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX: + dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset); + cxlm->mbox_regs = register_block; + break; + case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX: + dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset); + break; + case CXLDEV_CAP_CAP_ID_MEMDEV: + dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset); + cxlm->memdev_regs = register_block; + break; + default: + dev_dbg(dev, "Unknown cap ID: %d (0x%x)\n", cap_id, offset); + break; + } + } + + if (!cxlm->status_regs || !cxlm->mbox_regs || !cxlm->memdev_regs) { + dev_err(dev, "registers not found: %s%s%s\n", + !cxlm->status_regs ? "status " : "", + !cxlm->mbox_regs ? "mbox " : "", + !cxlm->memdev_regs ? "memdev" : ""); + return -ENXIO; + } + + return 0; +} + +static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) +{ + const int cap = readl(cxlm->mbox_regs + CXLDEV_MBOX_CAPS_OFFSET); + + cxlm->payload_size = + 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); + + /* + * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register + * + * If the size is too small, mandatory commands will not work and so + * there's no point in going forward. If the size is too large, there's + * no harm is soft limiting it. + */ + cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M); + if (cxlm->payload_size < 256) { + dev_err(&cxlm->pdev->dev, "Mailbox is too small (%zub)", + cxlm->payload_size); + return -ENXIO; + } + + dev_dbg(&cxlm->pdev->dev, "Mailbox payload sized %zu", + cxlm->payload_size); + + return 0; +} + +static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev, u32 reg_lo, + u32 reg_hi) +{ + struct device *dev = &pdev->dev; + struct cxl_mem *cxlm; + void __iomem *regs; + u64 offset; + u8 bar; + int rc; + + cxlm = devm_kzalloc(&pdev->dev, sizeof(*cxlm), GFP_KERNEL); + if (!cxlm) { + dev_err(dev, "No memory available\n"); + return NULL; + } + + offset = ((u64)reg_hi << 32) | FIELD_GET(CXL_REGLOC_ADDR_MASK, reg_lo); + bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); + + /* Basic sanity check that BAR is big enough */ + if (pci_resource_len(pdev, bar) < offset) { + dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, + &pdev->resource[bar], (unsigned long long)offset); + return NULL; + } + + rc = pcim_iomap_regions(pdev, BIT(bar), pci_name(pdev)); + if (rc) { + dev_err(dev, "failed to map registers\n"); + return NULL; + } + regs = pcim_iomap_table(pdev)[bar]; + + mutex_init(&cxlm->mbox_mutex); + cxlm->pdev = pdev; + cxlm->regs = regs + offset; + cxlm->enabled_cmds = + devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count), + sizeof(unsigned long), + GFP_KERNEL | __GFP_ZERO); + if (!cxlm->enabled_cmds) { + dev_err(dev, "No memory available for bitmap\n"); + return NULL; + } + + dev_dbg(dev, "Mapped CXL Memory Device resource\n"); + return cxlm; +} + +static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) +{ + int pos; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DVSEC); + if (!pos) + return 0; + + while (pos) { + u16 vendor, id; + + pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER1, &vendor); + pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER2, &id); + if (vendor == PCI_DVSEC_VENDOR_ID_CXL && dvsec == id) + return pos; + + pos = pci_find_next_ext_capability(pdev, pos, + PCI_EXT_CAP_ID_DVSEC); + } + + return 0; +} + +static struct cxl_memdev *to_cxl_memdev(struct device *dev) +{ + return container_of(dev, struct cxl_memdev, dev); +} + +static void cxl_memdev_release(struct device *dev) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + + percpu_ref_exit(&cxlmd->ops_active); + ida_free(&cxl_memdev_ida, cxlmd->id); + kfree(cxlmd); +} + +static char *cxl_memdev_devnode(struct device *dev, umode_t *mode, kuid_t *uid, + kgid_t *gid) +{ + return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); +} + +static ssize_t firmware_version_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + + return sprintf(buf, "%.16s\n", cxlm->firmware_version); +} +static DEVICE_ATTR_RO(firmware_version); + +static ssize_t payload_max_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + + return sprintf(buf, "%zu\n", cxlm->payload_size); +} +static DEVICE_ATTR_RO(payload_max); + +static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + unsigned long long len = range_len(&cxlm->ram_range); + + return sprintf(buf, "%#llx\n", len); +} + +static struct device_attribute dev_attr_ram_size = + __ATTR(size, 0444, ram_size_show, NULL); + +static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + unsigned long long len = range_len(&cxlm->pmem_range); + + return sprintf(buf, "%#llx\n", len); +} + +static struct device_attribute dev_attr_pmem_size = + __ATTR(size, 0444, pmem_size_show, NULL); + +static struct attribute *cxl_memdev_attributes[] = { + &dev_attr_firmware_version.attr, + &dev_attr_payload_max.attr, + NULL, +}; + +static struct attribute *cxl_memdev_pmem_attributes[] = { + &dev_attr_pmem_size.attr, + NULL, +}; + +static struct attribute *cxl_memdev_ram_attributes[] = { + &dev_attr_ram_size.attr, + NULL, +}; + +static struct attribute_group cxl_memdev_attribute_group = { + .attrs = cxl_memdev_attributes, +}; + +static struct attribute_group cxl_memdev_ram_attribute_group = { + .name = "ram", + .attrs = cxl_memdev_ram_attributes, +}; + +static struct attribute_group cxl_memdev_pmem_attribute_group = { + .name = "pmem", + .attrs = cxl_memdev_pmem_attributes, +}; + +static const struct attribute_group *cxl_memdev_attribute_groups[] = { + &cxl_memdev_attribute_group, + &cxl_memdev_ram_attribute_group, + &cxl_memdev_pmem_attribute_group, + NULL, +}; + +static const struct device_type cxl_memdev_type = { + .name = "cxl_memdev", + .release = cxl_memdev_release, + .devnode = cxl_memdev_devnode, + .groups = cxl_memdev_attribute_groups, +}; + +static void cxlmdev_unregister(void *_cxlmd) +{ + struct cxl_memdev *cxlmd = _cxlmd; + struct device *dev = &cxlmd->dev; + + percpu_ref_kill(&cxlmd->ops_active); + cdev_device_del(&cxlmd->cdev, dev); + wait_for_completion(&cxlmd->ops_dead); + cxlmd->cxlm = NULL; + put_device(dev); +} + +static void cxlmdev_ops_active_release(struct percpu_ref *ref) +{ + struct cxl_memdev *cxlmd = + container_of(ref, typeof(*cxlmd), ops_active); + + complete(&cxlmd->ops_dead); +} + +static int cxl_mem_add_memdev(struct cxl_mem *cxlm) +{ + struct pci_dev *pdev = cxlm->pdev; + struct cxl_memdev *cxlmd; + struct device *dev; + struct cdev *cdev; + int rc; + + cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); + if (!cxlmd) + return -ENOMEM; + init_completion(&cxlmd->ops_dead); + + /* + * @cxlm is deallocated when the driver unbinds so operations + * that are using it need to hold a live reference. + */ + cxlmd->cxlm = cxlm; + rc = percpu_ref_init(&cxlmd->ops_active, cxlmdev_ops_active_release, 0, + GFP_KERNEL); + if (rc) + goto err_ref; + + rc = ida_alloc_range(&cxl_memdev_ida, 0, CXL_MEM_MAX_DEVS, GFP_KERNEL); + if (rc < 0) + goto err_id; + cxlmd->id = rc; + + dev = &cxlmd->dev; + device_initialize(dev); + dev->parent = &pdev->dev; + dev->bus = &cxl_bus_type; + dev->devt = MKDEV(cxl_mem_major, cxlmd->id); + dev->type = &cxl_memdev_type; + dev_set_name(dev, "mem%d", cxlmd->id); + + cdev = &cxlmd->cdev; + cdev_init(cdev, &cxl_memdev_fops); + + rc = cdev_device_add(cdev, dev); + if (rc) + goto err_add; + + return devm_add_action_or_reset(dev->parent, cxlmdev_unregister, cxlmd); + +err_add: + ida_free(&cxl_memdev_ida, cxlmd->id); +err_id: + /* + * Theoretically userspace could have already entered the fops, + * so flush ops_active. + */ + percpu_ref_kill(&cxlmd->ops_active); + wait_for_completion(&cxlmd->ops_dead); + percpu_ref_exit(&cxlmd->ops_active); +err_ref: + kfree(cxlmd); + + return rc; +} + +static int cxl_xfer_log(struct cxl_mem *cxlm, uuid_t *uuid, u32 size, u8 *out) +{ + u32 remaining = size; + u32 offset = 0; + + while (remaining) { + u32 xfer_size = min_t(u32, remaining, cxlm->payload_size); + struct cxl_mbox_get_log { + uuid_t uuid; + __le32 offset; + __le32 length; + } __packed log = { + .uuid = *uuid, + .offset = cpu_to_le32(offset), + .length = cpu_to_le32(xfer_size) + }; + int rc; + + rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_LOG, &log, + sizeof(log), out, xfer_size); + if (rc < 0) + return rc; + + out += xfer_size; + remaining -= xfer_size; + offset += xfer_size; + } + + return 0; +} + +/** + * cxl_walk_cel() - Walk through the Command Effects Log. + * @cxlm: Device. + * @size: Length of the Command Effects Log. + * @cel: CEL + * + * Iterate over each entry in the CEL and determine if the driver supports the + * command. If so, the command is enabled for the device and can be used later. + */ +static void cxl_walk_cel(struct cxl_mem *cxlm, size_t size, u8 *cel) +{ + struct cel_entry { + __le16 opcode; + __le16 effect; + } __packed * cel_entry; + const int cel_entries = size / sizeof(*cel_entry); + int i; + + cel_entry = (struct cel_entry *)cel; + + for (i = 0; i < cel_entries; i++) { + u16 opcode = le16_to_cpu(cel_entry[i].opcode); + struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); + + if (!cmd) { + dev_dbg(&cxlm->pdev->dev, + "Opcode 0x%04x unsupported by driver", opcode); + continue; + } + + set_bit(cmd->info.id, cxlm->enabled_cmds); + } +} + +struct cxl_mbox_get_supported_logs { + __le16 entries; + u8 rsvd[6]; + struct gsl_entry { + uuid_t uuid; + __le32 size; + } __packed entry[]; +} __packed; + +static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_mem *cxlm) +{ + struct cxl_mbox_get_supported_logs *ret; + int rc; + + ret = kvmalloc(cxlm->payload_size, GFP_KERNEL); + if (!ret) + return ERR_PTR(-ENOMEM); + + rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_SUPPORTED_LOGS, NULL, + 0, ret, cxlm->payload_size); + if (rc < 0) { + kvfree(ret); + return ERR_PTR(rc); + } + + return ret; +} + +/** + * cxl_mem_enumerate_cmds() - Enumerate commands for a device. + * @cxlm: The device. + * + * Returns 0 if enumerate completed successfully. + * + * CXL devices have optional support for certain commands. This function will + * determine the set of supported commands for the hardware and update the + * enabled_cmds bitmap in the @cxlm. + */ +static int cxl_mem_enumerate_cmds(struct cxl_mem *cxlm) +{ + struct cxl_mbox_get_supported_logs *gsl; + struct device *dev = &cxlm->pdev->dev; + struct cxl_mem_command *cmd; + int i, rc; + + gsl = cxl_get_gsl(cxlm); + if (IS_ERR(gsl)) + return PTR_ERR(gsl); + + rc = -ENOENT; + for (i = 0; i < le16_to_cpu(gsl->entries); i++) { + u32 size = le32_to_cpu(gsl->entry[i].size); + uuid_t uuid = gsl->entry[i].uuid; + u8 *log; + + dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size); + + if (!uuid_equal(&uuid, &log_uuid[CEL_UUID])) + continue; + + log = kvmalloc(size, GFP_KERNEL); + if (!log) { + rc = -ENOMEM; + goto out; + } + + rc = cxl_xfer_log(cxlm, &uuid, size, log); + if (rc) { + kvfree(log); + goto out; + } + + cxl_walk_cel(cxlm, size, log); + kvfree(log); + + /* In case CEL was bogus, enable some default commands. */ + cxl_for_each_cmd(cmd) + if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE) + set_bit(cmd->info.id, cxlm->enabled_cmds); + + /* Found the required CEL */ + rc = 0; + } + +out: + kvfree(gsl); + return rc; +} + +/** + * cxl_mem_identify() - Send the IDENTIFY command to the device. + * @cxlm: The device to identify. + * + * Return: 0 if identify was executed successfully. + * + * This will dispatch the identify command to the device and on success populate + * structures to be exported to sysfs. + */ +static int cxl_mem_identify(struct cxl_mem *cxlm) +{ + struct cxl_mbox_identify { + char fw_revision[0x10]; + __le64 total_capacity; + __le64 volatile_capacity; + __le64 persistent_capacity; + __le64 partition_align; + __le16 info_event_log_size; + __le16 warning_event_log_size; + __le16 failure_event_log_size; + __le16 fatal_event_log_size; + __le32 lsa_size; + u8 poison_list_max_mer[3]; + __le16 inject_poison_limit; + u8 poison_caps; + u8 qos_telemetry_caps; + } __packed id; + int rc; + + rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_IDENTIFY, NULL, 0, &id, + sizeof(id)); + if (rc < 0) + return rc; + + /* + * TODO: enumerate DPA map, as 'ram' and 'pmem' do not alias. + * For now, only the capacity is exported in sysfs + */ + cxlm->ram_range.start = 0; + cxlm->ram_range.end = le64_to_cpu(id.volatile_capacity) - 1; + + cxlm->pmem_range.start = 0; + cxlm->pmem_range.end = le64_to_cpu(id.persistent_capacity) - 1; + + memcpy(cxlm->firmware_version, id.fw_revision, sizeof(id.fw_revision)); + + return 0; +} + +static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + struct cxl_mem *cxlm = NULL; + u32 regloc_size, regblocks; + int rc, regloc, i; + + rc = pcim_enable_device(pdev); + if (rc) + return rc; + + regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); + if (!regloc) { + dev_err(dev, "register location dvsec not found\n"); + return -ENXIO; + } + + /* Get the size of the Register Locator DVSEC */ + pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); + regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); + + regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; + regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; + + for (i = 0; i < regblocks; i++, regloc += 8) { + u32 reg_lo, reg_hi; + u8 reg_type; + + /* "register low and high" contain other bits */ + pci_read_config_dword(pdev, regloc, ®_lo); + pci_read_config_dword(pdev, regloc + 4, ®_hi); + + reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); + + if (reg_type == CXL_REGLOC_RBI_MEMDEV) { + cxlm = cxl_mem_create(pdev, reg_lo, reg_hi); + break; + } + } + + if (!cxlm) + return -ENODEV; + + rc = cxl_mem_setup_regs(cxlm); + if (rc) + return rc; + + rc = cxl_mem_setup_mailbox(cxlm); + if (rc) + return rc; + + rc = cxl_mem_enumerate_cmds(cxlm); + if (rc) + return rc; + + rc = cxl_mem_identify(cxlm); + if (rc) + return rc; + + return cxl_mem_add_memdev(cxlm); +} + +static const struct pci_device_id cxl_mem_pci_tbl[] = { + /* PCI class code for CXL.mem Type-3 Devices */ + { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, + { /* terminate list */ }, +}; +MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); + +static struct pci_driver cxl_mem_driver = { + .name = KBUILD_MODNAME, + .id_table = cxl_mem_pci_tbl, + .probe = cxl_mem_probe, + .driver = { + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; + +static __init int cxl_mem_init(void) +{ + struct dentry *mbox_debugfs; + dev_t devt; + int rc; + + rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); + if (rc) + return rc; + + cxl_mem_major = MAJOR(devt); + + rc = pci_register_driver(&cxl_mem_driver); + if (rc) { + unregister_chrdev_region(MKDEV(cxl_mem_major, 0), + CXL_MEM_MAX_DEVS); + return rc; + } + + cxl_debugfs = debugfs_create_dir("cxl", NULL); + mbox_debugfs = debugfs_create_dir("mbox", cxl_debugfs); + debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs, + &cxl_raw_allow_all); + + return 0; +} + +static __exit void cxl_mem_exit(void) +{ + debugfs_remove_recursive(cxl_debugfs); + pci_unregister_driver(&cxl_mem_driver); + unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); +} + +MODULE_LICENSE("GPL v2"); +module_init(cxl_mem_init); +module_exit(cxl_mem_exit); +MODULE_IMPORT_NS(CXL); diff --git a/drivers/cxl/pci.h b/drivers/cxl/pci.h new file mode 100644 index 000000000000..af3ec078cf6c --- /dev/null +++ b/drivers/cxl/pci.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#ifndef __CXL_PCI_H__ +#define __CXL_PCI_H__ + +#define CXL_MEMORY_PROGIF 0x10 + +/* + * See section 8.1 Configuration Space Registers in the CXL 2.0 + * Specification + */ +#define PCI_DVSEC_HEADER1_LENGTH_MASK GENMASK(31, 20) +#define PCI_DVSEC_VENDOR_ID_CXL 0x1E98 +#define PCI_DVSEC_ID_CXL 0x0 + +#define PCI_DVSEC_ID_CXL_REGLOC_OFFSET 0x8 +#define PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET 0xC + +/* BAR Indicator Register (BIR) */ +#define CXL_REGLOC_BIR_MASK GENMASK(2, 0) + +/* Register Block Identifier (RBI) */ +#define CXL_REGLOC_RBI_MASK GENMASK(15, 8) +#define CXL_REGLOC_RBI_EMPTY 0 +#define CXL_REGLOC_RBI_COMPONENT 1 +#define CXL_REGLOC_RBI_VIRT 2 +#define CXL_REGLOC_RBI_MEMDEV 3 + +#define CXL_REGLOC_ADDR_MASK GENMASK(31, 16) + +#endif /* __CXL_PCI_H__ */ diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index 737b207c9e30..452e85ae87a8 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -179,7 +179,10 @@ static int dax_bus_remove(struct device *dev) struct dax_device_driver *dax_drv = to_dax_drv(dev->driver); struct dev_dax *dev_dax = to_dev_dax(dev); - return dax_drv->remove(dev_dax); + if (dax_drv->remove) + dax_drv->remove(dev_dax); + + return 0; } static struct bus_type dax_bus_type = { @@ -1038,7 +1041,7 @@ static ssize_t range_parse(const char *opt, size_t len, struct range *range) { unsigned long long addr = 0; char *start, *end, *str; - ssize_t rc = EINVAL; + ssize_t rc = -EINVAL; str = kstrdup(opt, GFP_KERNEL); if (!str) @@ -1392,6 +1395,13 @@ int __dax_driver_register(struct dax_device_driver *dax_drv, struct device_driver *drv = &dax_drv->drv; int rc = 0; + /* + * dax_bus_probe() calls dax_drv->probe() unconditionally. + * So better be safe than sorry and ensure it is provided. + */ + if (!dax_drv->probe) + return -EINVAL; + INIT_LIST_HEAD(&dax_drv->ids); drv->owner = module; drv->name = mod_name; @@ -1409,7 +1419,15 @@ int __dax_driver_register(struct dax_device_driver *dax_drv, mutex_unlock(&dax_bus_lock); if (rc) return rc; - return driver_register(drv); + + rc = driver_register(drv); + if (rc && dax_drv->match_always) { + mutex_lock(&dax_bus_lock); + match_always_count -= dax_drv->match_always; + mutex_unlock(&dax_bus_lock); + } + + return rc; } EXPORT_SYMBOL_GPL(__dax_driver_register); diff --git a/drivers/dax/bus.h b/drivers/dax/bus.h index 72b92f95509f..1e946ad7780a 100644 --- a/drivers/dax/bus.h +++ b/drivers/dax/bus.h @@ -39,7 +39,7 @@ struct dax_device_driver { struct list_head ids; int match_always; int (*probe)(struct dev_dax *dev); - int (*remove)(struct dev_dax *dev); + void (*remove)(struct dev_dax *dev); }; int __dax_driver_register(struct dax_device_driver *dax_drv, diff --git a/drivers/dax/device.c b/drivers/dax/device.c index 5da2980bb16b..db92573c94e8 100644 --- a/drivers/dax/device.c +++ b/drivers/dax/device.c @@ -452,15 +452,9 @@ int dev_dax_probe(struct dev_dax *dev_dax) } EXPORT_SYMBOL_GPL(dev_dax_probe); -static int dev_dax_remove(struct dev_dax *dev_dax) -{ - /* all probe actions are unwound by devm */ - return 0; -} - static struct dax_device_driver device_dax_driver = { .probe = dev_dax_probe, - .remove = dev_dax_remove, + /* all probe actions are unwound by devm, so .remove isn't necessary */ .match_always = 1, }; diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index 403ec42472d1..ac231cc36359 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -136,7 +136,7 @@ err_res_name: } #ifdef CONFIG_MEMORY_HOTREMOVE -static int dev_dax_kmem_remove(struct dev_dax *dev_dax) +static void dev_dax_kmem_remove(struct dev_dax *dev_dax) { int i, success = 0; struct device *dev = &dev_dax->dev; @@ -176,11 +176,9 @@ static int dev_dax_kmem_remove(struct dev_dax *dev_dax) kfree(data); dev_set_drvdata(dev, NULL); } - - return 0; } #else -static int dev_dax_kmem_remove(struct dev_dax *dev_dax) +static void dev_dax_kmem_remove(struct dev_dax *dev_dax) { /* * Without hotremove purposely leak the request_mem_region() for the @@ -190,7 +188,6 @@ static int dev_dax_kmem_remove(struct dev_dax *dev_dax) * request_mem_region(). */ any_hotremove_failed = true; - return 0; } #endif /* CONFIG_MEMORY_HOTREMOVE */ diff --git a/drivers/dax/pmem/compat.c b/drivers/dax/pmem/compat.c index 863c114fd88c..d81dc35fd65d 100644 --- a/drivers/dax/pmem/compat.c +++ b/drivers/dax/pmem/compat.c @@ -41,10 +41,9 @@ static int dax_pmem_compat_release(struct device *dev, void *data) return 0; } -static int dax_pmem_compat_remove(struct device *dev) +static void dax_pmem_compat_remove(struct device *dev) { device_for_each_child(dev, NULL, dax_pmem_compat_release); - return 0; } static struct nd_device_driver dax_pmem_compat_driver = { diff --git a/drivers/dax/super.c b/drivers/dax/super.c index cadbd0a1a1ef..5fa6ae9dbc8b 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -480,7 +480,7 @@ static void dax_free_inode(struct inode *inode) kfree(dax_dev->host); dax_dev->host = NULL; if (inode->i_rdev) - ida_simple_remove(&dax_minor_ida, MINOR(inode->i_rdev)); + ida_simple_remove(&dax_minor_ida, iminor(inode)); kmem_cache_free(dax_cache, dax_dev); } diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index 7475e09b0680..d64fc03929be 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -312,22 +312,25 @@ void __dma_fence_might_wait(void) /** - * dma_fence_signal_locked - signal completion of a fence + * dma_fence_signal_timestamp_locked - signal completion of a fence * @fence: the fence to signal + * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain * * Signal completion for software callbacks on a fence, this will unblock * dma_fence_wait() calls and run all the callbacks added with * dma_fence_add_callback(). Can be called multiple times, but since a fence * can only go from the unsignaled to the signaled state and not back, it will - * only be effective the first time. + * only be effective the first time. Set the timestamp provided as the fence + * signal timestamp. * - * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock - * held. + * Unlike dma_fence_signal_timestamp(), this function must be called with + * &dma_fence.lock held. * * Returns 0 on success and a negative error value when @fence has been * signalled already. */ -int dma_fence_signal_locked(struct dma_fence *fence) +int dma_fence_signal_timestamp_locked(struct dma_fence *fence, + ktime_t timestamp) { struct dma_fence_cb *cur, *tmp; struct list_head cb_list; @@ -341,7 +344,7 @@ int dma_fence_signal_locked(struct dma_fence *fence) /* Stash the cb_list before replacing it with the timestamp */ list_replace(&fence->cb_list, &cb_list); - fence->timestamp = ktime_get(); + fence->timestamp = timestamp; set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); trace_dma_fence_signaled(fence); @@ -352,6 +355,59 @@ int dma_fence_signal_locked(struct dma_fence *fence) return 0; } +EXPORT_SYMBOL(dma_fence_signal_timestamp_locked); + +/** + * dma_fence_signal_timestamp - signal completion of a fence + * @fence: the fence to signal + * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain + * + * Signal completion for software callbacks on a fence, this will unblock + * dma_fence_wait() calls and run all the callbacks added with + * dma_fence_add_callback(). Can be called multiple times, but since a fence + * can only go from the unsignaled to the signaled state and not back, it will + * only be effective the first time. Set the timestamp provided as the fence + * signal timestamp. + * + * Returns 0 on success and a negative error value when @fence has been + * signalled already. + */ +int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp) +{ + unsigned long flags; + int ret; + + if (!fence) + return -EINVAL; + + spin_lock_irqsave(fence->lock, flags); + ret = dma_fence_signal_timestamp_locked(fence, timestamp); + spin_unlock_irqrestore(fence->lock, flags); + + return ret; +} +EXPORT_SYMBOL(dma_fence_signal_timestamp); + +/** + * dma_fence_signal_locked - signal completion of a fence + * @fence: the fence to signal + * + * Signal completion for software callbacks on a fence, this will unblock + * dma_fence_wait() calls and run all the callbacks added with + * dma_fence_add_callback(). Can be called multiple times, but since a fence + * can only go from the unsignaled to the signaled state and not back, it will + * only be effective the first time. + * + * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock + * held. + * + * Returns 0 on success and a negative error value when @fence has been + * signalled already. + */ +int dma_fence_signal_locked(struct dma_fence *fence) +{ + return dma_fence_signal_timestamp_locked(fence, ktime_get()); +} EXPORT_SYMBOL(dma_fence_signal_locked); /** @@ -379,7 +435,7 @@ int dma_fence_signal(struct dma_fence *fence) tmp = dma_fence_begin_signalling(); spin_lock_irqsave(fence->lock, flags); - ret = dma_fence_signal_locked(fence); + ret = dma_fence_signal_timestamp_locked(fence, ktime_get()); spin_unlock_irqrestore(fence->lock, flags); dma_fence_end_signalling(tmp); diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c index afd22c9dbdcf..6b5db954569f 100644 --- a/drivers/dma-buf/dma-heap.c +++ b/drivers/dma-buf/dma-heap.c @@ -52,6 +52,9 @@ static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, unsigned int fd_flags, unsigned int heap_flags) { + struct dma_buf *dmabuf; + int fd; + /* * Allocations from all heaps have to begin * and end on page boundaries. @@ -60,7 +63,16 @@ static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, if (!len) return -EINVAL; - return heap->ops->allocate(heap, len, fd_flags, heap_flags); + dmabuf = heap->ops->allocate(heap, len, fd_flags, heap_flags); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + fd = dma_buf_fd(dmabuf, fd_flags); + if (fd < 0) { + dma_buf_put(dmabuf); + /* just return, as put will call release and that will free */ + } + return fd; } static int dma_heap_open(struct inode *inode, struct file *file) diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c index 364fc2f3e499..5d64eccd21d6 100644 --- a/drivers/dma-buf/heaps/cma_heap.c +++ b/drivers/dma-buf/heaps/cma_heap.c @@ -271,10 +271,10 @@ static const struct dma_buf_ops cma_heap_buf_ops = { .release = cma_heap_dma_buf_release, }; -static int cma_heap_allocate(struct dma_heap *heap, - unsigned long len, - unsigned long fd_flags, - unsigned long heap_flags) +static struct dma_buf *cma_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) { struct cma_heap *cma_heap = dma_heap_get_drvdata(heap); struct cma_heap_buffer *buffer; @@ -289,7 +289,7 @@ static int cma_heap_allocate(struct dma_heap *heap, buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); if (!buffer) - return -ENOMEM; + return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&buffer->attachments); mutex_init(&buffer->lock); @@ -348,15 +348,7 @@ static int cma_heap_allocate(struct dma_heap *heap, ret = PTR_ERR(dmabuf); goto free_pages; } - - ret = dma_buf_fd(dmabuf, fd_flags); - if (ret < 0) { - dma_buf_put(dmabuf); - /* just return, as put will call release and that will free */ - return ret; - } - - return ret; + return dmabuf; free_pages: kfree(buffer->pages); @@ -365,7 +357,7 @@ free_cma: free_buffer: kfree(buffer); - return ret; + return ERR_PTR(ret); } static const struct dma_heap_ops cma_heap_ops = { diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index 17e0e9a68baf..29e49ac17251 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -331,10 +331,10 @@ static struct page *alloc_largest_available(unsigned long size, return NULL; } -static int system_heap_allocate(struct dma_heap *heap, - unsigned long len, - unsigned long fd_flags, - unsigned long heap_flags) +static struct dma_buf *system_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) { struct system_heap_buffer *buffer; DEFINE_DMA_BUF_EXPORT_INFO(exp_info); @@ -349,7 +349,7 @@ static int system_heap_allocate(struct dma_heap *heap, buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); if (!buffer) - return -ENOMEM; + return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&buffer->attachments); mutex_init(&buffer->lock); @@ -363,8 +363,10 @@ static int system_heap_allocate(struct dma_heap *heap, * Avoid trying to allocate memory if the process * has been killed by SIGKILL */ - if (fatal_signal_pending(current)) + if (fatal_signal_pending(current)) { + ret = -EINTR; goto free_buffer; + } page = alloc_largest_available(size_remaining, max_order); if (!page) @@ -397,14 +399,7 @@ static int system_heap_allocate(struct dma_heap *heap, ret = PTR_ERR(dmabuf); goto free_pages; } - - ret = dma_buf_fd(dmabuf, fd_flags); - if (ret < 0) { - dma_buf_put(dmabuf); - /* just return, as put will call release and that will free */ - return ret; - } - return ret; + return dmabuf; free_pages: for_each_sgtable_sg(table, sg, i) { @@ -418,7 +413,7 @@ free_buffer: __free_pages(page, compound_order(page)); kfree(buffer); - return ret; + return ERR_PTR(ret); } static const struct dma_heap_ops system_heap_ops = { diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index d242c7632621..0c2827fd8c19 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -124,13 +124,6 @@ config BCM_SBA_RAID has the capability to offload memcpy, xor and pq computation for raid5/6. -config COH901318 - bool "ST-Ericsson COH901318 DMA support" - select DMA_ENGINE - depends on ARCH_U300 || COMPILE_TEST - help - Enable support for ST-Ericsson COH 901 318 DMA. - config DMA_BCM2835 tristate "BCM2835 DMA engine support" depends on ARCH_BCM2835 @@ -179,6 +172,7 @@ config DMA_SUN6I config DW_AXI_DMAC tristate "Synopsys DesignWare AXI DMA support" depends on OF || COMPILE_TEST + depends on HAS_IOMEM select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help @@ -378,14 +372,14 @@ config MILBEAUT_XDMAC XDMAC device. config MMP_PDMA - bool "MMP PDMA support" + tristate "MMP PDMA support" depends on ARCH_MMP || ARCH_PXA || COMPILE_TEST select DMA_ENGINE help Support the MMP PDMA engine for PXA and MMP platform. config MMP_TDMA - bool "MMP Two-Channel DMA support" + tristate "MMP Two-Channel DMA support" depends on ARCH_MMP || COMPILE_TEST select DMA_ENGINE select GENERIC_ALLOCATOR @@ -519,13 +513,6 @@ config PLX_DMA These are exposed via extra functions on the switch's upstream port. Each function exposes one DMA channel. -config SIRF_DMA - tristate "CSR SiRFprimaII/SiRFmarco DMA support" - depends on ARCH_SIRF - select DMA_ENGINE - help - Enable support for the CSR SiRFprimaII DMA engine. - config STE_DMA40 bool "ST-Ericsson DMA40 support" depends on ARCH_U8500 @@ -710,15 +697,6 @@ config XILINX_ZYNQMP_DPDMA driver provides the dmaengine required by the DisplayPort subsystem display driver. -config ZX_DMA - tristate "ZTE ZX DMA support" - depends on ARCH_ZX || COMPILE_TEST - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Support the DMA engine for ZTE ZX family platform devices. - - # driver files source "drivers/dma/bestcomm/Kconfig" @@ -740,6 +718,8 @@ source "drivers/dma/ti/Kconfig" source "drivers/dma/fsl-dpaa2-qdma/Kconfig" +source "drivers/dma/lgm/Kconfig" + # clients comment "DMA Clients" depends on DMA_ENGINE diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 948a8da05f8b..aa69094e3547 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -20,7 +20,6 @@ obj-$(CONFIG_AT_HDMAC) += at_hdmac.o obj-$(CONFIG_AT_XDMAC) += at_xdmac.o obj-$(CONFIG_AXI_DMAC) += dma-axi-dmac.o obj-$(CONFIG_BCM_SBA_RAID) += bcm-sba-raid.o -obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o obj-$(CONFIG_DMA_BCM2835) += bcm2835-dma.o obj-$(CONFIG_DMA_JZ4780) += dma-jz4780.o obj-$(CONFIG_DMA_SA11X0) += sa11x0-dma.o @@ -65,7 +64,6 @@ obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/ obj-$(CONFIG_PXA_DMA) += pxa_dma.o obj-$(CONFIG_RENESAS_DMA) += sh/ obj-$(CONFIG_SF_PDMA) += sf-pdma/ -obj-$(CONFIG_SIRF_DMA) += sirf-dma.o obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o obj-$(CONFIG_STM32_DMA) += stm32-dma.o obj-$(CONFIG_STM32_DMAMUX) += stm32-dmamux.o @@ -79,9 +77,9 @@ obj-$(CONFIG_TIMB_DMA) += timb_dma.o obj-$(CONFIG_UNIPHIER_MDMAC) += uniphier-mdmac.o obj-$(CONFIG_UNIPHIER_XDMAC) += uniphier-xdmac.o obj-$(CONFIG_XGENE_DMA) += xgene-dma.o -obj-$(CONFIG_ZX_DMA) += zx_dma.o obj-$(CONFIG_ST_FDMA) += st_fdma.o obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma/ +obj-$(CONFIG_INTEL_LDMA) += lgm/ obj-y += mediatek/ obj-y += qcom/ diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 7eaee5b705b1..30ae36124b1d 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -54,6 +54,25 @@ module_param(init_nr_desc_per_channel, uint, 0644); MODULE_PARM_DESC(init_nr_desc_per_channel, "initial descriptors per channel (default: 64)"); +/** + * struct at_dma_platform_data - Controller configuration parameters + * @nr_channels: Number of channels supported by hardware (max 8) + * @cap_mask: dma_capability flags supported by the platform + */ +struct at_dma_platform_data { + unsigned int nr_channels; + dma_cap_mask_t cap_mask; +}; + +/** + * struct at_dma_slave - Controller-specific information about a slave + * @dma_dev: required DMA master device + * @cfg: Platform-specific initializer for the CFG register + */ +struct at_dma_slave { + struct device *dma_dev; + u32 cfg; +}; /* prototypes */ static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx); diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h index 80fc2fe8c77e..4d1ebc040031 100644 --- a/drivers/dma/at_hdmac_regs.h +++ b/drivers/dma/at_hdmac_regs.h @@ -7,8 +7,6 @@ #ifndef AT_HDMAC_REGS_H #define AT_HDMAC_REGS_H -#include <linux/platform_data/dma-atmel.h> - #define AT_DMA_MAX_NR_CHANNELS 8 @@ -148,7 +146,31 @@ #define ATC_AUTO (0x1 << 31) /* Auto multiple buffer tx enable */ /* Bitfields in CFG */ -/* are in at_hdmac.h */ +#define ATC_PER_MSB(h) ((0x30U & (h)) >> 4) /* Extract most significant bits of a handshaking identifier */ + +#define ATC_SRC_PER(h) (0xFU & (h)) /* Channel src rq associated with periph handshaking ifc h */ +#define ATC_DST_PER(h) ((0xFU & (h)) << 4) /* Channel dst rq associated with periph handshaking ifc h */ +#define ATC_SRC_REP (0x1 << 8) /* Source Replay Mod */ +#define ATC_SRC_H2SEL (0x1 << 9) /* Source Handshaking Mod */ +#define ATC_SRC_H2SEL_SW (0x0 << 9) +#define ATC_SRC_H2SEL_HW (0x1 << 9) +#define ATC_SRC_PER_MSB(h) (ATC_PER_MSB(h) << 10) /* Channel src rq (most significant bits) */ +#define ATC_DST_REP (0x1 << 12) /* Destination Replay Mod */ +#define ATC_DST_H2SEL (0x1 << 13) /* Destination Handshaking Mod */ +#define ATC_DST_H2SEL_SW (0x0 << 13) +#define ATC_DST_H2SEL_HW (0x1 << 13) +#define ATC_DST_PER_MSB(h) (ATC_PER_MSB(h) << 14) /* Channel dst rq (most significant bits) */ +#define ATC_SOD (0x1 << 16) /* Stop On Done */ +#define ATC_LOCK_IF (0x1 << 20) /* Interface Lock */ +#define ATC_LOCK_B (0x1 << 21) /* AHB Bus Lock */ +#define ATC_LOCK_IF_L (0x1 << 22) /* Master Interface Arbiter Lock */ +#define ATC_LOCK_IF_L_CHUNK (0x0 << 22) +#define ATC_LOCK_IF_L_BUFFER (0x1 << 22) +#define ATC_AHB_PROT_MASK (0x7 << 24) /* AHB Protection */ +#define ATC_FIFOCFG_MASK (0x3 << 28) /* FIFO Request Configuration */ +#define ATC_FIFOCFG_LARGESTBURST (0x0 << 28) +#define ATC_FIFOCFG_HALFFIFO (0x1 << 28) +#define ATC_FIFOCFG_ENOUGHSPACE (0x2 << 28) /* Bitfields in SPIP */ #define ATC_SPIP_HOLE(x) (0xFFFFU & (x)) diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c deleted file mode 100644 index 95b9b2f5358e..000000000000 --- a/drivers/dma/coh901318.c +++ /dev/null @@ -1,2808 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * driver/dma/coh901318.c - * - * Copyright (C) 2007-2009 ST-Ericsson - * DMA driver for COH 901 318 - * Author: Per Friden <per.friden@stericsson.com> - */ - -#include <linux/init.h> -#include <linux/module.h> -#include <linux/kernel.h> /* printk() */ -#include <linux/fs.h> /* everything... */ -#include <linux/scatterlist.h> -#include <linux/slab.h> /* kmalloc() */ -#include <linux/dmaengine.h> -#include <linux/platform_device.h> -#include <linux/device.h> -#include <linux/irqreturn.h> -#include <linux/interrupt.h> -#include <linux/io.h> -#include <linux/uaccess.h> -#include <linux/debugfs.h> -#include <linux/platform_data/dma-coh901318.h> -#include <linux/of_dma.h> - -#include "coh901318.h" -#include "dmaengine.h" - -#define COH901318_MOD32_MASK (0x1F) -#define COH901318_WORD_MASK (0xFFFFFFFF) -/* INT_STATUS - Interrupt Status Registers 32bit (R/-) */ -#define COH901318_INT_STATUS1 (0x0000) -#define COH901318_INT_STATUS2 (0x0004) -/* TC_INT_STATUS - Terminal Count Interrupt Status Registers 32bit (R/-) */ -#define COH901318_TC_INT_STATUS1 (0x0008) -#define COH901318_TC_INT_STATUS2 (0x000C) -/* TC_INT_CLEAR - Terminal Count Interrupt Clear Registers 32bit (-/W) */ -#define COH901318_TC_INT_CLEAR1 (0x0010) -#define COH901318_TC_INT_CLEAR2 (0x0014) -/* RAW_TC_INT_STATUS - Raw Term Count Interrupt Status Registers 32bit (R/-) */ -#define COH901318_RAW_TC_INT_STATUS1 (0x0018) -#define COH901318_RAW_TC_INT_STATUS2 (0x001C) -/* BE_INT_STATUS - Bus Error Interrupt Status Registers 32bit (R/-) */ -#define COH901318_BE_INT_STATUS1 (0x0020) -#define COH901318_BE_INT_STATUS2 (0x0024) -/* BE_INT_CLEAR - Bus Error Interrupt Clear Registers 32bit (-/W) */ -#define COH901318_BE_INT_CLEAR1 (0x0028) -#define COH901318_BE_INT_CLEAR2 (0x002C) -/* RAW_BE_INT_STATUS - Raw Term Count Interrupt Status Registers 32bit (R/-) */ -#define COH901318_RAW_BE_INT_STATUS1 (0x0030) -#define COH901318_RAW_BE_INT_STATUS2 (0x0034) - -/* - * CX_CFG - Channel Configuration Registers 32bit (R/W) - */ -#define COH901318_CX_CFG (0x0100) -#define COH901318_CX_CFG_SPACING (0x04) -/* Channel enable activates tha dma job */ -#define COH901318_CX_CFG_CH_ENABLE (0x00000001) -#define COH901318_CX_CFG_CH_DISABLE (0x00000000) -/* Request Mode */ -#define COH901318_CX_CFG_RM_MASK (0x00000006) -#define COH901318_CX_CFG_RM_MEMORY_TO_MEMORY (0x0 << 1) -#define COH901318_CX_CFG_RM_PRIMARY_TO_MEMORY (0x1 << 1) -#define COH901318_CX_CFG_RM_MEMORY_TO_PRIMARY (0x1 << 1) -#define COH901318_CX_CFG_RM_PRIMARY_TO_SECONDARY (0x3 << 1) -#define COH901318_CX_CFG_RM_SECONDARY_TO_PRIMARY (0x3 << 1) -/* Linked channel request field. RM must == 11 */ -#define COH901318_CX_CFG_LCRF_SHIFT 3 -#define COH901318_CX_CFG_LCRF_MASK (0x000001F8) -#define COH901318_CX_CFG_LCR_DISABLE (0x00000000) -/* Terminal Counter Interrupt Request Mask */ -#define COH901318_CX_CFG_TC_IRQ_ENABLE (0x00000200) -#define COH901318_CX_CFG_TC_IRQ_DISABLE (0x00000000) -/* Bus Error interrupt Mask */ -#define COH901318_CX_CFG_BE_IRQ_ENABLE (0x00000400) -#define COH901318_CX_CFG_BE_IRQ_DISABLE (0x00000000) - -/* - * CX_STAT - Channel Status Registers 32bit (R/-) - */ -#define COH901318_CX_STAT (0x0200) -#define COH901318_CX_STAT_SPACING (0x04) -#define COH901318_CX_STAT_RBE_IRQ_IND (0x00000008) -#define COH901318_CX_STAT_RTC_IRQ_IND (0x00000004) -#define COH901318_CX_STAT_ACTIVE (0x00000002) -#define COH901318_CX_STAT_ENABLED (0x00000001) - -/* - * CX_CTRL - Channel Control Registers 32bit (R/W) - */ -#define COH901318_CX_CTRL (0x0400) -#define COH901318_CX_CTRL_SPACING (0x10) -/* Transfer Count Enable */ -#define COH901318_CX_CTRL_TC_ENABLE (0x00001000) -#define COH901318_CX_CTRL_TC_DISABLE (0x00000000) -/* Transfer Count Value 0 - 4095 */ -#define COH901318_CX_CTRL_TC_VALUE_MASK (0x00000FFF) -/* Burst count */ -#define COH901318_CX_CTRL_BURST_COUNT_MASK (0x0000E000) -#define COH901318_CX_CTRL_BURST_COUNT_64_BYTES (0x7 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_48_BYTES (0x6 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_32_BYTES (0x5 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_16_BYTES (0x4 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_8_BYTES (0x3 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_4_BYTES (0x2 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_2_BYTES (0x1 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_1_BYTE (0x0 << 13) -/* Source bus size */ -#define COH901318_CX_CTRL_SRC_BUS_SIZE_MASK (0x00030000) -#define COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS (0x2 << 16) -#define COH901318_CX_CTRL_SRC_BUS_SIZE_16_BITS (0x1 << 16) -#define COH901318_CX_CTRL_SRC_BUS_SIZE_8_BITS (0x0 << 16) -/* Source address increment */ -#define COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE (0x00040000) -#define COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE (0x00000000) -/* Destination Bus Size */ -#define COH901318_CX_CTRL_DST_BUS_SIZE_MASK (0x00180000) -#define COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS (0x2 << 19) -#define COH901318_CX_CTRL_DST_BUS_SIZE_16_BITS (0x1 << 19) -#define COH901318_CX_CTRL_DST_BUS_SIZE_8_BITS (0x0 << 19) -/* Destination address increment */ -#define COH901318_CX_CTRL_DST_ADDR_INC_ENABLE (0x00200000) -#define COH901318_CX_CTRL_DST_ADDR_INC_DISABLE (0x00000000) -/* Master Mode (Master2 is only connected to MSL) */ -#define COH901318_CX_CTRL_MASTER_MODE_MASK (0x00C00000) -#define COH901318_CX_CTRL_MASTER_MODE_M2R_M1W (0x3 << 22) -#define COH901318_CX_CTRL_MASTER_MODE_M1R_M2W (0x2 << 22) -#define COH901318_CX_CTRL_MASTER_MODE_M2RW (0x1 << 22) -#define COH901318_CX_CTRL_MASTER_MODE_M1RW (0x0 << 22) -/* Terminal Count flag to PER enable */ -#define COH901318_CX_CTRL_TCP_ENABLE (0x01000000) -#define COH901318_CX_CTRL_TCP_DISABLE (0x00000000) -/* Terminal Count flags to CPU enable */ -#define COH901318_CX_CTRL_TC_IRQ_ENABLE (0x02000000) -#define COH901318_CX_CTRL_TC_IRQ_DISABLE (0x00000000) -/* Hand shake to peripheral */ -#define COH901318_CX_CTRL_HSP_ENABLE (0x04000000) -#define COH901318_CX_CTRL_HSP_DISABLE (0x00000000) -#define COH901318_CX_CTRL_HSS_ENABLE (0x08000000) -#define COH901318_CX_CTRL_HSS_DISABLE (0x00000000) -/* DMA mode */ -#define COH901318_CX_CTRL_DDMA_MASK (0x30000000) -#define COH901318_CX_CTRL_DDMA_LEGACY (0x0 << 28) -#define COH901318_CX_CTRL_DDMA_DEMAND_DMA1 (0x1 << 28) -#define COH901318_CX_CTRL_DDMA_DEMAND_DMA2 (0x2 << 28) -/* Primary Request Data Destination */ -#define COH901318_CX_CTRL_PRDD_MASK (0x40000000) -#define COH901318_CX_CTRL_PRDD_DEST (0x1 << 30) -#define COH901318_CX_CTRL_PRDD_SOURCE (0x0 << 30) - -/* - * CX_SRC_ADDR - Channel Source Address Registers 32bit (R/W) - */ -#define COH901318_CX_SRC_ADDR (0x0404) -#define COH901318_CX_SRC_ADDR_SPACING (0x10) - -/* - * CX_DST_ADDR - Channel Destination Address Registers 32bit R/W - */ -#define COH901318_CX_DST_ADDR (0x0408) -#define COH901318_CX_DST_ADDR_SPACING (0x10) - -/* - * CX_LNK_ADDR - Channel Link Address Registers 32bit (R/W) - */ -#define COH901318_CX_LNK_ADDR (0x040C) -#define COH901318_CX_LNK_ADDR_SPACING (0x10) -#define COH901318_CX_LNK_LINK_IMMEDIATE (0x00000001) - -/** - * struct coh901318_params - parameters for DMAC configuration - * @config: DMA config register - * @ctrl_lli_last: DMA control register for the last lli in the list - * @ctrl_lli: DMA control register for an lli - * @ctrl_lli_chained: DMA control register for a chained lli - */ -struct coh901318_params { - u32 config; - u32 ctrl_lli_last; - u32 ctrl_lli; - u32 ctrl_lli_chained; -}; - -/** - * struct coh_dma_channel - dma channel base - * @name: ascii name of dma channel - * @number: channel id number - * @desc_nbr_max: number of preallocated descriptors - * @priority_high: prio of channel, 0 low otherwise high. - * @param: configuration parameters - */ -struct coh_dma_channel { - const char name[32]; - const int number; - const int desc_nbr_max; - const int priority_high; - const struct coh901318_params param; -}; - -/** - * struct powersave - DMA power save structure - * @lock: lock protecting data in this struct - * @started_channels: bit mask indicating active dma channels - */ -struct powersave { - spinlock_t lock; - u64 started_channels; -}; - -/* points out all dma slave channels. - * Syntax is [A1, B1, A2, B2, .... ,-1,-1] - * Select all channels from A to B, end of list is marked with -1,-1 - */ -static int dma_slave_channels[] = { - U300_DMA_MSL_TX_0, U300_DMA_SPI_RX, - U300_DMA_UART1_TX, U300_DMA_UART1_RX, -1, -1}; - -/* points out all dma memcpy channels. */ -static int dma_memcpy_channels[] = { - U300_DMA_GENERAL_PURPOSE_0, U300_DMA_GENERAL_PURPOSE_8, -1, -1}; - -#define flags_memcpy_config (COH901318_CX_CFG_CH_DISABLE | \ - COH901318_CX_CFG_RM_MEMORY_TO_MEMORY | \ - COH901318_CX_CFG_LCR_DISABLE | \ - COH901318_CX_CFG_TC_IRQ_ENABLE | \ - COH901318_CX_CFG_BE_IRQ_ENABLE) -#define flags_memcpy_lli_chained (COH901318_CX_CTRL_TC_ENABLE | \ - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | \ - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_MASTER_MODE_M1RW | \ - COH901318_CX_CTRL_TCP_DISABLE | \ - COH901318_CX_CTRL_TC_IRQ_DISABLE | \ - COH901318_CX_CTRL_HSP_DISABLE | \ - COH901318_CX_CTRL_HSS_DISABLE | \ - COH901318_CX_CTRL_DDMA_LEGACY | \ - COH901318_CX_CTRL_PRDD_SOURCE) -#define flags_memcpy_lli (COH901318_CX_CTRL_TC_ENABLE | \ - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | \ - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_MASTER_MODE_M1RW | \ - COH901318_CX_CTRL_TCP_DISABLE | \ - COH901318_CX_CTRL_TC_IRQ_DISABLE | \ - COH901318_CX_CTRL_HSP_DISABLE | \ - COH901318_CX_CTRL_HSS_DISABLE | \ - COH901318_CX_CTRL_DDMA_LEGACY | \ - COH901318_CX_CTRL_PRDD_SOURCE) -#define flags_memcpy_lli_last (COH901318_CX_CTRL_TC_ENABLE | \ - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | \ - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_MASTER_MODE_M1RW | \ - COH901318_CX_CTRL_TCP_DISABLE | \ - COH901318_CX_CTRL_TC_IRQ_ENABLE | \ - COH901318_CX_CTRL_HSP_DISABLE | \ - COH901318_CX_CTRL_HSS_DISABLE | \ - COH901318_CX_CTRL_DDMA_LEGACY | \ - COH901318_CX_CTRL_PRDD_SOURCE) - -static const struct coh_dma_channel chan_config[U300_DMA_CHANNELS] = { - { - .number = U300_DMA_MSL_TX_0, - .name = "MSL TX 0", - .priority_high = 0, - }, - { - .number = U300_DMA_MSL_TX_1, - .name = "MSL TX 1", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_MSL_TX_2, - .name = "MSL TX 2", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .desc_nbr_max = 10, - }, - { - .number = U300_DMA_MSL_TX_3, - .name = "MSL TX 3", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_MSL_TX_4, - .name = "MSL TX 4", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_MSL_TX_5, - .name = "MSL TX 5", - .priority_high = 0, - }, - { - .number = U300_DMA_MSL_TX_6, - .name = "MSL TX 6", - .priority_high = 0, - }, - { - .number = U300_DMA_MSL_RX_0, - .name = "MSL RX 0", - .priority_high = 0, - }, - { - .number = U300_DMA_MSL_RX_1, - .name = "MSL RX 1", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_2, - .name = "MSL RX 2", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_3, - .name = "MSL RX 3", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_4, - .name = "MSL RX 4", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_5, - .name = "MSL RX 5", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_6, - .name = "MSL RX 6", - .priority_high = 0, - }, - /* - * Don't set up device address, burst count or size of src - * or dst bus for this peripheral - handled by PrimeCell - * DMA extension. - */ - { - .number = U300_DMA_MMCSD_RX_TX, - .name = "MMCSD RX TX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - - }, - { - .number = U300_DMA_MSPRO_TX, - .name = "MSPRO TX", - .priority_high = 0, - }, - { - .number = U300_DMA_MSPRO_RX, - .name = "MSPRO RX", - .priority_high = 0, - }, - /* - * Don't set up device address, burst count or size of src - * or dst bus for this peripheral - handled by PrimeCell - * DMA extension. - */ - { - .number = U300_DMA_UART0_TX, - .name = "UART0 TX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - }, - { - .number = U300_DMA_UART0_RX, - .name = "UART0 RX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - }, - { - .number = U300_DMA_APEX_TX, - .name = "APEX TX", - .priority_high = 0, - }, - { - .number = U300_DMA_APEX_RX, - .name = "APEX RX", - .priority_high = 0, - }, - { - .number = U300_DMA_PCM_I2S0_TX, - .name = "PCM I2S0 TX", - .priority_high = 1, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_PCM_I2S0_RX, - .name = "PCM I2S0 RX", - .priority_high = 1, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_PCM_I2S1_TX, - .name = "PCM I2S1 TX", - .priority_high = 1, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_PCM_I2S1_RX, - .name = "PCM I2S1 RX", - .priority_high = 1, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_XGAM_CDI, - .name = "XGAM CDI", - .priority_high = 0, - }, - { - .number = U300_DMA_XGAM_PDI, - .name = "XGAM PDI", - .priority_high = 0, - }, - /* - * Don't set up device address, burst count or size of src - * or dst bus for this peripheral - handled by PrimeCell - * DMA extension. - */ - { - .number = U300_DMA_SPI_TX, - .name = "SPI TX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - }, - { - .number = U300_DMA_SPI_RX, - .name = "SPI RX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - - }, - { - .number = U300_DMA_GENERAL_PURPOSE_0, - .name = "GENERAL 00", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_1, - .name = "GENERAL 01", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_2, - .name = "GENERAL 02", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_3, - .name = "GENERAL 03", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_4, - .name = "GENERAL 04", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_5, - .name = "GENERAL 05", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_6, - .name = "GENERAL 06", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_7, - .name = "GENERAL 07", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_8, - .name = "GENERAL 08", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_UART1_TX, - .name = "UART1 TX", - .priority_high = 0, - }, - { - .number = U300_DMA_UART1_RX, - .name = "UART1 RX", - .priority_high = 0, - } -}; - -#define COHC_2_DEV(cohc) (&cohc->chan.dev->device) - -#ifdef VERBOSE_DEBUG -#define COH_DBG(x) ({ if (1) x; 0; }) -#else -#define COH_DBG(x) ({ if (0) x; 0; }) -#endif - -struct coh901318_desc { - struct dma_async_tx_descriptor desc; - struct list_head node; - struct scatterlist *sg; - unsigned int sg_len; - struct coh901318_lli *lli; - enum dma_transfer_direction dir; - unsigned long flags; - u32 head_config; - u32 head_ctrl; -}; - -struct coh901318_base { - struct device *dev; - void __iomem *virtbase; - unsigned int irq; - struct coh901318_pool pool; - struct powersave pm; - struct dma_device dma_slave; - struct dma_device dma_memcpy; - struct coh901318_chan *chans; -}; - -struct coh901318_chan { - spinlock_t lock; - int allocated; - int id; - int stopped; - - struct work_struct free_work; - struct dma_chan chan; - - struct tasklet_struct tasklet; - - struct list_head active; - struct list_head queue; - struct list_head free; - - unsigned long nbr_active_done; - unsigned long busy; - - struct dma_slave_config config; - u32 addr; - u32 ctrl; - - struct coh901318_base *base; -}; - -static void coh901318_list_print(struct coh901318_chan *cohc, - struct coh901318_lli *lli) -{ - struct coh901318_lli *l = lli; - int i = 0; - - while (l) { - dev_vdbg(COHC_2_DEV(cohc), "i %d, lli %p, ctrl 0x%x, src %pad" - ", dst %pad, link %pad virt_link_addr 0x%p\n", - i, l, l->control, &l->src_addr, &l->dst_addr, - &l->link_addr, l->virt_link_addr); - i++; - l = l->virt_link_addr; - } -} - -#ifdef CONFIG_DEBUG_FS - -#define COH901318_DEBUGFS_ASSIGN(x, y) (x = y) - -static struct coh901318_base *debugfs_dma_base; -static struct dentry *dma_dentry; - -static ssize_t coh901318_debugfs_read(struct file *file, char __user *buf, - size_t count, loff_t *f_pos) -{ - u64 started_channels = debugfs_dma_base->pm.started_channels; - int pool_count = debugfs_dma_base->pool.debugfs_pool_counter; - char *dev_buf; - char *tmp; - int ret; - int i; - - dev_buf = kmalloc(4*1024, GFP_KERNEL); - if (dev_buf == NULL) - return -ENOMEM; - tmp = dev_buf; - - tmp += sprintf(tmp, "DMA -- enabled dma channels\n"); - - for (i = 0; i < U300_DMA_CHANNELS; i++) { - if (started_channels & (1ULL << i)) - tmp += sprintf(tmp, "channel %d\n", i); - } - - tmp += sprintf(tmp, "Pool alloc nbr %d\n", pool_count); - - ret = simple_read_from_buffer(buf, count, f_pos, dev_buf, - tmp - dev_buf); - kfree(dev_buf); - return ret; -} - -static const struct file_operations coh901318_debugfs_status_operations = { - .open = simple_open, - .read = coh901318_debugfs_read, - .llseek = default_llseek, -}; - - -static int __init init_coh901318_debugfs(void) -{ - - dma_dentry = debugfs_create_dir("dma", NULL); - - debugfs_create_file("status", S_IFREG | S_IRUGO, dma_dentry, NULL, - &coh901318_debugfs_status_operations); - return 0; -} - -static void __exit exit_coh901318_debugfs(void) -{ - debugfs_remove_recursive(dma_dentry); -} - -module_init(init_coh901318_debugfs); -module_exit(exit_coh901318_debugfs); -#else - -#define COH901318_DEBUGFS_ASSIGN(x, y) - -#endif /* CONFIG_DEBUG_FS */ - -static inline struct coh901318_chan *to_coh901318_chan(struct dma_chan *chan) -{ - return container_of(chan, struct coh901318_chan, chan); -} - -static int coh901318_dma_set_runtimeconfig(struct dma_chan *chan, - struct dma_slave_config *config, - enum dma_transfer_direction direction); - -static inline const struct coh901318_params * -cohc_chan_param(struct coh901318_chan *cohc) -{ - return &chan_config[cohc->id].param; -} - -static inline const struct coh_dma_channel * -cohc_chan_conf(struct coh901318_chan *cohc) -{ - return &chan_config[cohc->id]; -} - -static void enable_powersave(struct coh901318_chan *cohc) -{ - unsigned long flags; - struct powersave *pm = &cohc->base->pm; - - spin_lock_irqsave(&pm->lock, flags); - - pm->started_channels &= ~(1ULL << cohc->id); - - spin_unlock_irqrestore(&pm->lock, flags); -} -static void disable_powersave(struct coh901318_chan *cohc) -{ - unsigned long flags; - struct powersave *pm = &cohc->base->pm; - - spin_lock_irqsave(&pm->lock, flags); - - pm->started_channels |= (1ULL << cohc->id); - - spin_unlock_irqrestore(&pm->lock, flags); -} - -static inline int coh901318_set_ctrl(struct coh901318_chan *cohc, u32 control) -{ - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - writel(control, - virtbase + COH901318_CX_CTRL + - COH901318_CX_CTRL_SPACING * channel); - return 0; -} - -static inline int coh901318_set_conf(struct coh901318_chan *cohc, u32 conf) -{ - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - writel(conf, - virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING*channel); - return 0; -} - - -static int coh901318_start(struct coh901318_chan *cohc) -{ - u32 val; - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - disable_powersave(cohc); - - val = readl(virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - /* Enable channel */ - val |= COH901318_CX_CFG_CH_ENABLE; - writel(val, virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - return 0; -} - -static int coh901318_prep_linked_list(struct coh901318_chan *cohc, - struct coh901318_lli *lli) -{ - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - BUG_ON(readl(virtbase + COH901318_CX_STAT + - COH901318_CX_STAT_SPACING*channel) & - COH901318_CX_STAT_ACTIVE); - - writel(lli->src_addr, - virtbase + COH901318_CX_SRC_ADDR + - COH901318_CX_SRC_ADDR_SPACING * channel); - - writel(lli->dst_addr, virtbase + - COH901318_CX_DST_ADDR + - COH901318_CX_DST_ADDR_SPACING * channel); - - writel(lli->link_addr, virtbase + COH901318_CX_LNK_ADDR + - COH901318_CX_LNK_ADDR_SPACING * channel); - - writel(lli->control, virtbase + COH901318_CX_CTRL + - COH901318_CX_CTRL_SPACING * channel); - - return 0; -} - -static struct coh901318_desc * -coh901318_desc_get(struct coh901318_chan *cohc) -{ - struct coh901318_desc *desc; - - if (list_empty(&cohc->free)) { - /* alloc new desc because we're out of used ones - * TODO: alloc a pile of descs instead of just one, - * avoid many small allocations. - */ - desc = kzalloc(sizeof(struct coh901318_desc), GFP_NOWAIT); - if (desc == NULL) - goto out; - INIT_LIST_HEAD(&desc->node); - dma_async_tx_descriptor_init(&desc->desc, &cohc->chan); - } else { - /* Reuse an old desc. */ - desc = list_first_entry(&cohc->free, - struct coh901318_desc, - node); - list_del(&desc->node); - /* Initialize it a bit so it's not insane */ - desc->sg = NULL; - desc->sg_len = 0; - desc->desc.callback = NULL; - desc->desc.callback_param = NULL; - } - - out: - return desc; -} - -static void -coh901318_desc_free(struct coh901318_chan *cohc, struct coh901318_desc *cohd) -{ - list_add_tail(&cohd->node, &cohc->free); -} - -/* call with irq lock held */ -static void -coh901318_desc_submit(struct coh901318_chan *cohc, struct coh901318_desc *desc) -{ - list_add_tail(&desc->node, &cohc->active); -} - -static struct coh901318_desc * -coh901318_first_active_get(struct coh901318_chan *cohc) -{ - return list_first_entry_or_null(&cohc->active, struct coh901318_desc, - node); -} - -static void -coh901318_desc_remove(struct coh901318_desc *cohd) -{ - list_del(&cohd->node); -} - -static void -coh901318_desc_queue(struct coh901318_chan *cohc, struct coh901318_desc *desc) -{ - list_add_tail(&desc->node, &cohc->queue); -} - -static struct coh901318_desc * -coh901318_first_queued(struct coh901318_chan *cohc) -{ - return list_first_entry_or_null(&cohc->queue, struct coh901318_desc, - node); -} - -static inline u32 coh901318_get_bytes_in_lli(struct coh901318_lli *in_lli) -{ - struct coh901318_lli *lli = in_lli; - u32 bytes = 0; - - while (lli) { - bytes += lli->control & COH901318_CX_CTRL_TC_VALUE_MASK; - lli = lli->virt_link_addr; - } - return bytes; -} - -/* - * Get the number of bytes left to transfer on this channel, - * it is unwise to call this before stopping the channel for - * absolute measures, but for a rough guess you can still call - * it. - */ -static u32 coh901318_get_bytes_left(struct dma_chan *chan) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - struct coh901318_desc *cohd; - struct list_head *pos; - unsigned long flags; - u32 left = 0; - int i = 0; - - spin_lock_irqsave(&cohc->lock, flags); - - /* - * If there are many queued jobs, we iterate and add the - * size of them all. We take a special look on the first - * job though, since it is probably active. - */ - list_for_each(pos, &cohc->active) { - /* - * The first job in the list will be working on the - * hardware. The job can be stopped but still active, - * so that the transfer counter is somewhere inside - * the buffer. - */ - cohd = list_entry(pos, struct coh901318_desc, node); - - if (i == 0) { - struct coh901318_lli *lli; - dma_addr_t ladd; - - /* Read current transfer count value */ - left = readl(cohc->base->virtbase + - COH901318_CX_CTRL + - COH901318_CX_CTRL_SPACING * cohc->id) & - COH901318_CX_CTRL_TC_VALUE_MASK; - - /* See if the transfer is linked... */ - ladd = readl(cohc->base->virtbase + - COH901318_CX_LNK_ADDR + - COH901318_CX_LNK_ADDR_SPACING * - cohc->id) & - ~COH901318_CX_LNK_LINK_IMMEDIATE; - /* Single transaction */ - if (!ladd) - continue; - - /* - * Linked transaction, follow the lli, find the - * currently processing lli, and proceed to the next - */ - lli = cohd->lli; - while (lli && lli->link_addr != ladd) - lli = lli->virt_link_addr; - - if (lli) - lli = lli->virt_link_addr; - - /* - * Follow remaining lli links around to count the total - * number of bytes left - */ - left += coh901318_get_bytes_in_lli(lli); - } else { - left += coh901318_get_bytes_in_lli(cohd->lli); - } - i++; - } - - /* Also count bytes in the queued jobs */ - list_for_each(pos, &cohc->queue) { - cohd = list_entry(pos, struct coh901318_desc, node); - left += coh901318_get_bytes_in_lli(cohd->lli); - } - - spin_unlock_irqrestore(&cohc->lock, flags); - - return left; -} - -/* - * Pauses a transfer without losing data. Enables power save. - * Use this function in conjunction with coh901318_resume. - */ -static int coh901318_pause(struct dma_chan *chan) -{ - u32 val; - unsigned long flags; - struct coh901318_chan *cohc = to_coh901318_chan(chan); - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - spin_lock_irqsave(&cohc->lock, flags); - - /* Disable channel in HW */ - val = readl(virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - /* Stopping infinite transfer */ - if ((val & COH901318_CX_CTRL_TC_ENABLE) == 0 && - (val & COH901318_CX_CFG_CH_ENABLE)) - cohc->stopped = 1; - - - val &= ~COH901318_CX_CFG_CH_ENABLE; - /* Enable twice, HW bug work around */ - writel(val, virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - writel(val, virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - /* Spin-wait for it to actually go inactive */ - while (readl(virtbase + COH901318_CX_STAT+COH901318_CX_STAT_SPACING * - channel) & COH901318_CX_STAT_ACTIVE) - cpu_relax(); - - /* Check if we stopped an active job */ - if ((readl(virtbase + COH901318_CX_CTRL+COH901318_CX_CTRL_SPACING * - channel) & COH901318_CX_CTRL_TC_VALUE_MASK) > 0) - cohc->stopped = 1; - - enable_powersave(cohc); - - spin_unlock_irqrestore(&cohc->lock, flags); - return 0; -} - -/* Resumes a transfer that has been stopped via 300_dma_stop(..). - Power save is handled. -*/ -static int coh901318_resume(struct dma_chan *chan) -{ - u32 val; - unsigned long flags; - struct coh901318_chan *cohc = to_coh901318_chan(chan); - int channel = cohc->id; - - spin_lock_irqsave(&cohc->lock, flags); - - disable_powersave(cohc); - - if (cohc->stopped) { - /* Enable channel in HW */ - val = readl(cohc->base->virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - val |= COH901318_CX_CFG_CH_ENABLE; - - writel(val, cohc->base->virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING*channel); - - cohc->stopped = 0; - } - - spin_unlock_irqrestore(&cohc->lock, flags); - return 0; -} - -bool coh901318_filter_id(struct dma_chan *chan, void *chan_id) -{ - unsigned long ch_nr = (unsigned long) chan_id; - - if (ch_nr == to_coh901318_chan(chan)->id) - return true; - - return false; -} -EXPORT_SYMBOL(coh901318_filter_id); - -struct coh901318_filter_args { - struct coh901318_base *base; - unsigned int ch_nr; -}; - -static bool coh901318_filter_base_and_id(struct dma_chan *chan, void *data) -{ - struct coh901318_filter_args *args = data; - - if (&args->base->dma_slave == chan->device && - args->ch_nr == to_coh901318_chan(chan)->id) - return true; - - return false; -} - -static struct dma_chan *coh901318_xlate(struct of_phandle_args *dma_spec, - struct of_dma *ofdma) -{ - struct coh901318_filter_args args = { - .base = ofdma->of_dma_data, - .ch_nr = dma_spec->args[0], - }; - dma_cap_mask_t cap; - dma_cap_zero(cap); - dma_cap_set(DMA_SLAVE, cap); - - return dma_request_channel(cap, coh901318_filter_base_and_id, &args); -} -/* - * DMA channel allocation - */ -static int coh901318_config(struct coh901318_chan *cohc, - struct coh901318_params *param) -{ - const struct coh901318_params *p; - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - if (param) - p = param; - else - p = cohc_chan_param(cohc); - - /* Clear any pending BE or TC interrupt */ - if (channel < 32) { - writel(1 << channel, virtbase + COH901318_BE_INT_CLEAR1); - writel(1 << channel, virtbase + COH901318_TC_INT_CLEAR1); - } else { - writel(1 << (channel - 32), virtbase + - COH901318_BE_INT_CLEAR2); - writel(1 << (channel - 32), virtbase + - COH901318_TC_INT_CLEAR2); - } - - coh901318_set_conf(cohc, p->config); - coh901318_set_ctrl(cohc, p->ctrl_lli_last); - - return 0; -} - -/* must lock when calling this function - * start queued jobs, if any - * TODO: start all queued jobs in one go - * - * Returns descriptor if queued job is started otherwise NULL. - * If the queue is empty NULL is returned. - */ -static struct coh901318_desc *coh901318_queue_start(struct coh901318_chan *cohc) -{ - struct coh901318_desc *cohd; - - /* - * start queued jobs, if any - * TODO: transmit all queued jobs in one go - */ - cohd = coh901318_first_queued(cohc); - - if (cohd != NULL) { - /* Remove from queue */ - coh901318_desc_remove(cohd); - /* initiate DMA job */ - cohc->busy = 1; - - coh901318_desc_submit(cohc, cohd); - - /* Program the transaction head */ - coh901318_set_conf(cohc, cohd->head_config); - coh901318_set_ctrl(cohc, cohd->head_ctrl); - coh901318_prep_linked_list(cohc, cohd->lli); - - /* start dma job on this channel */ - coh901318_start(cohc); - - } - - return cohd; -} - -/* - * This tasklet is called from the interrupt handler to - * handle each descriptor (DMA job) that is sent to a channel. - */ -static void dma_tasklet(struct tasklet_struct *t) -{ - struct coh901318_chan *cohc = from_tasklet(cohc, t, tasklet); - struct coh901318_desc *cohd_fin; - unsigned long flags; - struct dmaengine_desc_callback cb; - - dev_vdbg(COHC_2_DEV(cohc), "[%s] chan_id %d" - " nbr_active_done %ld\n", __func__, - cohc->id, cohc->nbr_active_done); - - spin_lock_irqsave(&cohc->lock, flags); - - /* get first active descriptor entry from list */ - cohd_fin = coh901318_first_active_get(cohc); - - if (cohd_fin == NULL) - goto err; - - /* locate callback to client */ - dmaengine_desc_get_callback(&cohd_fin->desc, &cb); - - /* sign this job as completed on the channel */ - dma_cookie_complete(&cohd_fin->desc); - - /* release the lli allocation and remove the descriptor */ - coh901318_lli_free(&cohc->base->pool, &cohd_fin->lli); - - /* return desc to free-list */ - coh901318_desc_remove(cohd_fin); - coh901318_desc_free(cohc, cohd_fin); - - spin_unlock_irqrestore(&cohc->lock, flags); - - /* Call the callback when we're done */ - dmaengine_desc_callback_invoke(&cb, NULL); - - spin_lock_irqsave(&cohc->lock, flags); - - /* - * If another interrupt fired while the tasklet was scheduling, - * we don't get called twice, so we have this number of active - * counter that keep track of the number of IRQs expected to - * be handled for this channel. If there happen to be more than - * one IRQ to be ack:ed, we simply schedule this tasklet again. - */ - cohc->nbr_active_done--; - if (cohc->nbr_active_done) { - dev_dbg(COHC_2_DEV(cohc), "scheduling tasklet again, new IRQs " - "came in while we were scheduling this tasklet\n"); - if (cohc_chan_conf(cohc)->priority_high) - tasklet_hi_schedule(&cohc->tasklet); - else - tasklet_schedule(&cohc->tasklet); - } - - spin_unlock_irqrestore(&cohc->lock, flags); - - return; - - err: - spin_unlock_irqrestore(&cohc->lock, flags); - dev_err(COHC_2_DEV(cohc), "[%s] No active dma desc\n", __func__); -} - - -/* called from interrupt context */ -static void dma_tc_handle(struct coh901318_chan *cohc) -{ - /* - * If the channel is not allocated, then we shouldn't have - * any TC interrupts on it. - */ - if (!cohc->allocated) { - dev_err(COHC_2_DEV(cohc), "spurious interrupt from " - "unallocated channel\n"); - return; - } - - /* - * When we reach this point, at least one queue item - * should have been moved over from cohc->queue to - * cohc->active and run to completion, that is why we're - * getting a terminal count interrupt is it not? - * If you get this BUG() the most probable cause is that - * the individual nodes in the lli chain have IRQ enabled, - * so check your platform config for lli chain ctrl. - */ - BUG_ON(list_empty(&cohc->active)); - - cohc->nbr_active_done++; - - /* - * This attempt to take a job from cohc->queue, put it - * into cohc->active and start it. - */ - if (coh901318_queue_start(cohc) == NULL) - cohc->busy = 0; - - /* - * This tasklet will remove items from cohc->active - * and thus terminates them. - */ - if (cohc_chan_conf(cohc)->priority_high) - tasklet_hi_schedule(&cohc->tasklet); - else - tasklet_schedule(&cohc->tasklet); -} - - -static irqreturn_t dma_irq_handler(int irq, void *dev_id) -{ - u32 status1; - u32 status2; - int i; - int ch; - struct coh901318_base *base = dev_id; - struct coh901318_chan *cohc; - void __iomem *virtbase = base->virtbase; - - status1 = readl(virtbase + COH901318_INT_STATUS1); - status2 = readl(virtbase + COH901318_INT_STATUS2); - - if (unlikely(status1 == 0 && status2 == 0)) { - dev_warn(base->dev, "spurious DMA IRQ from no channel!\n"); - return IRQ_HANDLED; - } - - /* TODO: consider handle IRQ in tasklet here to - * minimize interrupt latency */ - - /* Check the first 32 DMA channels for IRQ */ - while (status1) { - /* Find first bit set, return as a number. */ - i = ffs(status1) - 1; - ch = i; - - cohc = &base->chans[ch]; - spin_lock(&cohc->lock); - - /* Mask off this bit */ - status1 &= ~(1 << i); - /* Check the individual channel bits */ - if (test_bit(i, virtbase + COH901318_BE_INT_STATUS1)) { - dev_crit(COHC_2_DEV(cohc), - "DMA bus error on channel %d!\n", ch); - BUG_ON(1); - /* Clear BE interrupt */ - __set_bit(i, virtbase + COH901318_BE_INT_CLEAR1); - } else { - /* Caused by TC, really? */ - if (unlikely(!test_bit(i, virtbase + - COH901318_TC_INT_STATUS1))) { - dev_warn(COHC_2_DEV(cohc), - "ignoring interrupt not caused by terminal count on channel %d\n", ch); - /* Clear TC interrupt */ - BUG_ON(1); - __set_bit(i, virtbase + COH901318_TC_INT_CLEAR1); - } else { - /* Enable powersave if transfer has finished */ - if (!(readl(virtbase + COH901318_CX_STAT + - COH901318_CX_STAT_SPACING*ch) & - COH901318_CX_STAT_ENABLED)) { - enable_powersave(cohc); - } - - /* Must clear TC interrupt before calling - * dma_tc_handle - * in case tc_handle initiate a new dma job - */ - __set_bit(i, virtbase + COH901318_TC_INT_CLEAR1); - - dma_tc_handle(cohc); - } - } - spin_unlock(&cohc->lock); - } - - /* Check the remaining 32 DMA channels for IRQ */ - while (status2) { - /* Find first bit set, return as a number. */ - i = ffs(status2) - 1; - ch = i + 32; - cohc = &base->chans[ch]; - spin_lock(&cohc->lock); - - /* Mask off this bit */ - status2 &= ~(1 << i); - /* Check the individual channel bits */ - if (test_bit(i, virtbase + COH901318_BE_INT_STATUS2)) { - dev_crit(COHC_2_DEV(cohc), - "DMA bus error on channel %d!\n", ch); - /* Clear BE interrupt */ - BUG_ON(1); - __set_bit(i, virtbase + COH901318_BE_INT_CLEAR2); - } else { - /* Caused by TC, really? */ - if (unlikely(!test_bit(i, virtbase + - COH901318_TC_INT_STATUS2))) { - dev_warn(COHC_2_DEV(cohc), - "ignoring interrupt not caused by terminal count on channel %d\n", ch); - /* Clear TC interrupt */ - __set_bit(i, virtbase + COH901318_TC_INT_CLEAR2); - BUG_ON(1); - } else { - /* Enable powersave if transfer has finished */ - if (!(readl(virtbase + COH901318_CX_STAT + - COH901318_CX_STAT_SPACING*ch) & - COH901318_CX_STAT_ENABLED)) { - enable_powersave(cohc); - } - /* Must clear TC interrupt before calling - * dma_tc_handle - * in case tc_handle initiate a new dma job - */ - __set_bit(i, virtbase + COH901318_TC_INT_CLEAR2); - - dma_tc_handle(cohc); - } - } - spin_unlock(&cohc->lock); - } - - return IRQ_HANDLED; -} - -static int coh901318_terminate_all(struct dma_chan *chan) -{ - unsigned long flags; - struct coh901318_chan *cohc = to_coh901318_chan(chan); - struct coh901318_desc *cohd; - void __iomem *virtbase = cohc->base->virtbase; - - /* The remainder of this function terminates the transfer */ - coh901318_pause(chan); - spin_lock_irqsave(&cohc->lock, flags); - - /* Clear any pending BE or TC interrupt */ - if (cohc->id < 32) { - writel(1 << cohc->id, virtbase + COH901318_BE_INT_CLEAR1); - writel(1 << cohc->id, virtbase + COH901318_TC_INT_CLEAR1); - } else { - writel(1 << (cohc->id - 32), virtbase + - COH901318_BE_INT_CLEAR2); - writel(1 << (cohc->id - 32), virtbase + - COH901318_TC_INT_CLEAR2); - } - - enable_powersave(cohc); - - while ((cohd = coh901318_first_active_get(cohc))) { - /* release the lli allocation*/ - coh901318_lli_free(&cohc->base->pool, &cohd->lli); - - /* return desc to free-list */ - coh901318_desc_remove(cohd); - coh901318_desc_free(cohc, cohd); - } - - while ((cohd = coh901318_first_queued(cohc))) { - /* release the lli allocation*/ - coh901318_lli_free(&cohc->base->pool, &cohd->lli); - - /* return desc to free-list */ - coh901318_desc_remove(cohd); - coh901318_desc_free(cohc, cohd); - } - - - cohc->nbr_active_done = 0; - cohc->busy = 0; - - spin_unlock_irqrestore(&cohc->lock, flags); - - return 0; -} - -static int coh901318_alloc_chan_resources(struct dma_chan *chan) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - unsigned long flags; - - dev_vdbg(COHC_2_DEV(cohc), "[%s] DMA channel %d\n", - __func__, cohc->id); - - if (chan->client_count > 1) - return -EBUSY; - - spin_lock_irqsave(&cohc->lock, flags); - - coh901318_config(cohc, NULL); - - cohc->allocated = 1; - dma_cookie_init(chan); - - spin_unlock_irqrestore(&cohc->lock, flags); - - return 1; -} - -static void -coh901318_free_chan_resources(struct dma_chan *chan) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - int channel = cohc->id; - unsigned long flags; - - spin_lock_irqsave(&cohc->lock, flags); - - /* Disable HW */ - writel(0x00000000U, cohc->base->virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING*channel); - writel(0x00000000U, cohc->base->virtbase + COH901318_CX_CTRL + - COH901318_CX_CTRL_SPACING*channel); - - cohc->allocated = 0; - - spin_unlock_irqrestore(&cohc->lock, flags); - - coh901318_terminate_all(chan); -} - - -static dma_cookie_t -coh901318_tx_submit(struct dma_async_tx_descriptor *tx) -{ - struct coh901318_desc *cohd = container_of(tx, struct coh901318_desc, - desc); - struct coh901318_chan *cohc = to_coh901318_chan(tx->chan); - unsigned long flags; - dma_cookie_t cookie; - - spin_lock_irqsave(&cohc->lock, flags); - cookie = dma_cookie_assign(tx); - - coh901318_desc_queue(cohc, cohd); - - spin_unlock_irqrestore(&cohc->lock, flags); - - return cookie; -} - -static struct dma_async_tx_descriptor * -coh901318_prep_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, - size_t size, unsigned long flags) -{ - struct coh901318_lli *lli; - struct coh901318_desc *cohd; - unsigned long flg; - struct coh901318_chan *cohc = to_coh901318_chan(chan); - int lli_len; - u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last; - int ret; - - spin_lock_irqsave(&cohc->lock, flg); - - dev_vdbg(COHC_2_DEV(cohc), - "[%s] channel %d src %pad dest %pad size %zu\n", - __func__, cohc->id, &src, &dest, size); - - if (flags & DMA_PREP_INTERRUPT) - /* Trigger interrupt after last lli */ - ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE; - - lli_len = size >> MAX_DMA_PACKET_SIZE_SHIFT; - if ((lli_len << MAX_DMA_PACKET_SIZE_SHIFT) < size) - lli_len++; - - lli = coh901318_lli_alloc(&cohc->base->pool, lli_len); - - if (lli == NULL) - goto err; - - ret = coh901318_lli_fill_memcpy( - &cohc->base->pool, lli, src, size, dest, - cohc_chan_param(cohc)->ctrl_lli_chained, - ctrl_last); - if (ret) - goto err; - - COH_DBG(coh901318_list_print(cohc, lli)); - - /* Pick a descriptor to handle this transfer */ - cohd = coh901318_desc_get(cohc); - cohd->lli = lli; - cohd->flags = flags; - cohd->desc.tx_submit = coh901318_tx_submit; - - spin_unlock_irqrestore(&cohc->lock, flg); - - return &cohd->desc; - err: - spin_unlock_irqrestore(&cohc->lock, flg); - return NULL; -} - -static struct dma_async_tx_descriptor * -coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, - unsigned int sg_len, enum dma_transfer_direction direction, - unsigned long flags, void *context) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - struct coh901318_lli *lli; - struct coh901318_desc *cohd; - const struct coh901318_params *params; - struct scatterlist *sg; - int len = 0; - int size; - int i; - u32 ctrl_chained = cohc_chan_param(cohc)->ctrl_lli_chained; - u32 ctrl = cohc_chan_param(cohc)->ctrl_lli; - u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last; - u32 config; - unsigned long flg; - int ret; - - if (!sgl) - goto out; - if (sg_dma_len(sgl) == 0) - goto out; - - spin_lock_irqsave(&cohc->lock, flg); - - dev_vdbg(COHC_2_DEV(cohc), "[%s] sg_len %d dir %d\n", - __func__, sg_len, direction); - - if (flags & DMA_PREP_INTERRUPT) - /* Trigger interrupt after last lli */ - ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE; - - params = cohc_chan_param(cohc); - config = params->config; - /* - * Add runtime-specific control on top, make - * sure the bits you set per peripheral channel are - * cleared in the default config from the platform. - */ - ctrl_chained |= cohc->ctrl; - ctrl_last |= cohc->ctrl; - ctrl |= cohc->ctrl; - - if (direction == DMA_MEM_TO_DEV) { - u32 tx_flags = COH901318_CX_CTRL_PRDD_SOURCE | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE; - - config |= COH901318_CX_CFG_RM_MEMORY_TO_PRIMARY; - ctrl_chained |= tx_flags; - ctrl_last |= tx_flags; - ctrl |= tx_flags; - } else if (direction == DMA_DEV_TO_MEM) { - u32 rx_flags = COH901318_CX_CTRL_PRDD_DEST | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE; - - config |= COH901318_CX_CFG_RM_PRIMARY_TO_MEMORY; - ctrl_chained |= rx_flags; - ctrl_last |= rx_flags; - ctrl |= rx_flags; - } else - goto err_direction; - - /* The dma only supports transmitting packages up to - * MAX_DMA_PACKET_SIZE. Calculate to total number of - * dma elemts required to send the entire sg list - */ - for_each_sg(sgl, sg, sg_len, i) { - unsigned int factor; - size = sg_dma_len(sg); - - if (size <= MAX_DMA_PACKET_SIZE) { - len++; - continue; - } - - factor = size >> MAX_DMA_PACKET_SIZE_SHIFT; - if ((factor << MAX_DMA_PACKET_SIZE_SHIFT) < size) - factor++; - - len += factor; - } - - pr_debug("Allocate %d lli:s for this transfer\n", len); - lli = coh901318_lli_alloc(&cohc->base->pool, len); - - if (lli == NULL) - goto err_dma_alloc; - - coh901318_dma_set_runtimeconfig(chan, &cohc->config, direction); - - /* initiate allocated lli list */ - ret = coh901318_lli_fill_sg(&cohc->base->pool, lli, sgl, sg_len, - cohc->addr, - ctrl_chained, - ctrl, - ctrl_last, - direction, COH901318_CX_CTRL_TC_IRQ_ENABLE); - if (ret) - goto err_lli_fill; - - - COH_DBG(coh901318_list_print(cohc, lli)); - - /* Pick a descriptor to handle this transfer */ - cohd = coh901318_desc_get(cohc); - cohd->head_config = config; - /* - * Set the default head ctrl for the channel to the one from the - * lli, things may have changed due to odd buffer alignment - * etc. - */ - cohd->head_ctrl = lli->control; - cohd->dir = direction; - cohd->flags = flags; - cohd->desc.tx_submit = coh901318_tx_submit; - cohd->lli = lli; - - spin_unlock_irqrestore(&cohc->lock, flg); - - return &cohd->desc; - err_lli_fill: - err_dma_alloc: - err_direction: - spin_unlock_irqrestore(&cohc->lock, flg); - out: - return NULL; -} - -static enum dma_status -coh901318_tx_status(struct dma_chan *chan, dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - enum dma_status ret; - - ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_COMPLETE || !txstate) - return ret; - - dma_set_residue(txstate, coh901318_get_bytes_left(chan)); - - if (ret == DMA_IN_PROGRESS && cohc->stopped) - ret = DMA_PAUSED; - - return ret; -} - -static void -coh901318_issue_pending(struct dma_chan *chan) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&cohc->lock, flags); - - /* - * Busy means that pending jobs are already being processed, - * and then there is no point in starting the queue: the - * terminal count interrupt on the channel will take the next - * job on the queue and execute it anyway. - */ - if (!cohc->busy) - coh901318_queue_start(cohc); - - spin_unlock_irqrestore(&cohc->lock, flags); -} - -/* - * Here we wrap in the runtime dma control interface - */ -struct burst_table { - int burst_8bit; - int burst_16bit; - int burst_32bit; - u32 reg; -}; - -static const struct burst_table burst_sizes[] = { - { - .burst_8bit = 64, - .burst_16bit = 32, - .burst_32bit = 16, - .reg = COH901318_CX_CTRL_BURST_COUNT_64_BYTES, - }, - { - .burst_8bit = 48, - .burst_16bit = 24, - .burst_32bit = 12, - .reg = COH901318_CX_CTRL_BURST_COUNT_48_BYTES, - }, - { - .burst_8bit = 32, - .burst_16bit = 16, - .burst_32bit = 8, - .reg = COH901318_CX_CTRL_BURST_COUNT_32_BYTES, - }, - { - .burst_8bit = 16, - .burst_16bit = 8, - .burst_32bit = 4, - .reg = COH901318_CX_CTRL_BURST_COUNT_16_BYTES, - }, - { - .burst_8bit = 8, - .burst_16bit = 4, - .burst_32bit = 2, - .reg = COH901318_CX_CTRL_BURST_COUNT_8_BYTES, - }, - { - .burst_8bit = 4, - .burst_16bit = 2, - .burst_32bit = 1, - .reg = COH901318_CX_CTRL_BURST_COUNT_4_BYTES, - }, - { - .burst_8bit = 2, - .burst_16bit = 1, - .burst_32bit = 0, - .reg = COH901318_CX_CTRL_BURST_COUNT_2_BYTES, - }, - { - .burst_8bit = 1, - .burst_16bit = 0, - .burst_32bit = 0, - .reg = COH901318_CX_CTRL_BURST_COUNT_1_BYTE, - }, -}; - -static int coh901318_dma_set_runtimeconfig(struct dma_chan *chan, - struct dma_slave_config *config, - enum dma_transfer_direction direction) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - dma_addr_t addr; - enum dma_slave_buswidth addr_width; - u32 maxburst; - u32 ctrl = 0; - int i = 0; - - /* We only support mem to per or per to mem transfers */ - if (direction == DMA_DEV_TO_MEM) { - addr = config->src_addr; - addr_width = config->src_addr_width; - maxburst = config->src_maxburst; - } else if (direction == DMA_MEM_TO_DEV) { - addr = config->dst_addr; - addr_width = config->dst_addr_width; - maxburst = config->dst_maxburst; - } else { - dev_err(COHC_2_DEV(cohc), "illegal channel mode\n"); - return -EINVAL; - } - - dev_dbg(COHC_2_DEV(cohc), "configure channel for %d byte transfers\n", - addr_width); - switch (addr_width) { - case DMA_SLAVE_BUSWIDTH_1_BYTE: - ctrl |= - COH901318_CX_CTRL_SRC_BUS_SIZE_8_BITS | - COH901318_CX_CTRL_DST_BUS_SIZE_8_BITS; - - while (i < ARRAY_SIZE(burst_sizes)) { - if (burst_sizes[i].burst_8bit <= maxburst) - break; - i++; - } - - break; - case DMA_SLAVE_BUSWIDTH_2_BYTES: - ctrl |= - COH901318_CX_CTRL_SRC_BUS_SIZE_16_BITS | - COH901318_CX_CTRL_DST_BUS_SIZE_16_BITS; - - while (i < ARRAY_SIZE(burst_sizes)) { - if (burst_sizes[i].burst_16bit <= maxburst) - break; - i++; - } - - break; - case DMA_SLAVE_BUSWIDTH_4_BYTES: - /* Direction doesn't matter here, it's 32/32 bits */ - ctrl |= - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS; - - while (i < ARRAY_SIZE(burst_sizes)) { - if (burst_sizes[i].burst_32bit <= maxburst) - break; - i++; - } - - break; - default: - dev_err(COHC_2_DEV(cohc), - "bad runtimeconfig: alien address width\n"); - return -EINVAL; - } - - ctrl |= burst_sizes[i].reg; - dev_dbg(COHC_2_DEV(cohc), - "selected burst size %d bytes for address width %d bytes, maxburst %d\n", - burst_sizes[i].burst_8bit, addr_width, maxburst); - - cohc->addr = addr; - cohc->ctrl = ctrl; - - return 0; -} - -static int coh901318_dma_slave_config(struct dma_chan *chan, - struct dma_slave_config *config) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - - memcpy(&cohc->config, config, sizeof(*config)); - - return 0; -} - -static void coh901318_base_init(struct dma_device *dma, const int *pick_chans, - struct coh901318_base *base) -{ - int chans_i; - int i = 0; - struct coh901318_chan *cohc; - - INIT_LIST_HEAD(&dma->channels); - - for (chans_i = 0; pick_chans[chans_i] != -1; chans_i += 2) { - for (i = pick_chans[chans_i]; i <= pick_chans[chans_i+1]; i++) { - cohc = &base->chans[i]; - - cohc->base = base; - cohc->chan.device = dma; - cohc->id = i; - - /* TODO: do we really need this lock if only one - * client is connected to each channel? - */ - - spin_lock_init(&cohc->lock); - - cohc->nbr_active_done = 0; - cohc->busy = 0; - INIT_LIST_HEAD(&cohc->free); - INIT_LIST_HEAD(&cohc->active); - INIT_LIST_HEAD(&cohc->queue); - - tasklet_setup(&cohc->tasklet, dma_tasklet); - - list_add_tail(&cohc->chan.device_node, - &dma->channels); - } - } -} - -static int __init coh901318_probe(struct platform_device *pdev) -{ - int err = 0; - struct coh901318_base *base; - int irq; - struct resource *io; - - io = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!io) - return -ENODEV; - - /* Map DMA controller registers to virtual memory */ - if (devm_request_mem_region(&pdev->dev, - io->start, - resource_size(io), - pdev->dev.driver->name) == NULL) - return -ENOMEM; - - base = devm_kzalloc(&pdev->dev, - ALIGN(sizeof(struct coh901318_base), 4) + - U300_DMA_CHANNELS * - sizeof(struct coh901318_chan), - GFP_KERNEL); - if (!base) - return -ENOMEM; - - base->chans = ((void *)base) + ALIGN(sizeof(struct coh901318_base), 4); - - base->virtbase = devm_ioremap(&pdev->dev, io->start, resource_size(io)); - if (!base->virtbase) - return -ENOMEM; - - base->dev = &pdev->dev; - spin_lock_init(&base->pm.lock); - base->pm.started_channels = 0; - - COH901318_DEBUGFS_ASSIGN(debugfs_dma_base, base); - - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return irq; - - err = devm_request_irq(&pdev->dev, irq, dma_irq_handler, 0, - "coh901318", base); - if (err) - return err; - - base->irq = irq; - - err = coh901318_pool_create(&base->pool, &pdev->dev, - sizeof(struct coh901318_lli), - 32); - if (err) - return err; - - /* init channels for device transfers */ - coh901318_base_init(&base->dma_slave, dma_slave_channels, - base); - - dma_cap_zero(base->dma_slave.cap_mask); - dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask); - - base->dma_slave.device_alloc_chan_resources = coh901318_alloc_chan_resources; - base->dma_slave.device_free_chan_resources = coh901318_free_chan_resources; - base->dma_slave.device_prep_slave_sg = coh901318_prep_slave_sg; - base->dma_slave.device_tx_status = coh901318_tx_status; - base->dma_slave.device_issue_pending = coh901318_issue_pending; - base->dma_slave.device_config = coh901318_dma_slave_config; - base->dma_slave.device_pause = coh901318_pause; - base->dma_slave.device_resume = coh901318_resume; - base->dma_slave.device_terminate_all = coh901318_terminate_all; - base->dma_slave.dev = &pdev->dev; - - err = dma_async_device_register(&base->dma_slave); - - if (err) - goto err_register_slave; - - /* init channels for memcpy */ - coh901318_base_init(&base->dma_memcpy, dma_memcpy_channels, - base); - - dma_cap_zero(base->dma_memcpy.cap_mask); - dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask); - - base->dma_memcpy.device_alloc_chan_resources = coh901318_alloc_chan_resources; - base->dma_memcpy.device_free_chan_resources = coh901318_free_chan_resources; - base->dma_memcpy.device_prep_dma_memcpy = coh901318_prep_memcpy; - base->dma_memcpy.device_tx_status = coh901318_tx_status; - base->dma_memcpy.device_issue_pending = coh901318_issue_pending; - base->dma_memcpy.device_config = coh901318_dma_slave_config; - base->dma_memcpy.device_pause = coh901318_pause; - base->dma_memcpy.device_resume = coh901318_resume; - base->dma_memcpy.device_terminate_all = coh901318_terminate_all; - base->dma_memcpy.dev = &pdev->dev; - /* - * This controller can only access address at even 32bit boundaries, - * i.e. 2^2 - */ - base->dma_memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES; - err = dma_async_device_register(&base->dma_memcpy); - - if (err) - goto err_register_memcpy; - - err = of_dma_controller_register(pdev->dev.of_node, coh901318_xlate, - base); - if (err) - goto err_register_of_dma; - - platform_set_drvdata(pdev, base); - dev_info(&pdev->dev, "Initialized COH901318 DMA on virtual base 0x%p\n", - base->virtbase); - - return err; - - err_register_of_dma: - dma_async_device_unregister(&base->dma_memcpy); - err_register_memcpy: - dma_async_device_unregister(&base->dma_slave); - err_register_slave: - coh901318_pool_destroy(&base->pool); - return err; -} -static void coh901318_base_remove(struct coh901318_base *base, const int *pick_chans) -{ - int chans_i; - int i = 0; - struct coh901318_chan *cohc; - - for (chans_i = 0; pick_chans[chans_i] != -1; chans_i += 2) { - for (i = pick_chans[chans_i]; i <= pick_chans[chans_i+1]; i++) { - cohc = &base->chans[i]; - - tasklet_kill(&cohc->tasklet); - } - } - -} - -static int coh901318_remove(struct platform_device *pdev) -{ - struct coh901318_base *base = platform_get_drvdata(pdev); - - devm_free_irq(&pdev->dev, base->irq, base); - - coh901318_base_remove(base, dma_slave_channels); - coh901318_base_remove(base, dma_memcpy_channels); - - of_dma_controller_free(pdev->dev.of_node); - dma_async_device_unregister(&base->dma_memcpy); - dma_async_device_unregister(&base->dma_slave); - coh901318_pool_destroy(&base->pool); - return 0; -} - -static const struct of_device_id coh901318_dt_match[] = { - { .compatible = "stericsson,coh901318" }, - {}, -}; - -static struct platform_driver coh901318_driver = { - .remove = coh901318_remove, - .driver = { - .name = "coh901318", - .of_match_table = coh901318_dt_match, - }, -}; - -static int __init coh901318_init(void) -{ - return platform_driver_probe(&coh901318_driver, coh901318_probe); -} -subsys_initcall(coh901318_init); - -static void __exit coh901318_exit(void) -{ - platform_driver_unregister(&coh901318_driver); -} -module_exit(coh901318_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Per Friden"); diff --git a/drivers/dma/coh901318.h b/drivers/dma/coh901318.h deleted file mode 100644 index bbf533600558..000000000000 --- a/drivers/dma/coh901318.h +++ /dev/null @@ -1,141 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2007-2013 ST-Ericsson - * DMA driver for COH 901 318 - * Author: Per Friden <per.friden@stericsson.com> - */ - -#ifndef COH901318_H -#define COH901318_H - -#define MAX_DMA_PACKET_SIZE_SHIFT 11 -#define MAX_DMA_PACKET_SIZE (1 << MAX_DMA_PACKET_SIZE_SHIFT) - -struct device; - -struct coh901318_pool { - spinlock_t lock; - struct dma_pool *dmapool; - struct device *dev; - -#ifdef CONFIG_DEBUG_FS - int debugfs_pool_counter; -#endif -}; - -/** - * struct coh901318_lli - linked list item for DMAC - * @control: control settings for DMAC - * @src_addr: transfer source address - * @dst_addr: transfer destination address - * @link_addr: physical address to next lli - * @virt_link_addr: virtual address of next lli (only used by pool_free) - * @phy_this: physical address of current lli (only used by pool_free) - */ -struct coh901318_lli { - u32 control; - dma_addr_t src_addr; - dma_addr_t dst_addr; - dma_addr_t link_addr; - - void *virt_link_addr; - dma_addr_t phy_this; -}; - -/** - * coh901318_pool_create() - Creates an dma pool for lli:s - * @pool: pool handle - * @dev: dma device - * @lli_nbr: number of lli:s in the pool - * @algin: address alignemtn of lli:s - * returns 0 on success otherwise none zero - */ -int coh901318_pool_create(struct coh901318_pool *pool, - struct device *dev, - size_t lli_nbr, size_t align); - -/** - * coh901318_pool_destroy() - Destroys the dma pool - * @pool: pool handle - * returns 0 on success otherwise none zero - */ -int coh901318_pool_destroy(struct coh901318_pool *pool); - -/** - * coh901318_lli_alloc() - Allocates a linked list - * - * @pool: pool handle - * @len: length to list - * return: none NULL if success otherwise NULL - */ -struct coh901318_lli * -coh901318_lli_alloc(struct coh901318_pool *pool, - unsigned int len); - -/** - * coh901318_lli_free() - Returns the linked list items to the pool - * @pool: pool handle - * @lli: reference to lli pointer to be freed - */ -void coh901318_lli_free(struct coh901318_pool *pool, - struct coh901318_lli **lli); - -/** - * coh901318_lli_fill_memcpy() - Prepares the lli:s for dma memcpy - * @pool: pool handle - * @lli: allocated lli - * @src: src address - * @size: transfer size - * @dst: destination address - * @ctrl_chained: ctrl for chained lli - * @ctrl_last: ctrl for the last lli - * returns number of CPU interrupts for the lli, negative on error. - */ -int -coh901318_lli_fill_memcpy(struct coh901318_pool *pool, - struct coh901318_lli *lli, - dma_addr_t src, unsigned int size, - dma_addr_t dst, u32 ctrl_chained, u32 ctrl_last); - -/** - * coh901318_lli_fill_single() - Prepares the lli:s for dma single transfer - * @pool: pool handle - * @lli: allocated lli - * @buf: transfer buffer - * @size: transfer size - * @dev_addr: address of periphal - * @ctrl_chained: ctrl for chained lli - * @ctrl_last: ctrl for the last lli - * @dir: direction of transfer (to or from device) - * returns number of CPU interrupts for the lli, negative on error. - */ -int -coh901318_lli_fill_single(struct coh901318_pool *pool, - struct coh901318_lli *lli, - dma_addr_t buf, unsigned int size, - dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl_last, - enum dma_transfer_direction dir); - -/** - * coh901318_lli_fill_single() - Prepares the lli:s for dma scatter list transfer - * @pool: pool handle - * @lli: allocated lli - * @sg: scatter gather list - * @nents: number of entries in sg - * @dev_addr: address of periphal - * @ctrl_chained: ctrl for chained lli - * @ctrl: ctrl of middle lli - * @ctrl_last: ctrl for the last lli - * @dir: direction of transfer (to or from device) - * @ctrl_irq_mask: ctrl mask for CPU interrupt - * returns number of CPU interrupts for the lli, negative on error. - */ -int -coh901318_lli_fill_sg(struct coh901318_pool *pool, - struct coh901318_lli *lli, - struct scatterlist *sg, unsigned int nents, - dma_addr_t dev_addr, u32 ctrl_chained, - u32 ctrl, u32 ctrl_last, - enum dma_transfer_direction dir, u32 ctrl_irq_mask); - -#endif /* COH901318_H */ diff --git a/drivers/dma/coh901318_lli.c b/drivers/dma/coh901318_lli.c deleted file mode 100644 index 6b6c2fd0865a..000000000000 --- a/drivers/dma/coh901318_lli.c +++ /dev/null @@ -1,313 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * driver/dma/coh901318_lli.c - * - * Copyright (C) 2007-2009 ST-Ericsson - * Support functions for handling lli for dma - * Author: Per Friden <per.friden@stericsson.com> - */ - -#include <linux/spinlock.h> -#include <linux/memory.h> -#include <linux/gfp.h> -#include <linux/dmapool.h> -#include <linux/dmaengine.h> - -#include "coh901318.h" - -#if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_U300_DEBUG)) -#define DEBUGFS_POOL_COUNTER_RESET(pool) (pool->debugfs_pool_counter = 0) -#define DEBUGFS_POOL_COUNTER_ADD(pool, add) (pool->debugfs_pool_counter += add) -#else -#define DEBUGFS_POOL_COUNTER_RESET(pool) -#define DEBUGFS_POOL_COUNTER_ADD(pool, add) -#endif - -static struct coh901318_lli * -coh901318_lli_next(struct coh901318_lli *data) -{ - if (data == NULL || data->link_addr == 0) - return NULL; - - return (struct coh901318_lli *) data->virt_link_addr; -} - -int coh901318_pool_create(struct coh901318_pool *pool, - struct device *dev, - size_t size, size_t align) -{ - spin_lock_init(&pool->lock); - pool->dev = dev; - pool->dmapool = dma_pool_create("lli_pool", dev, size, align, 0); - - DEBUGFS_POOL_COUNTER_RESET(pool); - return 0; -} - -int coh901318_pool_destroy(struct coh901318_pool *pool) -{ - - dma_pool_destroy(pool->dmapool); - return 0; -} - -struct coh901318_lli * -coh901318_lli_alloc(struct coh901318_pool *pool, unsigned int len) -{ - int i; - struct coh901318_lli *head; - struct coh901318_lli *lli; - struct coh901318_lli *lli_prev; - dma_addr_t phy; - - if (len == 0) - return NULL; - - spin_lock(&pool->lock); - - head = dma_pool_alloc(pool->dmapool, GFP_NOWAIT, &phy); - - if (head == NULL) - goto err; - - DEBUGFS_POOL_COUNTER_ADD(pool, 1); - - lli = head; - lli->phy_this = phy; - lli->link_addr = 0x00000000; - lli->virt_link_addr = NULL; - - for (i = 1; i < len; i++) { - lli_prev = lli; - - lli = dma_pool_alloc(pool->dmapool, GFP_NOWAIT, &phy); - - if (lli == NULL) - goto err_clean_up; - - DEBUGFS_POOL_COUNTER_ADD(pool, 1); - lli->phy_this = phy; - lli->link_addr = 0x00000000; - lli->virt_link_addr = NULL; - - lli_prev->link_addr = phy; - lli_prev->virt_link_addr = lli; - } - - spin_unlock(&pool->lock); - - return head; - - err: - spin_unlock(&pool->lock); - return NULL; - - err_clean_up: - lli_prev->link_addr = 0x00000000U; - spin_unlock(&pool->lock); - coh901318_lli_free(pool, &head); - return NULL; -} - -void coh901318_lli_free(struct coh901318_pool *pool, - struct coh901318_lli **lli) -{ - struct coh901318_lli *l; - struct coh901318_lli *next; - - if (lli == NULL) - return; - - l = *lli; - - if (l == NULL) - return; - - spin_lock(&pool->lock); - - while (l->link_addr) { - next = l->virt_link_addr; - dma_pool_free(pool->dmapool, l, l->phy_this); - DEBUGFS_POOL_COUNTER_ADD(pool, -1); - l = next; - } - dma_pool_free(pool->dmapool, l, l->phy_this); - DEBUGFS_POOL_COUNTER_ADD(pool, -1); - - spin_unlock(&pool->lock); - *lli = NULL; -} - -int -coh901318_lli_fill_memcpy(struct coh901318_pool *pool, - struct coh901318_lli *lli, - dma_addr_t source, unsigned int size, - dma_addr_t destination, u32 ctrl_chained, - u32 ctrl_eom) -{ - int s = size; - dma_addr_t src = source; - dma_addr_t dst = destination; - - lli->src_addr = src; - lli->dst_addr = dst; - - while (lli->link_addr) { - lli->control = ctrl_chained | MAX_DMA_PACKET_SIZE; - lli->src_addr = src; - lli->dst_addr = dst; - - s -= MAX_DMA_PACKET_SIZE; - lli = coh901318_lli_next(lli); - - src += MAX_DMA_PACKET_SIZE; - dst += MAX_DMA_PACKET_SIZE; - } - - lli->control = ctrl_eom | s; - lli->src_addr = src; - lli->dst_addr = dst; - - return 0; -} - -int -coh901318_lli_fill_single(struct coh901318_pool *pool, - struct coh901318_lli *lli, - dma_addr_t buf, unsigned int size, - dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl_eom, - enum dma_transfer_direction dir) -{ - int s = size; - dma_addr_t src; - dma_addr_t dst; - - - if (dir == DMA_MEM_TO_DEV) { - src = buf; - dst = dev_addr; - - } else if (dir == DMA_DEV_TO_MEM) { - - src = dev_addr; - dst = buf; - } else { - return -EINVAL; - } - - while (lli->link_addr) { - size_t block_size = MAX_DMA_PACKET_SIZE; - lli->control = ctrl_chained | MAX_DMA_PACKET_SIZE; - - /* If we are on the next-to-final block and there will - * be less than half a DMA packet left for the last - * block, then we want to make this block a little - * smaller to balance the sizes. This is meant to - * avoid too small transfers if the buffer size is - * (MAX_DMA_PACKET_SIZE*N + 1) */ - if (s < (MAX_DMA_PACKET_SIZE + MAX_DMA_PACKET_SIZE/2)) - block_size = MAX_DMA_PACKET_SIZE/2; - - s -= block_size; - lli->src_addr = src; - lli->dst_addr = dst; - - lli = coh901318_lli_next(lli); - - if (dir == DMA_MEM_TO_DEV) - src += block_size; - else if (dir == DMA_DEV_TO_MEM) - dst += block_size; - } - - lli->control = ctrl_eom | s; - lli->src_addr = src; - lli->dst_addr = dst; - - return 0; -} - -int -coh901318_lli_fill_sg(struct coh901318_pool *pool, - struct coh901318_lli *lli, - struct scatterlist *sgl, unsigned int nents, - dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl, - u32 ctrl_last, - enum dma_transfer_direction dir, u32 ctrl_irq_mask) -{ - int i; - struct scatterlist *sg; - u32 ctrl_sg; - dma_addr_t src = 0; - dma_addr_t dst = 0; - u32 bytes_to_transfer; - u32 elem_size; - - if (lli == NULL) - goto err; - - spin_lock(&pool->lock); - - if (dir == DMA_MEM_TO_DEV) - dst = dev_addr; - else if (dir == DMA_DEV_TO_MEM) - src = dev_addr; - else - goto err; - - for_each_sg(sgl, sg, nents, i) { - if (sg_is_chain(sg)) { - /* sg continues to the next sg-element don't - * send ctrl_finish until the last - * sg-element in the chain - */ - ctrl_sg = ctrl_chained; - } else if (i == nents - 1) - ctrl_sg = ctrl_last; - else - ctrl_sg = ctrl ? ctrl : ctrl_last; - - - if (dir == DMA_MEM_TO_DEV) - /* increment source address */ - src = sg_dma_address(sg); - else - /* increment destination address */ - dst = sg_dma_address(sg); - - bytes_to_transfer = sg_dma_len(sg); - - while (bytes_to_transfer) { - u32 val; - - if (bytes_to_transfer > MAX_DMA_PACKET_SIZE) { - elem_size = MAX_DMA_PACKET_SIZE; - val = ctrl_chained; - } else { - elem_size = bytes_to_transfer; - val = ctrl_sg; - } - - lli->control = val | elem_size; - lli->src_addr = src; - lli->dst_addr = dst; - - if (dir == DMA_DEV_TO_MEM) - dst += elem_size; - else - src += elem_size; - - BUG_ON(lli->link_addr & 3); - - bytes_to_transfer -= elem_size; - lli = coh901318_lli_next(lli); - } - - } - spin_unlock(&pool->lock); - - return 0; - err: - spin_unlock(&pool->lock); - return -EINVAL; -} diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c index 612d353648cf..ebee94dbd630 100644 --- a/drivers/dma/dma-jz4780.c +++ b/drivers/dma/dma-jz4780.c @@ -1004,6 +1004,18 @@ static const struct jz4780_dma_soc_data jz4725b_dma_soc_data = { JZ_SOC_DATA_BREAK_LINKS, }; +static const struct jz4780_dma_soc_data jz4760_dma_soc_data = { + .nb_channels = 5, + .transfer_ord_max = 6, + .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, +}; + +static const struct jz4780_dma_soc_data jz4760b_dma_soc_data = { + .nb_channels = 5, + .transfer_ord_max = 6, + .flags = JZ_SOC_DATA_PER_CHAN_PM, +}; + static const struct jz4780_dma_soc_data jz4770_dma_soc_data = { .nb_channels = 6, .transfer_ord_max = 6, @@ -1031,6 +1043,8 @@ static const struct jz4780_dma_soc_data x1830_dma_soc_data = { static const struct of_device_id jz4780_dma_dt_match[] = { { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data }, { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data }, + { .compatible = "ingenic,jz4760-dma", .data = &jz4760_dma_soc_data }, + { .compatible = "ingenic,jz4760b-dma", .data = &jz4760b_dma_soc_data }, { .compatible = "ingenic,jz4770-dma", .data = &jz4770_dma_soc_data }, { .compatible = "ingenic,jz4780-dma", .data = &jz4780_dma_soc_data }, { .compatible = "ingenic,x1000-dma", .data = &x1000_dma_soc_data }, diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index e164f3295f5d..d9e4ac3edb4e 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -12,15 +12,20 @@ #include <linux/device.h> #include <linux/dmaengine.h> #include <linux/dmapool.h> +#include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/io-64-nonatomic-lo-hi.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> +#include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/property.h> +#include <linux/slab.h> #include <linux/types.h> #include "dw-axi-dmac.h" @@ -195,43 +200,56 @@ static inline const char *axi_chan_name(struct axi_dma_chan *chan) return dma_chan_name(&chan->vc.chan); } -static struct axi_dma_desc *axi_desc_get(struct axi_dma_chan *chan) +static struct axi_dma_desc *axi_desc_alloc(u32 num) { - struct dw_axi_dma *dw = chan->chip->dw; struct axi_dma_desc *desc; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->hw_desc = kcalloc(num, sizeof(*desc->hw_desc), GFP_NOWAIT); + if (!desc->hw_desc) { + kfree(desc); + return NULL; + } + + return desc; +} + +static struct axi_dma_lli *axi_desc_get(struct axi_dma_chan *chan, + dma_addr_t *addr) +{ + struct axi_dma_lli *lli; dma_addr_t phys; - desc = dma_pool_zalloc(dw->desc_pool, GFP_NOWAIT, &phys); - if (unlikely(!desc)) { + lli = dma_pool_zalloc(chan->desc_pool, GFP_NOWAIT, &phys); + if (unlikely(!lli)) { dev_err(chan2dev(chan), "%s: not enough descriptors available\n", axi_chan_name(chan)); return NULL; } atomic_inc(&chan->descs_allocated); - INIT_LIST_HEAD(&desc->xfer_list); - desc->vd.tx.phys = phys; - desc->chan = chan; + *addr = phys; - return desc; + return lli; } static void axi_desc_put(struct axi_dma_desc *desc) { struct axi_dma_chan *chan = desc->chan; - struct dw_axi_dma *dw = chan->chip->dw; - struct axi_dma_desc *child, *_next; - unsigned int descs_put = 0; + int count = atomic_read(&chan->descs_allocated); + struct axi_dma_hw_desc *hw_desc; + int descs_put; - list_for_each_entry_safe(child, _next, &desc->xfer_list, xfer_list) { - list_del(&child->xfer_list); - dma_pool_free(dw->desc_pool, child, child->vd.tx.phys); - descs_put++; + for (descs_put = 0; descs_put < count; descs_put++) { + hw_desc = &desc->hw_desc[descs_put]; + dma_pool_free(chan->desc_pool, hw_desc->lli, hw_desc->llp); } - dma_pool_free(dw->desc_pool, desc, desc->vd.tx.phys); - descs_put++; - + kfree(desc->hw_desc); + kfree(desc); atomic_sub(descs_put, &chan->descs_allocated); dev_vdbg(chan2dev(chan), "%s: %d descs put, %d still allocated\n", axi_chan_name(chan), descs_put, @@ -248,19 +266,41 @@ dma_chan_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, struct dma_tx_state *txstate) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); - enum dma_status ret; + struct virt_dma_desc *vdesc; + enum dma_status status; + u32 completed_length; + unsigned long flags; + u32 completed_blocks; + size_t bytes = 0; + u32 length; + u32 len; - ret = dma_cookie_status(dchan, cookie, txstate); + status = dma_cookie_status(dchan, cookie, txstate); + if (status == DMA_COMPLETE || !txstate) + return status; - if (chan->is_paused && ret == DMA_IN_PROGRESS) - ret = DMA_PAUSED; + spin_lock_irqsave(&chan->vc.lock, flags); - return ret; + vdesc = vchan_find_desc(&chan->vc, cookie); + if (vdesc) { + length = vd_to_axi_desc(vdesc)->length; + completed_blocks = vd_to_axi_desc(vdesc)->completed_blocks; + len = vd_to_axi_desc(vdesc)->hw_desc[0].len; + completed_length = completed_blocks * len; + bytes = length - completed_length; + } else { + bytes = vd_to_axi_desc(vdesc)->length; + } + + spin_unlock_irqrestore(&chan->vc.lock, flags); + dma_set_residue(txstate, bytes); + + return status; } -static void write_desc_llp(struct axi_dma_desc *desc, dma_addr_t adr) +static void write_desc_llp(struct axi_dma_hw_desc *desc, dma_addr_t adr) { - desc->lli.llp = cpu_to_le64(adr); + desc->lli->llp = cpu_to_le64(adr); } static void write_chan_llp(struct axi_dma_chan *chan, dma_addr_t adr) @@ -268,6 +308,29 @@ static void write_chan_llp(struct axi_dma_chan *chan, dma_addr_t adr) axi_chan_iowrite64(chan, CH_LLP, adr); } +static void dw_axi_dma_set_byte_halfword(struct axi_dma_chan *chan, bool set) +{ + u32 offset = DMAC_APB_BYTE_WR_CH_EN; + u32 reg_width, val; + + if (!chan->chip->apb_regs) { + dev_dbg(chan->chip->dev, "apb_regs not initialized\n"); + return; + } + + reg_width = __ffs(chan->config.dst_addr_width); + if (reg_width == DWAXIDMAC_TRANS_WIDTH_16) + offset = DMAC_APB_HALFWORD_WR_CH_EN; + + val = ioread32(chan->chip->apb_regs + offset); + + if (set) + val |= BIT(chan->id); + else + val &= ~BIT(chan->id); + + iowrite32(val, chan->chip->apb_regs + offset); +} /* Called in chan locked context */ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, struct axi_dma_desc *first) @@ -293,9 +356,26 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, priority << CH_CFG_H_PRIORITY_POS | DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_DST_POS | DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_SRC_POS); + switch (chan->direction) { + case DMA_MEM_TO_DEV: + dw_axi_dma_set_byte_halfword(chan, true); + reg |= (chan->config.device_fc ? + DWAXIDMAC_TT_FC_MEM_TO_PER_DST : + DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC) + << CH_CFG_H_TT_FC_POS; + break; + case DMA_DEV_TO_MEM: + reg |= (chan->config.device_fc ? + DWAXIDMAC_TT_FC_PER_TO_MEM_SRC : + DWAXIDMAC_TT_FC_PER_TO_MEM_DMAC) + << CH_CFG_H_TT_FC_POS; + break; + default: + break; + } axi_chan_iowrite32(chan, CH_CFG_H, reg); - write_chan_llp(chan, first->vd.tx.phys | lms); + write_chan_llp(chan, first->hw_desc[0].llp | lms); irq_mask = DWAXIDMAC_IRQ_DMA_TRF | DWAXIDMAC_IRQ_ALL_ERR; axi_chan_irq_sig_set(chan, irq_mask); @@ -333,6 +413,13 @@ static void dma_chan_issue_pending(struct dma_chan *dchan) spin_unlock_irqrestore(&chan->vc.lock, flags); } +static void dw_axi_dma_synchronize(struct dma_chan *dchan) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + + vchan_synchronize(&chan->vc); +} + static int dma_chan_alloc_chan_resources(struct dma_chan *dchan) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); @@ -344,6 +431,15 @@ static int dma_chan_alloc_chan_resources(struct dma_chan *dchan) return -EBUSY; } + /* LLI address must be aligned to a 64-byte boundary */ + chan->desc_pool = dma_pool_create(dev_name(chan2dev(chan)), + chan->chip->dev, + sizeof(struct axi_dma_lli), + 64, 0); + if (!chan->desc_pool) { + dev_err(chan2dev(chan), "No memory for descriptors\n"); + return -ENOMEM; + } dev_vdbg(dchan2dev(dchan), "%s: allocating\n", axi_chan_name(chan)); pm_runtime_get(chan->chip->dev); @@ -365,6 +461,8 @@ static void dma_chan_free_chan_resources(struct dma_chan *dchan) vchan_free_chan_resources(&chan->vc); + dma_pool_destroy(chan->desc_pool); + chan->desc_pool = NULL; dev_vdbg(dchan2dev(dchan), "%s: free resources, descriptor still allocated: %u\n", axi_chan_name(chan), atomic_read(&chan->descs_allocated)); @@ -372,73 +470,398 @@ static void dma_chan_free_chan_resources(struct dma_chan *dchan) pm_runtime_put(chan->chip->dev); } +static void dw_axi_dma_set_hw_channel(struct axi_dma_chip *chip, + u32 handshake_num, bool set) +{ + unsigned long start = 0; + unsigned long reg_value; + unsigned long reg_mask; + unsigned long reg_set; + unsigned long mask; + unsigned long val; + + if (!chip->apb_regs) { + dev_dbg(chip->dev, "apb_regs not initialized\n"); + return; + } + + /* + * An unused DMA channel has a default value of 0x3F. + * Lock the DMA channel by assign a handshake number to the channel. + * Unlock the DMA channel by assign 0x3F to the channel. + */ + if (set) { + reg_set = UNUSED_CHANNEL; + val = handshake_num; + } else { + reg_set = handshake_num; + val = UNUSED_CHANNEL; + } + + reg_value = lo_hi_readq(chip->apb_regs + DMAC_APB_HW_HS_SEL_0); + + for_each_set_clump8(start, reg_mask, ®_value, 64) { + if (reg_mask == reg_set) { + mask = GENMASK_ULL(start + 7, start); + reg_value &= ~mask; + reg_value |= rol64(val, start); + lo_hi_writeq(reg_value, + chip->apb_regs + DMAC_APB_HW_HS_SEL_0); + break; + } + } +} + /* * If DW_axi_dmac sees CHx_CTL.ShadowReg_Or_LLI_Last bit of the fetched LLI * as 1, it understands that the current block is the final block in the * transfer and completes the DMA transfer operation at the end of current * block transfer. */ -static void set_desc_last(struct axi_dma_desc *desc) +static void set_desc_last(struct axi_dma_hw_desc *desc) { u32 val; - val = le32_to_cpu(desc->lli.ctl_hi); + val = le32_to_cpu(desc->lli->ctl_hi); val |= CH_CTL_H_LLI_LAST; - desc->lli.ctl_hi = cpu_to_le32(val); + desc->lli->ctl_hi = cpu_to_le32(val); } -static void write_desc_sar(struct axi_dma_desc *desc, dma_addr_t adr) +static void write_desc_sar(struct axi_dma_hw_desc *desc, dma_addr_t adr) { - desc->lli.sar = cpu_to_le64(adr); + desc->lli->sar = cpu_to_le64(adr); } -static void write_desc_dar(struct axi_dma_desc *desc, dma_addr_t adr) +static void write_desc_dar(struct axi_dma_hw_desc *desc, dma_addr_t adr) { - desc->lli.dar = cpu_to_le64(adr); + desc->lli->dar = cpu_to_le64(adr); } -static void set_desc_src_master(struct axi_dma_desc *desc) +static void set_desc_src_master(struct axi_dma_hw_desc *desc) { u32 val; /* Select AXI0 for source master */ - val = le32_to_cpu(desc->lli.ctl_lo); + val = le32_to_cpu(desc->lli->ctl_lo); val &= ~CH_CTL_L_SRC_MAST; - desc->lli.ctl_lo = cpu_to_le32(val); + desc->lli->ctl_lo = cpu_to_le32(val); } -static void set_desc_dest_master(struct axi_dma_desc *desc) +static void set_desc_dest_master(struct axi_dma_hw_desc *hw_desc, + struct axi_dma_desc *desc) { u32 val; /* Select AXI1 for source master if available */ - val = le32_to_cpu(desc->lli.ctl_lo); + val = le32_to_cpu(hw_desc->lli->ctl_lo); if (desc->chan->chip->dw->hdata->nr_masters > 1) val |= CH_CTL_L_DST_MAST; else val &= ~CH_CTL_L_DST_MAST; - desc->lli.ctl_lo = cpu_to_le32(val); + hw_desc->lli->ctl_lo = cpu_to_le32(val); +} + +static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan, + struct axi_dma_hw_desc *hw_desc, + dma_addr_t mem_addr, size_t len) +{ + unsigned int data_width = BIT(chan->chip->dw->hdata->m_data_width); + unsigned int reg_width; + unsigned int mem_width; + dma_addr_t device_addr; + size_t axi_block_ts; + size_t block_ts; + u32 ctllo, ctlhi; + u32 burst_len; + + axi_block_ts = chan->chip->dw->hdata->block_size[chan->id]; + + mem_width = __ffs(data_width | mem_addr | len); + if (mem_width > DWAXIDMAC_TRANS_WIDTH_32) + mem_width = DWAXIDMAC_TRANS_WIDTH_32; + + if (!IS_ALIGNED(mem_addr, 4)) { + dev_err(chan->chip->dev, "invalid buffer alignment\n"); + return -EINVAL; + } + + switch (chan->direction) { + case DMA_MEM_TO_DEV: + reg_width = __ffs(chan->config.dst_addr_width); + device_addr = chan->config.dst_addr; + ctllo = reg_width << CH_CTL_L_DST_WIDTH_POS | + mem_width << CH_CTL_L_SRC_WIDTH_POS | + DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_DST_INC_POS | + DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS; + block_ts = len >> mem_width; + break; + case DMA_DEV_TO_MEM: + reg_width = __ffs(chan->config.src_addr_width); + device_addr = chan->config.src_addr; + ctllo = reg_width << CH_CTL_L_SRC_WIDTH_POS | + mem_width << CH_CTL_L_DST_WIDTH_POS | + DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS | + DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_SRC_INC_POS; + block_ts = len >> reg_width; + break; + default: + return -EINVAL; + } + + if (block_ts > axi_block_ts) + return -EINVAL; + + hw_desc->lli = axi_desc_get(chan, &hw_desc->llp); + if (unlikely(!hw_desc->lli)) + return -ENOMEM; + + ctlhi = CH_CTL_H_LLI_VALID; + + if (chan->chip->dw->hdata->restrict_axi_burst_len) { + burst_len = chan->chip->dw->hdata->axi_rw_burst_len; + ctlhi |= CH_CTL_H_ARLEN_EN | CH_CTL_H_AWLEN_EN | + burst_len << CH_CTL_H_ARLEN_POS | + burst_len << CH_CTL_H_AWLEN_POS; + } + + hw_desc->lli->ctl_hi = cpu_to_le32(ctlhi); + + if (chan->direction == DMA_MEM_TO_DEV) { + write_desc_sar(hw_desc, mem_addr); + write_desc_dar(hw_desc, device_addr); + } else { + write_desc_sar(hw_desc, device_addr); + write_desc_dar(hw_desc, mem_addr); + } + + hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1); + + ctllo |= DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS | + DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS; + hw_desc->lli->ctl_lo = cpu_to_le32(ctllo); + + set_desc_src_master(hw_desc); + + hw_desc->len = len; + return 0; +} + +static size_t calculate_block_len(struct axi_dma_chan *chan, + dma_addr_t dma_addr, size_t buf_len, + enum dma_transfer_direction direction) +{ + u32 data_width, reg_width, mem_width; + size_t axi_block_ts, block_len; + + axi_block_ts = chan->chip->dw->hdata->block_size[chan->id]; + + switch (direction) { + case DMA_MEM_TO_DEV: + data_width = BIT(chan->chip->dw->hdata->m_data_width); + mem_width = __ffs(data_width | dma_addr | buf_len); + if (mem_width > DWAXIDMAC_TRANS_WIDTH_32) + mem_width = DWAXIDMAC_TRANS_WIDTH_32; + + block_len = axi_block_ts << mem_width; + break; + case DMA_DEV_TO_MEM: + reg_width = __ffs(chan->config.src_addr_width); + block_len = axi_block_ts << reg_width; + break; + default: + block_len = 0; + } + + return block_len; +} + +static struct dma_async_tx_descriptor * +dw_axi_dma_chan_prep_cyclic(struct dma_chan *dchan, dma_addr_t dma_addr, + size_t buf_len, size_t period_len, + enum dma_transfer_direction direction, + unsigned long flags) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + struct axi_dma_hw_desc *hw_desc = NULL; + struct axi_dma_desc *desc = NULL; + dma_addr_t src_addr = dma_addr; + u32 num_periods, num_segments; + size_t axi_block_len; + u32 total_segments; + u32 segment_len; + unsigned int i; + int status; + u64 llp = 0; + u8 lms = 0; /* Select AXI0 master for LLI fetching */ + + num_periods = buf_len / period_len; + + axi_block_len = calculate_block_len(chan, dma_addr, buf_len, direction); + if (axi_block_len == 0) + return NULL; + + num_segments = DIV_ROUND_UP(period_len, axi_block_len); + segment_len = DIV_ROUND_UP(period_len, num_segments); + + total_segments = num_periods * num_segments; + + desc = axi_desc_alloc(total_segments); + if (unlikely(!desc)) + goto err_desc_get; + + chan->direction = direction; + desc->chan = chan; + chan->cyclic = true; + desc->length = 0; + desc->period_len = period_len; + + for (i = 0; i < total_segments; i++) { + hw_desc = &desc->hw_desc[i]; + + status = dw_axi_dma_set_hw_desc(chan, hw_desc, src_addr, + segment_len); + if (status < 0) + goto err_desc_get; + + desc->length += hw_desc->len; + /* Set end-of-link to the linked descriptor, so that cyclic + * callback function can be triggered during interrupt. + */ + set_desc_last(hw_desc); + + src_addr += segment_len; + } + + llp = desc->hw_desc[0].llp; + + /* Managed transfer list */ + do { + hw_desc = &desc->hw_desc[--total_segments]; + write_desc_llp(hw_desc, llp | lms); + llp = hw_desc->llp; + } while (total_segments); + + dw_axi_dma_set_hw_channel(chan->chip, chan->hw_handshake_num, true); + + return vchan_tx_prep(&chan->vc, &desc->vd, flags); + +err_desc_get: + if (desc) + axi_desc_put(desc); + + return NULL; +} + +static struct dma_async_tx_descriptor * +dw_axi_dma_chan_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + struct axi_dma_hw_desc *hw_desc = NULL; + struct axi_dma_desc *desc = NULL; + u32 num_segments, segment_len; + unsigned int loop = 0; + struct scatterlist *sg; + size_t axi_block_len; + u32 len, num_sgs = 0; + unsigned int i; + dma_addr_t mem; + int status; + u64 llp = 0; + u8 lms = 0; /* Select AXI0 master for LLI fetching */ + + if (unlikely(!is_slave_direction(direction) || !sg_len)) + return NULL; + + mem = sg_dma_address(sgl); + len = sg_dma_len(sgl); + + axi_block_len = calculate_block_len(chan, mem, len, direction); + if (axi_block_len == 0) + return NULL; + + for_each_sg(sgl, sg, sg_len, i) + num_sgs += DIV_ROUND_UP(sg_dma_len(sg), axi_block_len); + + desc = axi_desc_alloc(num_sgs); + if (unlikely(!desc)) + goto err_desc_get; + + desc->chan = chan; + desc->length = 0; + chan->direction = direction; + + for_each_sg(sgl, sg, sg_len, i) { + mem = sg_dma_address(sg); + len = sg_dma_len(sg); + num_segments = DIV_ROUND_UP(sg_dma_len(sg), axi_block_len); + segment_len = DIV_ROUND_UP(sg_dma_len(sg), num_segments); + + do { + hw_desc = &desc->hw_desc[loop++]; + status = dw_axi_dma_set_hw_desc(chan, hw_desc, mem, segment_len); + if (status < 0) + goto err_desc_get; + + desc->length += hw_desc->len; + len -= segment_len; + mem += segment_len; + } while (len >= segment_len); + } + + /* Set end-of-link to the last link descriptor of list */ + set_desc_last(&desc->hw_desc[num_sgs - 1]); + + /* Managed transfer list */ + do { + hw_desc = &desc->hw_desc[--num_sgs]; + write_desc_llp(hw_desc, llp | lms); + llp = hw_desc->llp; + } while (num_sgs); + + dw_axi_dma_set_hw_channel(chan->chip, chan->hw_handshake_num, true); + + return vchan_tx_prep(&chan->vc, &desc->vd, flags); + +err_desc_get: + if (desc) + axi_desc_put(desc); + + return NULL; } static struct dma_async_tx_descriptor * dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, dma_addr_t src_adr, size_t len, unsigned long flags) { - struct axi_dma_desc *first = NULL, *desc = NULL, *prev = NULL; struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); size_t block_ts, max_block_ts, xfer_len; - u32 xfer_width, reg; + struct axi_dma_hw_desc *hw_desc = NULL; + struct axi_dma_desc *desc = NULL; + u32 xfer_width, reg, num; + u64 llp = 0; u8 lms = 0; /* Select AXI0 master for LLI fetching */ dev_dbg(chan2dev(chan), "%s: memcpy: src: %pad dst: %pad length: %zd flags: %#lx", axi_chan_name(chan), &src_adr, &dst_adr, len, flags); max_block_ts = chan->chip->dw->hdata->block_size[chan->id]; + xfer_width = axi_chan_get_xfer_width(chan, src_adr, dst_adr, len); + num = DIV_ROUND_UP(len, max_block_ts << xfer_width); + desc = axi_desc_alloc(num); + if (unlikely(!desc)) + goto err_desc_get; + desc->chan = chan; + num = 0; + desc->length = 0; while (len) { xfer_len = len; + hw_desc = &desc->hw_desc[num]; /* * Take care for the alignment. * Actually source and destination widths can be different, but @@ -457,13 +880,13 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, xfer_len = max_block_ts << xfer_width; } - desc = axi_desc_get(chan); - if (unlikely(!desc)) + hw_desc->lli = axi_desc_get(chan, &hw_desc->llp); + if (unlikely(!hw_desc->lli)) goto err_desc_get; - write_desc_sar(desc, src_adr); - write_desc_dar(desc, dst_adr); - desc->lli.block_ts_lo = cpu_to_le32(block_ts - 1); + write_desc_sar(hw_desc, src_adr); + write_desc_dar(hw_desc, dst_adr); + hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1); reg = CH_CTL_H_LLI_VALID; if (chan->chip->dw->hdata->restrict_axi_burst_len) { @@ -474,7 +897,7 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, CH_CTL_H_AWLEN_EN | burst_len << CH_CTL_H_AWLEN_POS); } - desc->lli.ctl_hi = cpu_to_le32(reg); + hw_desc->lli->ctl_hi = cpu_to_le32(reg); reg = (DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS | DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS | @@ -482,62 +905,68 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, xfer_width << CH_CTL_L_SRC_WIDTH_POS | DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS | DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS); - desc->lli.ctl_lo = cpu_to_le32(reg); + hw_desc->lli->ctl_lo = cpu_to_le32(reg); - set_desc_src_master(desc); - set_desc_dest_master(desc); - - /* Manage transfer list (xfer_list) */ - if (!first) { - first = desc; - } else { - list_add_tail(&desc->xfer_list, &first->xfer_list); - write_desc_llp(prev, desc->vd.tx.phys | lms); - } - prev = desc; + set_desc_src_master(hw_desc); + set_desc_dest_master(hw_desc, desc); + hw_desc->len = xfer_len; + desc->length += hw_desc->len; /* update the length and addresses for the next loop cycle */ len -= xfer_len; dst_adr += xfer_len; src_adr += xfer_len; + num++; } - /* Total len of src/dest sg == 0, so no descriptor were allocated */ - if (unlikely(!first)) - return NULL; - /* Set end-of-link to the last link descriptor of list */ - set_desc_last(desc); + set_desc_last(&desc->hw_desc[num - 1]); + /* Managed transfer list */ + do { + hw_desc = &desc->hw_desc[--num]; + write_desc_llp(hw_desc, llp | lms); + llp = hw_desc->llp; + } while (num); - return vchan_tx_prep(&chan->vc, &first->vd, flags); + return vchan_tx_prep(&chan->vc, &desc->vd, flags); err_desc_get: - if (first) - axi_desc_put(first); + if (desc) + axi_desc_put(desc); return NULL; } +static int dw_axi_dma_chan_slave_config(struct dma_chan *dchan, + struct dma_slave_config *config) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + + memcpy(&chan->config, config, sizeof(*config)); + + return 0; +} + static void axi_chan_dump_lli(struct axi_dma_chan *chan, - struct axi_dma_desc *desc) + struct axi_dma_hw_desc *desc) { dev_err(dchan2dev(&chan->vc.chan), "SAR: 0x%llx DAR: 0x%llx LLP: 0x%llx BTS 0x%x CTL: 0x%x:%08x", - le64_to_cpu(desc->lli.sar), - le64_to_cpu(desc->lli.dar), - le64_to_cpu(desc->lli.llp), - le32_to_cpu(desc->lli.block_ts_lo), - le32_to_cpu(desc->lli.ctl_hi), - le32_to_cpu(desc->lli.ctl_lo)); + le64_to_cpu(desc->lli->sar), + le64_to_cpu(desc->lli->dar), + le64_to_cpu(desc->lli->llp), + le32_to_cpu(desc->lli->block_ts_lo), + le32_to_cpu(desc->lli->ctl_hi), + le32_to_cpu(desc->lli->ctl_lo)); } static void axi_chan_list_dump_lli(struct axi_dma_chan *chan, struct axi_dma_desc *desc_head) { - struct axi_dma_desc *desc; + int count = atomic_read(&chan->descs_allocated); + int i; - axi_chan_dump_lli(chan, desc_head); - list_for_each_entry(desc, &desc_head->xfer_list, xfer_list) - axi_chan_dump_lli(chan, desc); + for (i = 0; i < count; i++) + axi_chan_dump_lli(chan, &desc_head->hw_desc[i]); } static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) @@ -570,8 +999,13 @@ static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) { + int count = atomic_read(&chan->descs_allocated); + struct axi_dma_hw_desc *hw_desc; + struct axi_dma_desc *desc; struct virt_dma_desc *vd; unsigned long flags; + u64 llp; + int i; spin_lock_irqsave(&chan->vc.lock, flags); if (unlikely(axi_chan_is_hw_enable(chan))) { @@ -582,12 +1016,34 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) /* The completed descriptor currently is in the head of vc list */ vd = vchan_next_desc(&chan->vc); - /* Remove the completed descriptor from issued list before completing */ - list_del(&vd->node); - vchan_cookie_complete(vd); - /* Submit queued descriptors after processing the completed ones */ - axi_chan_start_first_queued(chan); + if (chan->cyclic) { + desc = vd_to_axi_desc(vd); + if (desc) { + llp = lo_hi_readq(chan->chan_regs + CH_LLP); + for (i = 0; i < count; i++) { + hw_desc = &desc->hw_desc[i]; + if (hw_desc->llp == llp) { + axi_chan_irq_clear(chan, hw_desc->lli->status_lo); + hw_desc->lli->ctl_hi |= CH_CTL_H_LLI_VALID; + desc->completed_blocks = i; + + if (((hw_desc->len * (i + 1)) % desc->period_len) == 0) + vchan_cyclic_callback(vd); + break; + } + } + + axi_chan_enable(chan); + } + } else { + /* Remove the completed descriptor from issued list before completing */ + list_del(&vd->node); + vchan_cookie_complete(vd); + + /* Submit queued descriptors after processing the completed ones */ + axi_chan_start_first_queued(chan); + } spin_unlock_irqrestore(&chan->vc.lock, flags); } @@ -627,15 +1083,31 @@ static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id) static int dma_chan_terminate_all(struct dma_chan *dchan) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + u32 chan_active = BIT(chan->id) << DMAC_CHAN_EN_SHIFT; unsigned long flags; + u32 val; + int ret; LIST_HEAD(head); - spin_lock_irqsave(&chan->vc.lock, flags); - axi_chan_disable(chan); + ret = readl_poll_timeout_atomic(chan->chip->regs + DMAC_CHEN, val, + !(val & chan_active), 1000, 10000); + if (ret == -ETIMEDOUT) + dev_warn(dchan2dev(dchan), + "%s failed to stop\n", axi_chan_name(chan)); + + if (chan->direction != DMA_MEM_TO_MEM) + dw_axi_dma_set_hw_channel(chan->chip, + chan->hw_handshake_num, false); + if (chan->direction == DMA_MEM_TO_DEV) + dw_axi_dma_set_byte_halfword(chan, false); + + spin_lock_irqsave(&chan->vc.lock, flags); + vchan_get_all_descriptors(&chan->vc, &head); + chan->cyclic = false; spin_unlock_irqrestore(&chan->vc.lock, flags); vchan_dma_desc_free_list(&chan->vc, &head); @@ -746,6 +1218,22 @@ static int __maybe_unused axi_dma_runtime_resume(struct device *dev) return axi_dma_resume(chip); } +static struct dma_chan *dw_axi_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct dw_axi_dma *dw = ofdma->of_dma_data; + struct axi_dma_chan *chan; + struct dma_chan *dchan; + + dchan = dma_get_any_slave_channel(&dw->dma); + if (!dchan) + return NULL; + + chan = dchan_to_axi_dma_chan(dchan); + chan->hw_handshake_num = dma_spec->args[0]; + return dchan; +} + static int parse_device_properties(struct axi_dma_chip *chip) { struct device *dev = chip->dev; @@ -816,6 +1304,7 @@ static int parse_device_properties(struct axi_dma_chip *chip) static int dw_probe(struct platform_device *pdev) { + struct device_node *node = pdev->dev.of_node; struct axi_dma_chip *chip; struct resource *mem; struct dw_axi_dma *dw; @@ -848,6 +1337,12 @@ static int dw_probe(struct platform_device *pdev) if (IS_ERR(chip->regs)) return PTR_ERR(chip->regs); + if (of_device_is_compatible(node, "intel,kmb-axi-dma")) { + chip->apb_regs = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(chip->apb_regs)) + return PTR_ERR(chip->apb_regs); + } + chip->core_clk = devm_clk_get(chip->dev, "core-clk"); if (IS_ERR(chip->core_clk)) return PTR_ERR(chip->core_clk); @@ -870,13 +1365,6 @@ static int dw_probe(struct platform_device *pdev) if (ret) return ret; - /* Lli address must be aligned to a 64-byte boundary */ - dw->desc_pool = dmam_pool_create(KBUILD_MODNAME, chip->dev, - sizeof(struct axi_dma_desc), 64, 0); - if (!dw->desc_pool) { - dev_err(chip->dev, "No memory for descriptors dma pool\n"); - return -ENOMEM; - } INIT_LIST_HEAD(&dw->dma.channels); for (i = 0; i < hdata->nr_channels; i++) { @@ -893,13 +1381,16 @@ static int dw_probe(struct platform_device *pdev) /* Set capabilities */ dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask); + dma_cap_set(DMA_SLAVE, dw->dma.cap_mask); + dma_cap_set(DMA_CYCLIC, dw->dma.cap_mask); /* DMA capabilities */ dw->dma.chancnt = hdata->nr_channels; dw->dma.src_addr_widths = AXI_DMA_BUSWIDTHS; dw->dma.dst_addr_widths = AXI_DMA_BUSWIDTHS; dw->dma.directions = BIT(DMA_MEM_TO_MEM); - dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + dw->dma.directions |= BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); + dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; dw->dma.dev = chip->dev; dw->dma.device_tx_status = dma_chan_tx_status; @@ -912,7 +1403,18 @@ static int dw_probe(struct platform_device *pdev) dw->dma.device_free_chan_resources = dma_chan_free_chan_resources; dw->dma.device_prep_dma_memcpy = dma_chan_prep_dma_memcpy; + dw->dma.device_synchronize = dw_axi_dma_synchronize; + dw->dma.device_config = dw_axi_dma_chan_slave_config; + dw->dma.device_prep_slave_sg = dw_axi_dma_chan_prep_slave_sg; + dw->dma.device_prep_dma_cyclic = dw_axi_dma_chan_prep_cyclic; + /* + * Synopsis DesignWare AxiDMA datasheet mentioned Maximum + * supported blocks is 1024. Device register width is 4 bytes. + * Therefore, set constraint to 1024 * 4. + */ + dw->dma.dev->dma_parms = &dw->dma_parms; + dma_set_max_seg_size(&pdev->dev, MAX_BLOCK_SIZE); platform_set_drvdata(pdev, chip); pm_runtime_enable(chip->dev); @@ -935,6 +1437,13 @@ static int dw_probe(struct platform_device *pdev) if (ret) goto err_pm_disable; + /* Register with OF helpers for DMA lookups */ + ret = of_dma_controller_register(pdev->dev.of_node, + dw_axi_dma_of_xlate, dw); + if (ret < 0) + dev_warn(&pdev->dev, + "Failed to register OF DMA controller, fallback to MEM_TO_MEM mode\n"); + dev_info(chip->dev, "DesignWare AXI DMA Controller, %d channels\n", dw->hdata->nr_channels); @@ -968,6 +1477,8 @@ static int dw_remove(struct platform_device *pdev) devm_free_irq(chip->dev, chip->irq, chip); + of_dma_controller_free(chip->dev->of_node); + list_for_each_entry_safe(chan, _chan, &dw->dma.channels, vc.chan.device_node) { list_del(&chan->vc.chan.device_node); @@ -983,6 +1494,7 @@ static const struct dev_pm_ops dw_axi_dma_pm_ops = { static const struct of_device_id dw_dma_of_id_table[] = { { .compatible = "snps,axi-dma-1.01a" }, + { .compatible = "intel,kmb-axi-dma" }, {} }; MODULE_DEVICE_TABLE(of, dw_dma_of_id_table); diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index 18b6014cf9b4..b69897887c76 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -37,10 +37,16 @@ struct axi_dma_chan { struct axi_dma_chip *chip; void __iomem *chan_regs; u8 id; + u8 hw_handshake_num; atomic_t descs_allocated; + struct dma_pool *desc_pool; struct virt_dma_chan vc; + struct axi_dma_desc *desc; + struct dma_slave_config config; + enum dma_transfer_direction direction; + bool cyclic; /* these other elements are all protected by vc.lock */ bool is_paused; }; @@ -48,7 +54,7 @@ struct axi_dma_chan { struct dw_axi_dma { struct dma_device dma; struct dw_axi_dma_hcfg *hdata; - struct dma_pool *desc_pool; + struct device_dma_parameters dma_parms; /* channels */ struct axi_dma_chan *chan; @@ -58,6 +64,7 @@ struct axi_dma_chip { struct device *dev; int irq; void __iomem *regs; + void __iomem *apb_regs; struct clk *core_clk; struct clk *cfgr_clk; struct dw_axi_dma *dw; @@ -80,12 +87,20 @@ struct __packed axi_dma_lli { __le32 reserved_hi; }; +struct axi_dma_hw_desc { + struct axi_dma_lli *lli; + dma_addr_t llp; + u32 len; +}; + struct axi_dma_desc { - struct axi_dma_lli lli; + struct axi_dma_hw_desc *hw_desc; struct virt_dma_desc vd; struct axi_dma_chan *chan; - struct list_head xfer_list; + u32 completed_blocks; + u32 length; + u32 period_len; }; static inline struct device *dchan2dev(struct dma_chan *dchan) @@ -157,6 +172,19 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan) #define CH_INTSIGNAL_ENA 0x090 /* R/W Chan Interrupt Signal Enable */ #define CH_INTCLEAR 0x098 /* W Chan Interrupt Clear */ +/* These Apb registers are used by Intel KeemBay SoC */ +#define DMAC_APB_CFG 0x000 /* DMAC Apb Configuration Register */ +#define DMAC_APB_STAT 0x004 /* DMAC Apb Status Register */ +#define DMAC_APB_DEBUG_STAT_0 0x008 /* DMAC Apb Debug Status Register 0 */ +#define DMAC_APB_DEBUG_STAT_1 0x00C /* DMAC Apb Debug Status Register 1 */ +#define DMAC_APB_HW_HS_SEL_0 0x010 /* DMAC Apb HW HS register 0 */ +#define DMAC_APB_HW_HS_SEL_1 0x014 /* DMAC Apb HW HS register 1 */ +#define DMAC_APB_LPI 0x018 /* DMAC Apb Low Power Interface Reg */ +#define DMAC_APB_BYTE_WR_CH_EN 0x01C /* DMAC Apb Byte Write Enable */ +#define DMAC_APB_HALFWORD_WR_CH_EN 0x020 /* DMAC Halfword write enables */ + +#define UNUSED_CHANNEL 0x3F /* Set unused DMA channel to 0x3F */ +#define MAX_BLOCK_SIZE 0x1000 /* 1024 blocks * 4 bytes data width */ /* DMAC_CFG */ #define DMAC_EN_POS 0 diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index 0feb323bae1e..f8459cc5315d 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -1214,6 +1214,7 @@ static int fsldma_of_probe(struct platform_device *op) { struct fsldma_device *fdev; struct device_node *child; + unsigned int i; int err; fdev = kzalloc(sizeof(*fdev), GFP_KERNEL); @@ -1292,6 +1293,10 @@ static int fsldma_of_probe(struct platform_device *op) return 0; out_free_fdev: + for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { + if (fdev->chan[i]) + fsl_dma_chan_remove(fdev->chan[i]); + } irq_dispose_mapping(fdev->irq); iounmap(fdev->regs); out_free: @@ -1314,6 +1319,7 @@ static int fsldma_of_remove(struct platform_device *op) if (fdev->chan[i]) fsl_dma_chan_remove(fdev->chan[i]); } + irq_dispose_mapping(fdev->irq); iounmap(fdev->regs); kfree(fdev); diff --git a/drivers/dma/hsu/pci.c b/drivers/dma/hsu/pci.c index 07cc7320a614..9045a6f7f589 100644 --- a/drivers/dma/hsu/pci.c +++ b/drivers/dma/hsu/pci.c @@ -26,22 +26,12 @@ static irqreturn_t hsu_pci_irq(int irq, void *dev) { struct hsu_dma_chip *chip = dev; - struct pci_dev *pdev = to_pci_dev(chip->dev); u32 dmaisr; u32 status; unsigned short i; int ret = 0; int err; - /* - * On Intel Tangier B0 and Anniedale the interrupt line, disregarding - * to have different numbers, is shared between HSU DMA and UART IPs. - * Thus on such SoCs we are expecting that IRQ handler is called in - * UART driver only. - */ - if (pdev->device == PCI_DEVICE_ID_INTEL_MRFLD_HSU_DMA) - return IRQ_HANDLED; - dmaisr = readl(chip->regs + HSU_PCI_DMAISR); for (i = 0; i < chip->hsu->nr_channels; i++) { if (dmaisr & 0x1) { @@ -105,6 +95,17 @@ static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto err_register_irq; + /* + * On Intel Tangier B0 and Anniedale the interrupt line, disregarding + * to have different numbers, is shared between HSU DMA and UART IPs. + * Thus on such SoCs we are expecting that IRQ handler is called in + * UART driver only. Instead of handling the spurious interrupt + * from HSU DMA here and waste CPU time and delay HSU UART interrupt + * handling, disable the interrupt entirely. + */ + if (pdev->device == PCI_DEVICE_ID_INTEL_MRFLD_HSU_DMA) + disable_irq_nosync(chip->irq); + pci_set_drvdata(pdev, chip); return 0; diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 71fd6e4c42cd..a15e50126434 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -165,6 +165,7 @@ int idxd_register_dma_device(struct idxd_device *idxd) INIT_LIST_HEAD(&dma->channels); dma->dev = &idxd->pdev->dev; + dma_cap_set(DMA_PRIVATE, dma->cap_mask); dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask); dma->device_release = idxd_dma_release; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index fa04acd5582a..085a0c3b62c6 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -26,12 +26,16 @@ MODULE_VERSION(IDXD_DRIVER_VERSION); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Intel Corporation"); +static bool sva = true; +module_param(sva, bool, 0644); +MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); + #define DRV_NAME "idxd" bool support_enqcmd; static struct idr idxd_idrs[IDXD_TYPE_MAX]; -static struct mutex idxd_idr_lock; +static DEFINE_MUTEX(idxd_idr_lock); static struct pci_device_id idxd_pci_tbl[] = { /* DSA ver 1.0 platforms */ @@ -341,12 +345,14 @@ static int idxd_probe(struct idxd_device *idxd) dev_dbg(dev, "IDXD reset complete\n"); - if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM)) { + if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { rc = idxd_enable_system_pasid(idxd); if (rc < 0) dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc); else set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); + } else if (!sva) { + dev_warn(dev, "User forced SVA off via module param.\n"); } idxd_read_caps(idxd); @@ -547,7 +553,6 @@ static int __init idxd_init_module(void) else support_enqcmd = true; - mutex_init(&idxd_idr_lock); for (i = 0; i < IDXD_TYPE_MAX; i++) idr_init(&idxd_idrs[i]); diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 41ba21eea7c8..d5590c08db51 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1952,8 +1952,6 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, static int sdma_probe(struct platform_device *pdev) { - const struct of_device_id *of_id = - of_match_device(sdma_dt_ids, &pdev->dev); struct device_node *np = pdev->dev.of_node; struct device_node *spba_bus; const char *fw_name; @@ -1961,17 +1959,9 @@ static int sdma_probe(struct platform_device *pdev) int irq; struct resource *iores; struct resource spba_res; - struct sdma_platform_data *pdata = dev_get_platdata(&pdev->dev); int i; struct sdma_engine *sdma; s32 *saddr_arr; - const struct sdma_driver_data *drvdata = NULL; - - drvdata = of_id->data; - if (!drvdata) { - dev_err(&pdev->dev, "unable to find driver data\n"); - return -EINVAL; - } ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (ret) @@ -1984,7 +1974,7 @@ static int sdma_probe(struct platform_device *pdev) spin_lock_init(&sdma->channel_0_lock); sdma->dev = &pdev->dev; - sdma->drvdata = drvdata; + sdma->drvdata = of_device_get_match_data(sdma->dev); irq = platform_get_irq(pdev, 0); if (irq < 0) @@ -2063,8 +2053,6 @@ static int sdma_probe(struct platform_device *pdev) if (sdma->drvdata->script_addrs) sdma_add_scripts(sdma, sdma->drvdata->script_addrs); - if (pdata && pdata->script_addrs) - sdma_add_scripts(sdma, pdata->script_addrs); sdma->dma_device.dev = &pdev->dev; @@ -2110,30 +2098,18 @@ static int sdma_probe(struct platform_device *pdev) } /* - * Kick off firmware loading as the very last step: - * attempt to load firmware only if we're not on the error path, because - * the firmware callback requires a fully functional and allocated sdma - * instance. + * Because that device tree does not encode ROM script address, + * the RAM script in firmware is mandatory for device tree + * probe, otherwise it fails. */ - if (pdata) { - ret = sdma_get_firmware(sdma, pdata->fw_name); - if (ret) - dev_warn(&pdev->dev, "failed to get firmware from platform data\n"); + ret = of_property_read_string(np, "fsl,sdma-ram-script-name", + &fw_name); + if (ret) { + dev_warn(&pdev->dev, "failed to get firmware name\n"); } else { - /* - * Because that device tree does not encode ROM script address, - * the RAM script in firmware is mandatory for device tree - * probe, otherwise it fails. - */ - ret = of_property_read_string(np, "fsl,sdma-ram-script-name", - &fw_name); - if (ret) { - dev_warn(&pdev->dev, "failed to get firmware name\n"); - } else { - ret = sdma_get_firmware(sdma, fw_name); - if (ret) - dev_warn(&pdev->dev, "failed to get firmware from device tree\n"); - } + ret = sdma_get_firmware(sdma, fw_name); + if (ret) + dev_warn(&pdev->dev, "failed to get firmware from device tree\n"); } return 0; diff --git a/drivers/dma/lgm/Kconfig b/drivers/dma/lgm/Kconfig new file mode 100644 index 000000000000..9194330ed0f2 --- /dev/null +++ b/drivers/dma/lgm/Kconfig @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +config INTEL_LDMA + bool "Lightning Mountain centralized DMA controllers" + depends on X86 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for Intel Lightning Mountain SOC DMA controllers. + These controllers provide DMA capabilities for a variety of on-chip + devices such as HSNAND and GSWIP (Gigabit Switch IP). diff --git a/drivers/dma/lgm/Makefile b/drivers/dma/lgm/Makefile new file mode 100644 index 000000000000..f318a8eff464 --- /dev/null +++ b/drivers/dma/lgm/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_INTEL_LDMA) += lgm-dma.o diff --git a/drivers/dma/lgm/lgm-dma.c b/drivers/dma/lgm/lgm-dma.c new file mode 100644 index 000000000000..efe8bd3a0e2a --- /dev/null +++ b/drivers/dma/lgm/lgm-dma.c @@ -0,0 +1,1739 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Lightning Mountain centralized DMA controller driver + * + * Copyright (c) 2016 - 2020 Intel Corporation. + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/dmapool.h> +#include <linux/err.h> +#include <linux/export.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/of_dma.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> +#include <linux/reset.h> + +#include "../dmaengine.h" +#include "../virt-dma.h" + +#define DRIVER_NAME "lgm-dma" + +#define DMA_ID 0x0008 +#define DMA_ID_REV GENMASK(7, 0) +#define DMA_ID_PNR GENMASK(19, 16) +#define DMA_ID_CHNR GENMASK(26, 20) +#define DMA_ID_DW_128B BIT(27) +#define DMA_ID_AW_36B BIT(28) +#define DMA_VER32 0x32 +#define DMA_VER31 0x31 +#define DMA_VER22 0x0A + +#define DMA_CTRL 0x0010 +#define DMA_CTRL_RST BIT(0) +#define DMA_CTRL_DSRAM_PATH BIT(1) +#define DMA_CTRL_DBURST_WR BIT(3) +#define DMA_CTRL_VLD_DF_ACK BIT(4) +#define DMA_CTRL_CH_FL BIT(6) +#define DMA_CTRL_DS_FOD BIT(7) +#define DMA_CTRL_DRB BIT(8) +#define DMA_CTRL_ENBE BIT(9) +#define DMA_CTRL_DESC_TMOUT_CNT_V31 GENMASK(27, 16) +#define DMA_CTRL_DESC_TMOUT_EN_V31 BIT(30) +#define DMA_CTRL_PKTARB BIT(31) + +#define DMA_CPOLL 0x0014 +#define DMA_CPOLL_CNT GENMASK(15, 4) +#define DMA_CPOLL_EN BIT(31) + +#define DMA_CS 0x0018 +#define DMA_CS_MASK GENMASK(5, 0) + +#define DMA_CCTRL 0x001C +#define DMA_CCTRL_ON BIT(0) +#define DMA_CCTRL_RST BIT(1) +#define DMA_CCTRL_CH_POLL_EN BIT(2) +#define DMA_CCTRL_CH_ABC BIT(3) /* Adaptive Burst Chop */ +#define DMA_CDBA_MSB GENMASK(7, 4) +#define DMA_CCTRL_DIR_TX BIT(8) +#define DMA_CCTRL_CLASS GENMASK(11, 9) +#define DMA_CCTRL_CLASSH GENMASK(19, 18) +#define DMA_CCTRL_WR_NP_EN BIT(21) +#define DMA_CCTRL_PDEN BIT(23) +#define DMA_MAX_CLASS (SZ_32 - 1) + +#define DMA_CDBA 0x0020 +#define DMA_CDLEN 0x0024 +#define DMA_CIS 0x0028 +#define DMA_CIE 0x002C +#define DMA_CI_EOP BIT(1) +#define DMA_CI_DUR BIT(2) +#define DMA_CI_DESCPT BIT(3) +#define DMA_CI_CHOFF BIT(4) +#define DMA_CI_RDERR BIT(5) +#define DMA_CI_ALL \ + (DMA_CI_EOP | DMA_CI_DUR | DMA_CI_DESCPT | DMA_CI_CHOFF | DMA_CI_RDERR) + +#define DMA_PS 0x0040 +#define DMA_PCTRL 0x0044 +#define DMA_PCTRL_RXBL16 BIT(0) +#define DMA_PCTRL_TXBL16 BIT(1) +#define DMA_PCTRL_RXBL GENMASK(3, 2) +#define DMA_PCTRL_RXBL_8 3 +#define DMA_PCTRL_TXBL GENMASK(5, 4) +#define DMA_PCTRL_TXBL_8 3 +#define DMA_PCTRL_PDEN BIT(6) +#define DMA_PCTRL_RXBL32 BIT(7) +#define DMA_PCTRL_RXENDI GENMASK(9, 8) +#define DMA_PCTRL_TXENDI GENMASK(11, 10) +#define DMA_PCTRL_TXBL32 BIT(15) +#define DMA_PCTRL_MEM_FLUSH BIT(16) + +#define DMA_IRNEN1 0x00E8 +#define DMA_IRNCR1 0x00EC +#define DMA_IRNEN 0x00F4 +#define DMA_IRNCR 0x00F8 +#define DMA_C_DP_TICK 0x100 +#define DMA_C_DP_TICK_TIKNARB GENMASK(15, 0) +#define DMA_C_DP_TICK_TIKARB GENMASK(31, 16) + +#define DMA_C_HDRM 0x110 +/* + * If header mode is set in DMA descriptor, + * If bit 30 is disabled, HDR_LEN must be configured according to channel + * requirement. + * If bit 30 is enabled(checksum with heade mode), HDR_LEN has no need to + * be configured. It will enable check sum for switch + * If header mode is not set in DMA descriptor, + * This register setting doesn't matter + */ +#define DMA_C_HDRM_HDR_SUM BIT(30) + +#define DMA_C_BOFF 0x120 +#define DMA_C_BOFF_BOF_LEN GENMASK(7, 0) +#define DMA_C_BOFF_EN BIT(31) + +#define DMA_ORRC 0x190 +#define DMA_ORRC_ORRCNT GENMASK(8, 4) +#define DMA_ORRC_EN BIT(31) + +#define DMA_C_ENDIAN 0x200 +#define DMA_C_END_DATAENDI GENMASK(1, 0) +#define DMA_C_END_DE_EN BIT(7) +#define DMA_C_END_DESENDI GENMASK(9, 8) +#define DMA_C_END_DES_EN BIT(16) + +/* DMA controller capability */ +#define DMA_ADDR_36BIT BIT(0) +#define DMA_DATA_128BIT BIT(1) +#define DMA_CHAN_FLOW_CTL BIT(2) +#define DMA_DESC_FOD BIT(3) +#define DMA_DESC_IN_SRAM BIT(4) +#define DMA_EN_BYTE_EN BIT(5) +#define DMA_DBURST_WR BIT(6) +#define DMA_VALID_DESC_FETCH_ACK BIT(7) +#define DMA_DFT_DRB BIT(8) + +#define DMA_ORRC_MAX_CNT (SZ_32 - 1) +#define DMA_DFT_POLL_CNT SZ_4 +#define DMA_DFT_BURST_V22 SZ_2 +#define DMA_BURSTL_8DW SZ_8 +#define DMA_BURSTL_16DW SZ_16 +#define DMA_BURSTL_32DW SZ_32 +#define DMA_DFT_BURST DMA_BURSTL_16DW +#define DMA_MAX_DESC_NUM (SZ_8K - 1) +#define DMA_CHAN_BOFF_MAX (SZ_256 - 1) +#define DMA_DFT_ENDIAN 0 + +#define DMA_DFT_DESC_TCNT 50 +#define DMA_HDR_LEN_MAX (SZ_16K - 1) + +/* DMA flags */ +#define DMA_TX_CH BIT(0) +#define DMA_RX_CH BIT(1) +#define DEVICE_ALLOC_DESC BIT(2) +#define CHAN_IN_USE BIT(3) +#define DMA_HW_DESC BIT(4) + +/* Descriptor fields */ +#define DESC_DATA_LEN GENMASK(15, 0) +#define DESC_BYTE_OFF GENMASK(25, 23) +#define DESC_EOP BIT(28) +#define DESC_SOP BIT(29) +#define DESC_C BIT(30) +#define DESC_OWN BIT(31) + +#define DMA_CHAN_RST 1 +#define DMA_MAX_SIZE (BIT(16) - 1) +#define MAX_LOWER_CHANS 32 +#define MASK_LOWER_CHANS GENMASK(4, 0) +#define DMA_OWN 1 +#define HIGH_4_BITS GENMASK(3, 0) +#define DMA_DFT_DESC_NUM 1 +#define DMA_PKT_DROP_DIS 0 + +enum ldma_chan_on_off { + DMA_CH_OFF = 0, + DMA_CH_ON = 1, +}; + +enum { + DMA_TYPE_TX = 0, + DMA_TYPE_RX, + DMA_TYPE_MCPY, +}; + +struct ldma_dev; +struct ldma_port; + +struct ldma_chan { + struct virt_dma_chan vchan; + struct ldma_port *port; /* back pointer */ + char name[8]; /* Channel name */ + int nr; /* Channel id in hardware */ + u32 flags; /* central way or channel based way */ + enum ldma_chan_on_off onoff; + dma_addr_t desc_phys; + void *desc_base; /* Virtual address */ + u32 desc_cnt; /* Number of descriptors */ + int rst; + u32 hdrm_len; + bool hdrm_csum; + u32 boff_len; + u32 data_endian; + u32 desc_endian; + bool pden; + bool desc_rx_np; + bool data_endian_en; + bool desc_endian_en; + bool abc_en; + bool desc_init; + struct dma_pool *desc_pool; /* Descriptors pool */ + u32 desc_num; + struct dw2_desc_sw *ds; + struct work_struct work; + struct dma_slave_config config; +}; + +struct ldma_port { + struct ldma_dev *ldev; /* back pointer */ + u32 portid; + u32 rxbl; + u32 txbl; + u32 rxendi; + u32 txendi; + u32 pkt_drop; +}; + +/* Instance specific data */ +struct ldma_inst_data { + bool desc_in_sram; + bool chan_fc; + bool desc_fod; /* Fetch On Demand */ + bool valid_desc_fetch_ack; + u32 orrc; /* Outstanding read count */ + const char *name; + u32 type; +}; + +struct ldma_dev { + struct device *dev; + void __iomem *base; + struct reset_control *rst; + struct clk *core_clk; + struct dma_device dma_dev; + u32 ver; + int irq; + struct ldma_port *ports; + struct ldma_chan *chans; /* channel list on this DMA or port */ + spinlock_t dev_lock; /* Controller register exclusive */ + u32 chan_nrs; + u32 port_nrs; + u32 channels_mask; + u32 flags; + u32 pollcnt; + const struct ldma_inst_data *inst; + struct workqueue_struct *wq; +}; + +struct dw2_desc { + u32 field; + u32 addr; +} __packed __aligned(8); + +struct dw2_desc_sw { + struct virt_dma_desc vdesc; + struct ldma_chan *chan; + dma_addr_t desc_phys; + size_t desc_cnt; + size_t size; + struct dw2_desc *desc_hw; +}; + +static inline void +ldma_update_bits(struct ldma_dev *d, u32 mask, u32 val, u32 ofs) +{ + u32 old_val, new_val; + + old_val = readl(d->base + ofs); + new_val = (old_val & ~mask) | (val & mask); + + if (new_val != old_val) + writel(new_val, d->base + ofs); +} + +static inline struct ldma_chan *to_ldma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct ldma_chan, vchan.chan); +} + +static inline struct ldma_dev *to_ldma_dev(struct dma_device *dma_dev) +{ + return container_of(dma_dev, struct ldma_dev, dma_dev); +} + +static inline struct dw2_desc_sw *to_lgm_dma_desc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct dw2_desc_sw, vdesc); +} + +static inline bool ldma_chan_tx(struct ldma_chan *c) +{ + return !!(c->flags & DMA_TX_CH); +} + +static inline bool ldma_chan_is_hw_desc(struct ldma_chan *c) +{ + return !!(c->flags & DMA_HW_DESC); +} + +static void ldma_dev_reset(struct ldma_dev *d) + +{ + unsigned long flags; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CTRL_RST, DMA_CTRL_RST, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_pkt_arb_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask = DMA_CTRL_PKTARB; + u32 val = enable ? DMA_CTRL_PKTARB : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_sram_desc_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask = DMA_CTRL_DSRAM_PATH; + u32 val = enable ? DMA_CTRL_DSRAM_PATH : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_chan_flow_ctl_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask, val; + + if (d->inst->type != DMA_TYPE_TX) + return; + + mask = DMA_CTRL_CH_FL; + val = enable ? DMA_CTRL_CH_FL : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_global_polling_enable(struct ldma_dev *d) +{ + unsigned long flags; + u32 mask = DMA_CPOLL_EN | DMA_CPOLL_CNT; + u32 val = DMA_CPOLL_EN; + + val |= FIELD_PREP(DMA_CPOLL_CNT, d->pollcnt); + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CPOLL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_desc_fetch_on_demand_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask, val; + + if (d->inst->type == DMA_TYPE_MCPY) + return; + + mask = DMA_CTRL_DS_FOD; + val = enable ? DMA_CTRL_DS_FOD : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_byte_enable_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask = DMA_CTRL_ENBE; + u32 val = enable ? DMA_CTRL_ENBE : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_orrc_cfg(struct ldma_dev *d) +{ + unsigned long flags; + u32 val = 0; + u32 mask; + + if (d->inst->type == DMA_TYPE_RX) + return; + + mask = DMA_ORRC_EN | DMA_ORRC_ORRCNT; + if (d->inst->orrc > 0 && d->inst->orrc <= DMA_ORRC_MAX_CNT) + val = DMA_ORRC_EN | FIELD_PREP(DMA_ORRC_ORRCNT, d->inst->orrc); + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_ORRC); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_df_tout_cfg(struct ldma_dev *d, bool enable, int tcnt) +{ + u32 mask = DMA_CTRL_DESC_TMOUT_CNT_V31; + unsigned long flags; + u32 val; + + if (enable) + val = DMA_CTRL_DESC_TMOUT_EN_V31 | FIELD_PREP(DMA_CTRL_DESC_TMOUT_CNT_V31, tcnt); + else + val = 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_dburst_wr_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask, val; + + if (d->inst->type != DMA_TYPE_RX && d->inst->type != DMA_TYPE_MCPY) + return; + + mask = DMA_CTRL_DBURST_WR; + val = enable ? DMA_CTRL_DBURST_WR : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_vld_fetch_ack_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask, val; + + if (d->inst->type != DMA_TYPE_TX) + return; + + mask = DMA_CTRL_VLD_DF_ACK; + val = enable ? DMA_CTRL_VLD_DF_ACK : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_drb_cfg(struct ldma_dev *d, int enable) +{ + unsigned long flags; + u32 mask = DMA_CTRL_DRB; + u32 val = enable ? DMA_CTRL_DRB : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static int ldma_dev_cfg(struct ldma_dev *d) +{ + bool enable; + + ldma_dev_pkt_arb_cfg(d, true); + ldma_dev_global_polling_enable(d); + + enable = !!(d->flags & DMA_DFT_DRB); + ldma_dev_drb_cfg(d, enable); + + enable = !!(d->flags & DMA_EN_BYTE_EN); + ldma_dev_byte_enable_cfg(d, enable); + + enable = !!(d->flags & DMA_CHAN_FLOW_CTL); + ldma_dev_chan_flow_ctl_cfg(d, enable); + + enable = !!(d->flags & DMA_DESC_FOD); + ldma_dev_desc_fetch_on_demand_cfg(d, enable); + + enable = !!(d->flags & DMA_DESC_IN_SRAM); + ldma_dev_sram_desc_cfg(d, enable); + + enable = !!(d->flags & DMA_DBURST_WR); + ldma_dev_dburst_wr_cfg(d, enable); + + enable = !!(d->flags & DMA_VALID_DESC_FETCH_ACK); + ldma_dev_vld_fetch_ack_cfg(d, enable); + + if (d->ver > DMA_VER22) { + ldma_dev_orrc_cfg(d); + ldma_dev_df_tout_cfg(d, true, DMA_DFT_DESC_TCNT); + } + + dev_dbg(d->dev, "%s Controller 0x%08x configuration done\n", + d->inst->name, readl(d->base + DMA_CTRL)); + + return 0; +} + +static int ldma_chan_cctrl_cfg(struct ldma_chan *c, u32 val) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 class_low, class_high; + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + reg = readl(d->base + DMA_CCTRL); + /* Read from hardware */ + if (reg & DMA_CCTRL_DIR_TX) + c->flags |= DMA_TX_CH; + else + c->flags |= DMA_RX_CH; + + /* Keep the class value unchanged */ + class_low = FIELD_GET(DMA_CCTRL_CLASS, reg); + class_high = FIELD_GET(DMA_CCTRL_CLASSH, reg); + val &= ~DMA_CCTRL_CLASS; + val |= FIELD_PREP(DMA_CCTRL_CLASS, class_low); + val &= ~DMA_CCTRL_CLASSH; + val |= FIELD_PREP(DMA_CCTRL_CLASSH, class_high); + writel(val, d->base + DMA_CCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + return 0; +} + +static void ldma_chan_irq_init(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + u32 enofs, crofs; + u32 cn_bit; + + if (c->nr < MAX_LOWER_CHANS) { + enofs = DMA_IRNEN; + crofs = DMA_IRNCR; + } else { + enofs = DMA_IRNEN1; + crofs = DMA_IRNCR1; + } + + cn_bit = BIT(c->nr & MASK_LOWER_CHANS); + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + + /* Clear all interrupts and disabled it */ + writel(0, d->base + DMA_CIE); + writel(DMA_CI_ALL, d->base + DMA_CIS); + + ldma_update_bits(d, cn_bit, 0, enofs); + writel(cn_bit, d->base + crofs); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_chan_set_class(struct ldma_chan *c, u32 val) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 class_val; + + if (d->inst->type == DMA_TYPE_MCPY || val > DMA_MAX_CLASS) + return; + + /* 3 bits low */ + class_val = FIELD_PREP(DMA_CCTRL_CLASS, val & 0x7); + /* 2 bits high */ + class_val |= FIELD_PREP(DMA_CCTRL_CLASSH, (val >> 3) & 0x3); + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, DMA_CCTRL_CLASS | DMA_CCTRL_CLASSH, class_val, + DMA_CCTRL); +} + +static int ldma_chan_on(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + + /* If descriptors not configured, not allow to turn on channel */ + if (WARN_ON(!c->desc_init)) + return -EINVAL; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, DMA_CCTRL_ON, DMA_CCTRL_ON, DMA_CCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + c->onoff = DMA_CH_ON; + + return 0; +} + +static int ldma_chan_off(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + u32 val; + int ret; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, DMA_CCTRL_ON, 0, DMA_CCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + ret = readl_poll_timeout_atomic(d->base + DMA_CCTRL, val, + !(val & DMA_CCTRL_ON), 0, 10000); + if (ret) + return ret; + + c->onoff = DMA_CH_OFF; + + return 0; +} + +static void ldma_chan_desc_hw_cfg(struct ldma_chan *c, dma_addr_t desc_base, + int desc_num) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + writel(lower_32_bits(desc_base), d->base + DMA_CDBA); + + /* Higher 4 bits of 36 bit addressing */ + if (IS_ENABLED(CONFIG_64BIT)) { + u32 hi = upper_32_bits(desc_base) & HIGH_4_BITS; + + ldma_update_bits(d, DMA_CDBA_MSB, + FIELD_PREP(DMA_CDBA_MSB, hi), DMA_CCTRL); + } + writel(desc_num, d->base + DMA_CDLEN); + spin_unlock_irqrestore(&d->dev_lock, flags); + + c->desc_init = true; +} + +static struct dma_async_tx_descriptor * +ldma_chan_desc_cfg(struct dma_chan *chan, dma_addr_t desc_base, int desc_num) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + struct dma_async_tx_descriptor *tx; + struct dw2_desc_sw *ds; + + if (!desc_num) { + dev_err(d->dev, "Channel %d must allocate descriptor first\n", + c->nr); + return NULL; + } + + if (desc_num > DMA_MAX_DESC_NUM) { + dev_err(d->dev, "Channel %d descriptor number out of range %d\n", + c->nr, desc_num); + return NULL; + } + + ldma_chan_desc_hw_cfg(c, desc_base, desc_num); + + c->flags |= DMA_HW_DESC; + c->desc_cnt = desc_num; + c->desc_phys = desc_base; + + ds = kzalloc(sizeof(*ds), GFP_NOWAIT); + if (!ds) + return NULL; + + tx = &ds->vdesc.tx; + dma_async_tx_descriptor_init(tx, chan); + + return tx; +} + +static int ldma_chan_reset(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + u32 val; + int ret; + + ret = ldma_chan_off(c); + if (ret) + return ret; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, DMA_CCTRL_RST, DMA_CCTRL_RST, DMA_CCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + ret = readl_poll_timeout_atomic(d->base + DMA_CCTRL, val, + !(val & DMA_CCTRL_RST), 0, 10000); + if (ret) + return ret; + + c->rst = 1; + c->desc_init = false; + + return 0; +} + +static void ldma_chan_byte_offset_cfg(struct ldma_chan *c, u32 boff_len) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask = DMA_C_BOFF_EN | DMA_C_BOFF_BOF_LEN; + u32 val; + + if (boff_len > 0 && boff_len <= DMA_CHAN_BOFF_MAX) + val = FIELD_PREP(DMA_C_BOFF_BOF_LEN, boff_len) | DMA_C_BOFF_EN; + else + val = 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_C_BOFF); +} + +static void ldma_chan_data_endian_cfg(struct ldma_chan *c, bool enable, + u32 endian_type) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask = DMA_C_END_DE_EN | DMA_C_END_DATAENDI; + u32 val; + + if (enable) + val = DMA_C_END_DE_EN | FIELD_PREP(DMA_C_END_DATAENDI, endian_type); + else + val = 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_C_ENDIAN); +} + +static void ldma_chan_desc_endian_cfg(struct ldma_chan *c, bool enable, + u32 endian_type) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask = DMA_C_END_DES_EN | DMA_C_END_DESENDI; + u32 val; + + if (enable) + val = DMA_C_END_DES_EN | FIELD_PREP(DMA_C_END_DESENDI, endian_type); + else + val = 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_C_ENDIAN); +} + +static void ldma_chan_hdr_mode_cfg(struct ldma_chan *c, u32 hdr_len, bool csum) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask, val; + + /* NB, csum disabled, hdr length must be provided */ + if (!csum && (!hdr_len || hdr_len > DMA_HDR_LEN_MAX)) + return; + + mask = DMA_C_HDRM_HDR_SUM; + val = DMA_C_HDRM_HDR_SUM; + + if (!csum && hdr_len) + val = hdr_len; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_C_HDRM); +} + +static void ldma_chan_rxwr_np_cfg(struct ldma_chan *c, bool enable) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask, val; + + /* Only valid for RX channel */ + if (ldma_chan_tx(c)) + return; + + mask = DMA_CCTRL_WR_NP_EN; + val = enable ? DMA_CCTRL_WR_NP_EN : 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_CCTRL); +} + +static void ldma_chan_abc_cfg(struct ldma_chan *c, bool enable) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask, val; + + if (d->ver < DMA_VER32 || ldma_chan_tx(c)) + return; + + mask = DMA_CCTRL_CH_ABC; + val = enable ? DMA_CCTRL_CH_ABC : 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_CCTRL); +} + +static int ldma_port_cfg(struct ldma_port *p) +{ + unsigned long flags; + struct ldma_dev *d; + u32 reg; + + d = p->ldev; + reg = FIELD_PREP(DMA_PCTRL_TXENDI, p->txendi); + reg |= FIELD_PREP(DMA_PCTRL_RXENDI, p->rxendi); + + if (d->ver == DMA_VER22) { + reg |= FIELD_PREP(DMA_PCTRL_TXBL, p->txbl); + reg |= FIELD_PREP(DMA_PCTRL_RXBL, p->rxbl); + } else { + reg |= FIELD_PREP(DMA_PCTRL_PDEN, p->pkt_drop); + + if (p->txbl == DMA_BURSTL_32DW) + reg |= DMA_PCTRL_TXBL32; + else if (p->txbl == DMA_BURSTL_16DW) + reg |= DMA_PCTRL_TXBL16; + else + reg |= FIELD_PREP(DMA_PCTRL_TXBL, DMA_PCTRL_TXBL_8); + + if (p->rxbl == DMA_BURSTL_32DW) + reg |= DMA_PCTRL_RXBL32; + else if (p->rxbl == DMA_BURSTL_16DW) + reg |= DMA_PCTRL_RXBL16; + else + reg |= FIELD_PREP(DMA_PCTRL_RXBL, DMA_PCTRL_RXBL_8); + } + + spin_lock_irqsave(&d->dev_lock, flags); + writel(p->portid, d->base + DMA_PS); + writel(reg, d->base + DMA_PCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + reg = readl(d->base + DMA_PCTRL); /* read back */ + dev_dbg(d->dev, "Port Control 0x%08x configuration done\n", reg); + + return 0; +} + +static int ldma_chan_cfg(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + u32 reg; + + reg = c->pden ? DMA_CCTRL_PDEN : 0; + reg |= c->onoff ? DMA_CCTRL_ON : 0; + reg |= c->rst ? DMA_CCTRL_RST : 0; + + ldma_chan_cctrl_cfg(c, reg); + ldma_chan_irq_init(c); + + if (d->ver <= DMA_VER22) + return 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_chan_set_class(c, c->nr); + ldma_chan_byte_offset_cfg(c, c->boff_len); + ldma_chan_data_endian_cfg(c, c->data_endian_en, c->data_endian); + ldma_chan_desc_endian_cfg(c, c->desc_endian_en, c->desc_endian); + ldma_chan_hdr_mode_cfg(c, c->hdrm_len, c->hdrm_csum); + ldma_chan_rxwr_np_cfg(c, c->desc_rx_np); + ldma_chan_abc_cfg(c, c->abc_en); + spin_unlock_irqrestore(&d->dev_lock, flags); + + if (ldma_chan_is_hw_desc(c)) + ldma_chan_desc_hw_cfg(c, c->desc_phys, c->desc_cnt); + + return 0; +} + +static void ldma_dev_init(struct ldma_dev *d) +{ + unsigned long ch_mask = (unsigned long)d->channels_mask; + struct ldma_port *p; + struct ldma_chan *c; + int i; + u32 j; + + spin_lock_init(&d->dev_lock); + ldma_dev_reset(d); + ldma_dev_cfg(d); + + /* DMA port initialization */ + for (i = 0; i < d->port_nrs; i++) { + p = &d->ports[i]; + ldma_port_cfg(p); + } + + /* DMA channel initialization */ + for_each_set_bit(j, &ch_mask, d->chan_nrs) { + c = &d->chans[j]; + ldma_chan_cfg(c); + } +} + +static int ldma_cfg_init(struct ldma_dev *d) +{ + struct fwnode_handle *fwnode = dev_fwnode(d->dev); + struct ldma_port *p; + int i; + + if (fwnode_property_read_bool(fwnode, "intel,dma-byte-en")) + d->flags |= DMA_EN_BYTE_EN; + + if (fwnode_property_read_bool(fwnode, "intel,dma-dburst-wr")) + d->flags |= DMA_DBURST_WR; + + if (fwnode_property_read_bool(fwnode, "intel,dma-drb")) + d->flags |= DMA_DFT_DRB; + + if (fwnode_property_read_u32(fwnode, "intel,dma-poll-cnt", + &d->pollcnt)) + d->pollcnt = DMA_DFT_POLL_CNT; + + if (d->inst->chan_fc) + d->flags |= DMA_CHAN_FLOW_CTL; + + if (d->inst->desc_fod) + d->flags |= DMA_DESC_FOD; + + if (d->inst->desc_in_sram) + d->flags |= DMA_DESC_IN_SRAM; + + if (d->inst->valid_desc_fetch_ack) + d->flags |= DMA_VALID_DESC_FETCH_ACK; + + if (d->ver > DMA_VER22) { + if (!d->port_nrs) + return -EINVAL; + + for (i = 0; i < d->port_nrs; i++) { + p = &d->ports[i]; + p->rxendi = DMA_DFT_ENDIAN; + p->txendi = DMA_DFT_ENDIAN; + p->rxbl = DMA_DFT_BURST; + p->txbl = DMA_DFT_BURST; + p->pkt_drop = DMA_PKT_DROP_DIS; + } + } + + return 0; +} + +static void dma_free_desc_resource(struct virt_dma_desc *vdesc) +{ + struct dw2_desc_sw *ds = to_lgm_dma_desc(vdesc); + struct ldma_chan *c = ds->chan; + + dma_pool_free(c->desc_pool, ds->desc_hw, ds->desc_phys); + kfree(ds); +} + +static struct dw2_desc_sw * +dma_alloc_desc_resource(int num, struct ldma_chan *c) +{ + struct device *dev = c->vchan.chan.device->dev; + struct dw2_desc_sw *ds; + + if (num > c->desc_num) { + dev_err(dev, "sg num %d exceed max %d\n", num, c->desc_num); + return NULL; + } + + ds = kzalloc(sizeof(*ds), GFP_NOWAIT); + if (!ds) + return NULL; + + ds->chan = c; + ds->desc_hw = dma_pool_zalloc(c->desc_pool, GFP_ATOMIC, + &ds->desc_phys); + if (!ds->desc_hw) { + dev_dbg(dev, "out of memory for link descriptor\n"); + kfree(ds); + return NULL; + } + ds->desc_cnt = num; + + return ds; +} + +static void ldma_chan_irq_en(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + + spin_lock_irqsave(&d->dev_lock, flags); + writel(c->nr, d->base + DMA_CS); + writel(DMA_CI_EOP, d->base + DMA_CIE); + writel(BIT(c->nr), d->base + DMA_IRNEN); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_issue_pending(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + + if (d->ver == DMA_VER22) { + spin_lock_irqsave(&c->vchan.lock, flags); + if (vchan_issue_pending(&c->vchan)) { + struct virt_dma_desc *vdesc; + + /* Get the next descriptor */ + vdesc = vchan_next_desc(&c->vchan); + if (!vdesc) { + c->ds = NULL; + spin_unlock_irqrestore(&c->vchan.lock, flags); + return; + } + list_del(&vdesc->node); + c->ds = to_lgm_dma_desc(vdesc); + ldma_chan_desc_hw_cfg(c, c->ds->desc_phys, c->ds->desc_cnt); + ldma_chan_irq_en(c); + } + spin_unlock_irqrestore(&c->vchan.lock, flags); + } + ldma_chan_on(c); +} + +static void ldma_synchronize(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + + /* + * clear any pending work if any. In that + * case the resource needs to be free here. + */ + cancel_work_sync(&c->work); + vchan_synchronize(&c->vchan); + if (c->ds) + dma_free_desc_resource(&c->ds->vdesc); +} + +static int ldma_terminate_all(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&c->vchan.lock, flags); + vchan_get_all_descriptors(&c->vchan, &head); + spin_unlock_irqrestore(&c->vchan.lock, flags); + vchan_dma_desc_free_list(&c->vchan, &head); + + return ldma_chan_reset(c); +} + +static int ldma_resume_chan(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + + ldma_chan_on(c); + + return 0; +} + +static int ldma_pause_chan(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + + return ldma_chan_off(c); +} + +static enum dma_status +ldma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + enum dma_status status = DMA_COMPLETE; + + if (d->ver == DMA_VER22) + status = dma_cookie_status(chan, cookie, txstate); + + return status; +} + +static void dma_chan_irq(int irq, void *data) +{ + struct ldma_chan *c = data; + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 stat; + + /* Disable channel interrupts */ + writel(c->nr, d->base + DMA_CS); + stat = readl(d->base + DMA_CIS); + if (!stat) + return; + + writel(readl(d->base + DMA_CIE) & ~DMA_CI_ALL, d->base + DMA_CIE); + writel(stat, d->base + DMA_CIS); + queue_work(d->wq, &c->work); +} + +static irqreturn_t dma_interrupt(int irq, void *dev_id) +{ + struct ldma_dev *d = dev_id; + struct ldma_chan *c; + unsigned long irncr; + u32 cid; + + irncr = readl(d->base + DMA_IRNCR); + if (!irncr) { + dev_err(d->dev, "dummy interrupt\n"); + return IRQ_NONE; + } + + for_each_set_bit(cid, &irncr, d->chan_nrs) { + /* Mask */ + writel(readl(d->base + DMA_IRNEN) & ~BIT(cid), d->base + DMA_IRNEN); + /* Ack */ + writel(readl(d->base + DMA_IRNCR) | BIT(cid), d->base + DMA_IRNCR); + + c = &d->chans[cid]; + dma_chan_irq(irq, c); + } + + return IRQ_HANDLED; +} + +static void prep_slave_burst_len(struct ldma_chan *c) +{ + struct ldma_port *p = c->port; + struct dma_slave_config *cfg = &c->config; + + if (cfg->dst_maxburst) + cfg->src_maxburst = cfg->dst_maxburst; + + /* TX and RX has the same burst length */ + p->txbl = ilog2(cfg->src_maxburst); + p->rxbl = p->txbl; +} + +static struct dma_async_tx_descriptor * +ldma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sglen, enum dma_transfer_direction dir, + unsigned long flags, void *context) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + size_t len, avail, total = 0; + struct dw2_desc *hw_ds; + struct dw2_desc_sw *ds; + struct scatterlist *sg; + int num = sglen, i; + dma_addr_t addr; + + if (!sgl) + return NULL; + + if (d->ver > DMA_VER22) + return ldma_chan_desc_cfg(chan, sgl->dma_address, sglen); + + for_each_sg(sgl, sg, sglen, i) { + avail = sg_dma_len(sg); + if (avail > DMA_MAX_SIZE) + num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1; + } + + ds = dma_alloc_desc_resource(num, c); + if (!ds) + return NULL; + + c->ds = ds; + + num = 0; + /* sop and eop has to be handled nicely */ + for_each_sg(sgl, sg, sglen, i) { + addr = sg_dma_address(sg); + avail = sg_dma_len(sg); + total += avail; + + do { + len = min_t(size_t, avail, DMA_MAX_SIZE); + + hw_ds = &ds->desc_hw[num]; + switch (sglen) { + case 1: + hw_ds->field &= ~DESC_SOP; + hw_ds->field |= FIELD_PREP(DESC_SOP, 1); + + hw_ds->field &= ~DESC_EOP; + hw_ds->field |= FIELD_PREP(DESC_EOP, 1); + break; + default: + if (num == 0) { + hw_ds->field &= ~DESC_SOP; + hw_ds->field |= FIELD_PREP(DESC_SOP, 1); + + hw_ds->field &= ~DESC_EOP; + hw_ds->field |= FIELD_PREP(DESC_EOP, 0); + } else if (num == (sglen - 1)) { + hw_ds->field &= ~DESC_SOP; + hw_ds->field |= FIELD_PREP(DESC_SOP, 0); + hw_ds->field &= ~DESC_EOP; + hw_ds->field |= FIELD_PREP(DESC_EOP, 1); + } else { + hw_ds->field &= ~DESC_SOP; + hw_ds->field |= FIELD_PREP(DESC_SOP, 0); + + hw_ds->field &= ~DESC_EOP; + hw_ds->field |= FIELD_PREP(DESC_EOP, 0); + } + break; + } + /* Only 32 bit address supported */ + hw_ds->addr = (u32)addr; + + hw_ds->field &= ~DESC_DATA_LEN; + hw_ds->field |= FIELD_PREP(DESC_DATA_LEN, len); + + hw_ds->field &= ~DESC_C; + hw_ds->field |= FIELD_PREP(DESC_C, 0); + + hw_ds->field &= ~DESC_BYTE_OFF; + hw_ds->field |= FIELD_PREP(DESC_BYTE_OFF, addr & 0x3); + + /* Ensure data ready before ownership change */ + wmb(); + hw_ds->field &= ~DESC_OWN; + hw_ds->field |= FIELD_PREP(DESC_OWN, DMA_OWN); + + /* Ensure ownership changed before moving forward */ + wmb(); + num++; + addr += len; + avail -= len; + } while (avail); + } + + ds->size = total; + prep_slave_burst_len(c); + + return vchan_tx_prep(&c->vchan, &ds->vdesc, DMA_CTRL_ACK); +} + +static int +ldma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) +{ + struct ldma_chan *c = to_ldma_chan(chan); + + memcpy(&c->config, cfg, sizeof(c->config)); + + return 0; +} + +static int ldma_alloc_chan_resources(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + struct device *dev = c->vchan.chan.device->dev; + size_t desc_sz; + + if (d->ver > DMA_VER22) { + c->flags |= CHAN_IN_USE; + return 0; + } + + if (c->desc_pool) + return c->desc_num; + + desc_sz = c->desc_num * sizeof(struct dw2_desc); + c->desc_pool = dma_pool_create(c->name, dev, desc_sz, + __alignof__(struct dw2_desc), 0); + + if (!c->desc_pool) { + dev_err(dev, "unable to allocate descriptor pool\n"); + return -ENOMEM; + } + + return c->desc_num; +} + +static void ldma_free_chan_resources(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + + if (d->ver == DMA_VER22) { + dma_pool_destroy(c->desc_pool); + c->desc_pool = NULL; + vchan_free_chan_resources(to_virt_chan(chan)); + ldma_chan_reset(c); + } else { + c->flags &= ~CHAN_IN_USE; + } +} + +static void dma_work(struct work_struct *work) +{ + struct ldma_chan *c = container_of(work, struct ldma_chan, work); + struct dma_async_tx_descriptor *tx = &c->ds->vdesc.tx; + struct virt_dma_chan *vc = &c->vchan; + struct dmaengine_desc_callback cb; + struct virt_dma_desc *vd, *_vd; + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&c->vchan.lock, flags); + list_splice_tail_init(&vc->desc_completed, &head); + spin_unlock_irqrestore(&c->vchan.lock, flags); + dmaengine_desc_get_callback(tx, &cb); + dma_cookie_complete(tx); + dmaengine_desc_callback_invoke(&cb, NULL); + + list_for_each_entry_safe(vd, _vd, &head, node) { + dmaengine_desc_get_callback(tx, &cb); + dma_cookie_complete(tx); + list_del(&vd->node); + dmaengine_desc_callback_invoke(&cb, NULL); + + vchan_vdesc_fini(vd); + } + c->ds = NULL; +} + +static void +update_burst_len_v22(struct ldma_chan *c, struct ldma_port *p, u32 burst) +{ + if (ldma_chan_tx(c)) + p->txbl = ilog2(burst); + else + p->rxbl = ilog2(burst); +} + +static void +update_burst_len_v3X(struct ldma_chan *c, struct ldma_port *p, u32 burst) +{ + if (ldma_chan_tx(c)) + p->txbl = burst; + else + p->rxbl = burst; +} + +static int +update_client_configs(struct of_dma *ofdma, struct of_phandle_args *spec) +{ + struct ldma_dev *d = ofdma->of_dma_data; + u32 chan_id = spec->args[0]; + u32 port_id = spec->args[1]; + u32 burst = spec->args[2]; + struct ldma_port *p; + struct ldma_chan *c; + + if (chan_id >= d->chan_nrs || port_id >= d->port_nrs) + return 0; + + p = &d->ports[port_id]; + c = &d->chans[chan_id]; + c->port = p; + + if (d->ver == DMA_VER22) + update_burst_len_v22(c, p, burst); + else + update_burst_len_v3X(c, p, burst); + + ldma_port_cfg(p); + + return 1; +} + +static struct dma_chan *ldma_xlate(struct of_phandle_args *spec, + struct of_dma *ofdma) +{ + struct ldma_dev *d = ofdma->of_dma_data; + u32 chan_id = spec->args[0]; + int ret; + + if (!spec->args_count) + return NULL; + + /* if args_count is 1 driver use default settings */ + if (spec->args_count > 1) { + ret = update_client_configs(ofdma, spec); + if (!ret) + return NULL; + } + + return dma_get_slave_channel(&d->chans[chan_id].vchan.chan); +} + +static void ldma_dma_init_v22(int i, struct ldma_dev *d) +{ + struct ldma_chan *c; + + c = &d->chans[i]; + c->nr = i; /* Real channel number */ + c->rst = DMA_CHAN_RST; + c->desc_num = DMA_DFT_DESC_NUM; + snprintf(c->name, sizeof(c->name), "chan%d", c->nr); + INIT_WORK(&c->work, dma_work); + c->vchan.desc_free = dma_free_desc_resource; + vchan_init(&c->vchan, &d->dma_dev); +} + +static void ldma_dma_init_v3X(int i, struct ldma_dev *d) +{ + struct ldma_chan *c; + + c = &d->chans[i]; + c->data_endian = DMA_DFT_ENDIAN; + c->desc_endian = DMA_DFT_ENDIAN; + c->data_endian_en = false; + c->desc_endian_en = false; + c->desc_rx_np = false; + c->flags |= DEVICE_ALLOC_DESC; + c->onoff = DMA_CH_OFF; + c->rst = DMA_CHAN_RST; + c->abc_en = true; + c->hdrm_csum = false; + c->boff_len = 0; + c->nr = i; + c->vchan.desc_free = dma_free_desc_resource; + vchan_init(&c->vchan, &d->dma_dev); +} + +static int ldma_init_v22(struct ldma_dev *d, struct platform_device *pdev) +{ + int ret; + + ret = device_property_read_u32(d->dev, "dma-channels", &d->chan_nrs); + if (ret < 0) { + dev_err(d->dev, "unable to read dma-channels property\n"); + return ret; + } + + d->irq = platform_get_irq(pdev, 0); + if (d->irq < 0) + return d->irq; + + ret = devm_request_irq(&pdev->dev, d->irq, dma_interrupt, 0, + DRIVER_NAME, d); + if (ret) + return ret; + + d->wq = alloc_ordered_workqueue("dma_wq", WQ_MEM_RECLAIM | + WQ_HIGHPRI); + if (!d->wq) + return -ENOMEM; + + return 0; +} + +static void ldma_clk_disable(void *data) +{ + struct ldma_dev *d = data; + + clk_disable_unprepare(d->core_clk); + reset_control_assert(d->rst); +} + +static const struct ldma_inst_data dma0 = { + .name = "dma0", + .chan_fc = false, + .desc_fod = false, + .desc_in_sram = false, + .valid_desc_fetch_ack = false, +}; + +static const struct ldma_inst_data dma2tx = { + .name = "dma2tx", + .type = DMA_TYPE_TX, + .orrc = 16, + .chan_fc = true, + .desc_fod = true, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct ldma_inst_data dma1rx = { + .name = "dma1rx", + .type = DMA_TYPE_RX, + .orrc = 16, + .chan_fc = false, + .desc_fod = true, + .desc_in_sram = true, + .valid_desc_fetch_ack = false, +}; + +static const struct ldma_inst_data dma1tx = { + .name = "dma1tx", + .type = DMA_TYPE_TX, + .orrc = 16, + .chan_fc = true, + .desc_fod = true, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct ldma_inst_data dma0tx = { + .name = "dma0tx", + .type = DMA_TYPE_TX, + .orrc = 16, + .chan_fc = true, + .desc_fod = true, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct ldma_inst_data dma3 = { + .name = "dma3", + .type = DMA_TYPE_MCPY, + .orrc = 16, + .chan_fc = false, + .desc_fod = false, + .desc_in_sram = true, + .valid_desc_fetch_ack = false, +}; + +static const struct ldma_inst_data toe_dma30 = { + .name = "toe_dma30", + .type = DMA_TYPE_MCPY, + .orrc = 16, + .chan_fc = false, + .desc_fod = false, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct ldma_inst_data toe_dma31 = { + .name = "toe_dma31", + .type = DMA_TYPE_MCPY, + .orrc = 16, + .chan_fc = false, + .desc_fod = false, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct of_device_id intel_ldma_match[] = { + { .compatible = "intel,lgm-cdma", .data = &dma0}, + { .compatible = "intel,lgm-dma2tx", .data = &dma2tx}, + { .compatible = "intel,lgm-dma1rx", .data = &dma1rx}, + { .compatible = "intel,lgm-dma1tx", .data = &dma1tx}, + { .compatible = "intel,lgm-dma0tx", .data = &dma0tx}, + { .compatible = "intel,lgm-dma3", .data = &dma3}, + { .compatible = "intel,lgm-toe-dma30", .data = &toe_dma30}, + { .compatible = "intel,lgm-toe-dma31", .data = &toe_dma31}, + {} +}; + +static int intel_ldma_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dma_device *dma_dev; + unsigned long ch_mask; + struct ldma_chan *c; + struct ldma_port *p; + struct ldma_dev *d; + u32 id, bitn = 32, j; + int i, ret; + + d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL); + if (!d) + return -ENOMEM; + + /* Link controller to platform device */ + d->dev = &pdev->dev; + + d->inst = device_get_match_data(dev); + if (!d->inst) { + dev_err(dev, "No device match found\n"); + return -ENODEV; + } + + d->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(d->base)) + return PTR_ERR(d->base); + + /* Power up and reset the dma engine, some DMAs always on?? */ + d->core_clk = devm_clk_get_optional(dev, NULL); + if (IS_ERR(d->core_clk)) + return PTR_ERR(d->core_clk); + clk_prepare_enable(d->core_clk); + + d->rst = devm_reset_control_get_optional(dev, NULL); + if (IS_ERR(d->rst)) + return PTR_ERR(d->rst); + reset_control_deassert(d->rst); + + ret = devm_add_action_or_reset(dev, ldma_clk_disable, d); + if (ret) { + dev_err(dev, "Failed to devm_add_action_or_reset, %d\n", ret); + return ret; + } + + id = readl(d->base + DMA_ID); + d->chan_nrs = FIELD_GET(DMA_ID_CHNR, id); + d->port_nrs = FIELD_GET(DMA_ID_PNR, id); + d->ver = FIELD_GET(DMA_ID_REV, id); + + if (id & DMA_ID_AW_36B) + d->flags |= DMA_ADDR_36BIT; + + if (IS_ENABLED(CONFIG_64BIT) && (id & DMA_ID_AW_36B)) + bitn = 36; + + if (id & DMA_ID_DW_128B) + d->flags |= DMA_DATA_128BIT; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(bitn)); + if (ret) { + dev_err(dev, "No usable DMA configuration\n"); + return ret; + } + + if (d->ver == DMA_VER22) { + ret = ldma_init_v22(d, pdev); + if (ret) + return ret; + } + + ret = device_property_read_u32(dev, "dma-channel-mask", &d->channels_mask); + if (ret < 0) + d->channels_mask = GENMASK(d->chan_nrs - 1, 0); + + dma_dev = &d->dma_dev; + + dma_cap_zero(dma_dev->cap_mask); + dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); + + /* Channel initializations */ + INIT_LIST_HEAD(&dma_dev->channels); + + /* Port Initializations */ + d->ports = devm_kcalloc(dev, d->port_nrs, sizeof(*p), GFP_KERNEL); + if (!d->ports) + return -ENOMEM; + + /* Channels Initializations */ + d->chans = devm_kcalloc(d->dev, d->chan_nrs, sizeof(*c), GFP_KERNEL); + if (!d->chans) + return -ENOMEM; + + for (i = 0; i < d->port_nrs; i++) { + p = &d->ports[i]; + p->portid = i; + p->ldev = d; + } + + ret = ldma_cfg_init(d); + if (ret) + return ret; + + dma_dev->dev = &pdev->dev; + + ch_mask = (unsigned long)d->channels_mask; + for_each_set_bit(j, &ch_mask, d->chan_nrs) { + if (d->ver == DMA_VER22) + ldma_dma_init_v22(j, d); + else + ldma_dma_init_v3X(j, d); + } + + dma_dev->device_alloc_chan_resources = ldma_alloc_chan_resources; + dma_dev->device_free_chan_resources = ldma_free_chan_resources; + dma_dev->device_terminate_all = ldma_terminate_all; + dma_dev->device_issue_pending = ldma_issue_pending; + dma_dev->device_tx_status = ldma_tx_status; + dma_dev->device_resume = ldma_resume_chan; + dma_dev->device_pause = ldma_pause_chan; + dma_dev->device_prep_slave_sg = ldma_prep_slave_sg; + + if (d->ver == DMA_VER22) { + dma_dev->device_config = ldma_slave_config; + dma_dev->device_synchronize = ldma_synchronize; + dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + dma_dev->directions = BIT(DMA_MEM_TO_DEV) | + BIT(DMA_DEV_TO_MEM); + dma_dev->residue_granularity = + DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + } + + platform_set_drvdata(pdev, d); + + ldma_dev_init(d); + + ret = dma_async_device_register(dma_dev); + if (ret) { + dev_err(dev, "Failed to register slave DMA engine device\n"); + return ret; + } + + ret = of_dma_controller_register(pdev->dev.of_node, ldma_xlate, d); + if (ret) { + dev_err(dev, "Failed to register of DMA controller\n"); + dma_async_device_unregister(dma_dev); + return ret; + } + + dev_info(dev, "Init done - rev: %x, ports: %d channels: %d\n", d->ver, + d->port_nrs, d->chan_nrs); + + return 0; +} + +static struct platform_driver intel_ldma_driver = { + .probe = intel_ldma_probe, + .driver = { + .name = DRIVER_NAME, + .of_match_table = intel_ldma_match, + }, +}; + +/* + * Perform this driver as device_initcall to make sure initialization happens + * before its DMA clients of some are platform specific and also to provide + * registered DMA channels and DMA capabilities to clients before their + * initialization. + */ +static int __init intel_ldma_init(void) +{ + return platform_driver_register(&intel_ldma_driver); +} + +device_initcall(intel_ldma_init); diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index b84303be8edf..89f1814ff27a 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -18,7 +18,6 @@ #include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/of.h> -#include <linux/dma/mmp-pdma.h> #include "dmaengine.h" @@ -1148,19 +1147,6 @@ static struct platform_driver mmp_pdma_driver = { .remove = mmp_pdma_remove, }; -bool mmp_pdma_filter_fn(struct dma_chan *chan, void *param) -{ - struct mmp_pdma_chan *c = to_mmp_pdma_chan(chan); - - if (chan->device->dev->driver != &mmp_pdma_driver.driver) - return false; - - c->drcmr = *(unsigned int *)param; - - return true; -} -EXPORT_SYMBOL_GPL(mmp_pdma_filter_fn); - module_platform_driver(mmp_pdma_driver); MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver"); diff --git a/drivers/dma/owl-dma.c b/drivers/dma/owl-dma.c index 9fede32641e9..1f0bbaed4643 100644 --- a/drivers/dma/owl-dma.c +++ b/drivers/dma/owl-dma.c @@ -1080,8 +1080,9 @@ static struct dma_chan *owl_dma_of_xlate(struct of_phandle_args *dma_spec, } static const struct of_device_id owl_dma_match[] = { - { .compatible = "actions,s900-dma", .data = (void *)S900_DMA,}, + { .compatible = "actions,s500-dma", .data = (void *)S900_DMA,}, { .compatible = "actions,s700-dma", .data = (void *)S700_DMA,}, + { .compatible = "actions,s900-dma", .data = (void *)S900_DMA,}, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, owl_dma_match); @@ -1245,6 +1246,7 @@ static int owl_dma_remove(struct platform_device *pdev) owl_dma_free(od); clk_disable_unprepare(od->clk); + dma_pool_destroy(od->lli_pool); return 0; } diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 88579857ca1d..c8a77b428b52 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -1270,13 +1270,13 @@ static int bam_dma_probe(struct platform_device *pdev) dev_err(bdev->dev, "num-ees unspecified in dt\n"); } - bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk"); - if (IS_ERR(bdev->bamclk)) { - if (!bdev->controlled_remotely) - return PTR_ERR(bdev->bamclk); + if (bdev->controlled_remotely) + bdev->bamclk = devm_clk_get_optional(bdev->dev, "bam_clk"); + else + bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk"); - bdev->bamclk = NULL; - } + if (IS_ERR(bdev->bamclk)) + return PTR_ERR(bdev->bamclk); ret = clk_prepare_enable(bdev->bamclk); if (ret) { @@ -1350,7 +1350,7 @@ static int bam_dma_probe(struct platform_device *pdev) if (ret) goto err_unregister_dma; - if (bdev->controlled_remotely) { + if (!bdev->bamclk) { pm_runtime_disable(&pdev->dev); return 0; } @@ -1438,10 +1438,10 @@ static int __maybe_unused bam_dma_suspend(struct device *dev) { struct bam_device *bdev = dev_get_drvdata(dev); - if (!bdev->controlled_remotely) + if (bdev->bamclk) { pm_runtime_force_suspend(dev); - - clk_unprepare(bdev->bamclk); + clk_unprepare(bdev->bamclk); + } return 0; } @@ -1451,12 +1451,13 @@ static int __maybe_unused bam_dma_resume(struct device *dev) struct bam_device *bdev = dev_get_drvdata(dev); int ret; - ret = clk_prepare(bdev->bamclk); - if (ret) - return ret; + if (bdev->bamclk) { + ret = clk_prepare(bdev->bamclk); + if (ret) + return ret; - if (!bdev->controlled_remotely) pm_runtime_force_resume(dev); + } return 0; } diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c index 1a0bf6b0567a..57f5ee4235c7 100644 --- a/drivers/dma/qcom/gpi.c +++ b/drivers/dma/qcom/gpi.c @@ -584,7 +584,7 @@ static inline void gpi_write_reg_field(struct gpii *gpii, void __iomem *addr, gpi_write_reg(gpii, addr, val); } -static inline void +static __always_inline void gpi_update_reg(struct gpii *gpii, u32 offset, u32 mask, u32 val) { void __iomem *addr = gpii->regs + offset; @@ -1700,7 +1700,7 @@ static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc, tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT); - }; + } for (i = 0; i < tre_idx; i++) dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0], diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index a57705356e8b..d530c1bf11d9 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -189,7 +189,8 @@ struct rcar_dmac_chan { * struct rcar_dmac - R-Car Gen2 DMA Controller * @engine: base DMA engine object * @dev: the hardware device - * @iomem: remapped I/O memory base + * @dmac_base: remapped base register block + * @chan_base: remapped channel register block (optional) * @n_channels: number of available channels * @channels: array of DMAC channels * @channels_mask: bitfield of which DMA channels are managed by this driver @@ -198,7 +199,8 @@ struct rcar_dmac_chan { struct rcar_dmac { struct dma_device engine; struct device *dev; - void __iomem *iomem; + void __iomem *dmac_base; + void __iomem *chan_base; unsigned int n_channels; struct rcar_dmac_chan *channels; @@ -209,6 +211,10 @@ struct rcar_dmac { #define to_rcar_dmac(d) container_of(d, struct rcar_dmac, engine) +#define for_each_rcar_dmac_chan(i, dmac, chan) \ + for (i = 0, chan = &(dmac)->channels[0]; i < (dmac)->n_channels; i++, chan++) \ + if (!((dmac)->channels_mask & BIT(i))) continue; else + /* * struct rcar_dmac_of_data - This driver's OF data * @chan_offset_base: DMAC channels base offset @@ -230,7 +236,7 @@ struct rcar_dmac_of_data { #define RCAR_DMAOR_PRI_ROUND_ROBIN (3 << 8) #define RCAR_DMAOR_AE (1 << 2) #define RCAR_DMAOR_DME (1 << 0) -#define RCAR_DMACHCLR 0x0080 +#define RCAR_DMACHCLR 0x0080 /* Not on R-Car V3U */ #define RCAR_DMADPSEC 0x00a0 #define RCAR_DMASAR 0x0000 @@ -293,6 +299,9 @@ struct rcar_dmac_of_data { #define RCAR_DMAFIXDAR 0x0014 #define RCAR_DMAFIXDPBASE 0x0060 +/* For R-Car V3U */ +#define RCAR_V3U_DMACHCLR 0x0100 + /* Hardcode the MEMCPY transfer size to 4 bytes. */ #define RCAR_DMAC_MEMCPY_XFER_SIZE 4 @@ -303,17 +312,17 @@ struct rcar_dmac_of_data { static void rcar_dmac_write(struct rcar_dmac *dmac, u32 reg, u32 data) { if (reg == RCAR_DMAOR) - writew(data, dmac->iomem + reg); + writew(data, dmac->dmac_base + reg); else - writel(data, dmac->iomem + reg); + writel(data, dmac->dmac_base + reg); } static u32 rcar_dmac_read(struct rcar_dmac *dmac, u32 reg) { if (reg == RCAR_DMAOR) - return readw(dmac->iomem + reg); + return readw(dmac->dmac_base + reg); else - return readl(dmac->iomem + reg); + return readl(dmac->dmac_base + reg); } static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg) @@ -332,6 +341,28 @@ static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data) writel(data, chan->iomem + reg); } +static void rcar_dmac_chan_clear(struct rcar_dmac *dmac, + struct rcar_dmac_chan *chan) +{ + if (dmac->chan_base) + rcar_dmac_chan_write(chan, RCAR_V3U_DMACHCLR, 1); + else + rcar_dmac_write(dmac, RCAR_DMACHCLR, BIT(chan->index)); +} + +static void rcar_dmac_chan_clear_all(struct rcar_dmac *dmac) +{ + struct rcar_dmac_chan *chan; + unsigned int i; + + if (dmac->chan_base) { + for_each_rcar_dmac_chan(i, dmac, chan) + rcar_dmac_chan_write(chan, RCAR_V3U_DMACHCLR, 1); + } else { + rcar_dmac_write(dmac, RCAR_DMACHCLR, dmac->channels_mask); + } +} + /* ----------------------------------------------------------------------------- * Initialization and configuration */ @@ -447,7 +478,7 @@ static int rcar_dmac_init(struct rcar_dmac *dmac) u16 dmaor; /* Clear all channels and enable the DMAC globally. */ - rcar_dmac_write(dmac, RCAR_DMACHCLR, dmac->channels_mask); + rcar_dmac_chan_clear_all(dmac); rcar_dmac_write(dmac, RCAR_DMAOR, RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME); @@ -817,15 +848,11 @@ static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan) static void rcar_dmac_stop_all_chan(struct rcar_dmac *dmac) { + struct rcar_dmac_chan *chan; unsigned int i; /* Stop all channels. */ - for (i = 0; i < dmac->n_channels; ++i) { - struct rcar_dmac_chan *chan = &dmac->channels[i]; - - if (!(dmac->channels_mask & BIT(i))) - continue; - + for_each_rcar_dmac_chan(i, dmac, chan) { /* Stop and reinitialize the channel. */ spin_lock_irq(&chan->lock); rcar_dmac_chan_halt(chan); @@ -1566,7 +1593,7 @@ static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev) * because channel is already stopped in error case. * We need to clear register and check DE bit as recovery. */ - rcar_dmac_write(dmac, RCAR_DMACHCLR, 1 << chan->index); + rcar_dmac_chan_clear(dmac, chan); rcar_dmac_chcr_de_barrier(chan); reinit = true; goto spin_lock_end; @@ -1732,9 +1759,7 @@ static const struct dev_pm_ops rcar_dmac_pm = { */ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, - struct rcar_dmac_chan *rchan, - const struct rcar_dmac_of_data *data, - unsigned int index) + struct rcar_dmac_chan *rchan) { struct platform_device *pdev = to_platform_device(dmac->dev); struct dma_chan *chan = &rchan->chan; @@ -1742,9 +1767,6 @@ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, char *irqname; int ret; - rchan->index = index; - rchan->iomem = dmac->iomem + data->chan_offset_base + - data->chan_offset_stride * index; rchan->mid_rid = -EINVAL; spin_lock_init(&rchan->lock); @@ -1756,13 +1778,13 @@ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, INIT_LIST_HEAD(&rchan->desc.wait); /* Request the channel interrupt. */ - sprintf(pdev_irqname, "ch%u", index); + sprintf(pdev_irqname, "ch%u", rchan->index); rchan->irq = platform_get_irq_byname(pdev, pdev_irqname); if (rchan->irq < 0) return -ENODEV; irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", - dev_name(dmac->dev), index); + dev_name(dmac->dev), rchan->index); if (!irqname) return -ENOMEM; @@ -1828,9 +1850,11 @@ static int rcar_dmac_probe(struct platform_device *pdev) DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES | DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES | DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES; + const struct rcar_dmac_of_data *data; + struct rcar_dmac_chan *chan; struct dma_device *engine; + void __iomem *chan_base; struct rcar_dmac *dmac; - const struct rcar_dmac_of_data *data; unsigned int i; int ret; @@ -1868,9 +1892,24 @@ static int rcar_dmac_probe(struct platform_device *pdev) return -ENOMEM; /* Request resources. */ - dmac->iomem = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(dmac->iomem)) - return PTR_ERR(dmac->iomem); + dmac->dmac_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dmac->dmac_base)) + return PTR_ERR(dmac->dmac_base); + + if (!data->chan_offset_base) { + dmac->chan_base = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(dmac->chan_base)) + return PTR_ERR(dmac->chan_base); + + chan_base = dmac->chan_base; + } else { + chan_base = dmac->dmac_base + data->chan_offset_base; + } + + for_each_rcar_dmac_chan(i, dmac, chan) { + chan->index = i; + chan->iomem = chan_base + i * data->chan_offset_stride; + } /* Enable runtime PM and initialize the device. */ pm_runtime_enable(&pdev->dev); @@ -1916,11 +1955,8 @@ static int rcar_dmac_probe(struct platform_device *pdev) INIT_LIST_HEAD(&engine->channels); - for (i = 0; i < dmac->n_channels; ++i) { - if (!(dmac->channels_mask & BIT(i))) - continue; - - ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i], data, i); + for_each_rcar_dmac_chan(i, dmac, chan) { + ret = rcar_dmac_chan_probe(dmac, chan); if (ret < 0) goto error; } @@ -1968,14 +2004,22 @@ static void rcar_dmac_shutdown(struct platform_device *pdev) } static const struct rcar_dmac_of_data rcar_dmac_data = { - .chan_offset_base = 0x8000, - .chan_offset_stride = 0x80, + .chan_offset_base = 0x8000, + .chan_offset_stride = 0x80, +}; + +static const struct rcar_dmac_of_data rcar_v3u_dmac_data = { + .chan_offset_base = 0x0, + .chan_offset_stride = 0x1000, }; static const struct of_device_id rcar_dmac_of_ids[] = { { .compatible = "renesas,rcar-dmac", .data = &rcar_dmac_data, + }, { + .compatible = "renesas,dmac-r8a779a0", + .data = &rcar_v3u_dmac_data, }, { /* Sentinel */ } }; diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c deleted file mode 100644 index a5c2843384fd..000000000000 --- a/drivers/dma/sirf-dma.c +++ /dev/null @@ -1,1170 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * DMA controller driver for CSR SiRFprimaII - * - * Copyright (c) 2011 Cambridge Silicon Radio Limited, a CSR plc group company. - */ - -#include <linux/module.h> -#include <linux/dmaengine.h> -#include <linux/dma-mapping.h> -#include <linux/pm_runtime.h> -#include <linux/interrupt.h> -#include <linux/io.h> -#include <linux/slab.h> -#include <linux/of_irq.h> -#include <linux/of_address.h> -#include <linux/of_device.h> -#include <linux/of_platform.h> -#include <linux/clk.h> -#include <linux/of_dma.h> -#include <linux/sirfsoc_dma.h> - -#include "dmaengine.h" - -#define SIRFSOC_DMA_VER_A7V1 1 -#define SIRFSOC_DMA_VER_A7V2 2 -#define SIRFSOC_DMA_VER_A6 4 - -#define SIRFSOC_DMA_DESCRIPTORS 16 -#define SIRFSOC_DMA_CHANNELS 16 -#define SIRFSOC_DMA_TABLE_NUM 256 - -#define SIRFSOC_DMA_CH_ADDR 0x00 -#define SIRFSOC_DMA_CH_XLEN 0x04 -#define SIRFSOC_DMA_CH_YLEN 0x08 -#define SIRFSOC_DMA_CH_CTRL 0x0C - -#define SIRFSOC_DMA_WIDTH_0 0x100 -#define SIRFSOC_DMA_CH_VALID 0x140 -#define SIRFSOC_DMA_CH_INT 0x144 -#define SIRFSOC_DMA_INT_EN 0x148 -#define SIRFSOC_DMA_INT_EN_CLR 0x14C -#define SIRFSOC_DMA_CH_LOOP_CTRL 0x150 -#define SIRFSOC_DMA_CH_LOOP_CTRL_CLR 0x154 -#define SIRFSOC_DMA_WIDTH_ATLAS7 0x10 -#define SIRFSOC_DMA_VALID_ATLAS7 0x14 -#define SIRFSOC_DMA_INT_ATLAS7 0x18 -#define SIRFSOC_DMA_INT_EN_ATLAS7 0x1c -#define SIRFSOC_DMA_LOOP_CTRL_ATLAS7 0x20 -#define SIRFSOC_DMA_CUR_DATA_ADDR 0x34 -#define SIRFSOC_DMA_MUL_ATLAS7 0x38 -#define SIRFSOC_DMA_CH_LOOP_CTRL_ATLAS7 0x158 -#define SIRFSOC_DMA_CH_LOOP_CTRL_CLR_ATLAS7 0x15C -#define SIRFSOC_DMA_IOBG_SCMD_EN 0x800 -#define SIRFSOC_DMA_EARLY_RESP_SET 0x818 -#define SIRFSOC_DMA_EARLY_RESP_CLR 0x81C - -#define SIRFSOC_DMA_MODE_CTRL_BIT 4 -#define SIRFSOC_DMA_DIR_CTRL_BIT 5 -#define SIRFSOC_DMA_MODE_CTRL_BIT_ATLAS7 2 -#define SIRFSOC_DMA_CHAIN_CTRL_BIT_ATLAS7 3 -#define SIRFSOC_DMA_DIR_CTRL_BIT_ATLAS7 4 -#define SIRFSOC_DMA_TAB_NUM_ATLAS7 7 -#define SIRFSOC_DMA_CHAIN_INT_BIT_ATLAS7 5 -#define SIRFSOC_DMA_CHAIN_FLAG_SHIFT_ATLAS7 25 -#define SIRFSOC_DMA_CHAIN_ADDR_SHIFT 32 - -#define SIRFSOC_DMA_INT_FINI_INT_ATLAS7 BIT(0) -#define SIRFSOC_DMA_INT_CNT_INT_ATLAS7 BIT(1) -#define SIRFSOC_DMA_INT_PAU_INT_ATLAS7 BIT(2) -#define SIRFSOC_DMA_INT_LOOP_INT_ATLAS7 BIT(3) -#define SIRFSOC_DMA_INT_INV_INT_ATLAS7 BIT(4) -#define SIRFSOC_DMA_INT_END_INT_ATLAS7 BIT(5) -#define SIRFSOC_DMA_INT_ALL_ATLAS7 0x3F - -/* xlen and dma_width register is in 4 bytes boundary */ -#define SIRFSOC_DMA_WORD_LEN 4 -#define SIRFSOC_DMA_XLEN_MAX_V1 0x800 -#define SIRFSOC_DMA_XLEN_MAX_V2 0x1000 - -struct sirfsoc_dma_desc { - struct dma_async_tx_descriptor desc; - struct list_head node; - - /* SiRFprimaII 2D-DMA parameters */ - - int xlen; /* DMA xlen */ - int ylen; /* DMA ylen */ - int width; /* DMA width */ - int dir; - bool cyclic; /* is loop DMA? */ - bool chain; /* is chain DMA? */ - u32 addr; /* DMA buffer address */ - u64 chain_table[SIRFSOC_DMA_TABLE_NUM]; /* chain tbl */ -}; - -struct sirfsoc_dma_chan { - struct dma_chan chan; - struct list_head free; - struct list_head prepared; - struct list_head queued; - struct list_head active; - struct list_head completed; - unsigned long happened_cyclic; - unsigned long completed_cyclic; - - /* Lock for this structure */ - spinlock_t lock; - - int mode; -}; - -struct sirfsoc_dma_regs { - u32 ctrl[SIRFSOC_DMA_CHANNELS]; - u32 interrupt_en; -}; - -struct sirfsoc_dma { - struct dma_device dma; - struct tasklet_struct tasklet; - struct sirfsoc_dma_chan channels[SIRFSOC_DMA_CHANNELS]; - void __iomem *base; - int irq; - struct clk *clk; - int type; - void (*exec_desc)(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base); - struct sirfsoc_dma_regs regs_save; -}; - -struct sirfsoc_dmadata { - void (*exec)(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base); - int type; -}; - -enum sirfsoc_dma_chain_flag { - SIRFSOC_DMA_CHAIN_NORMAL = 0x01, - SIRFSOC_DMA_CHAIN_PAUSE = 0x02, - SIRFSOC_DMA_CHAIN_LOOP = 0x03, - SIRFSOC_DMA_CHAIN_END = 0x04 -}; - -#define DRV_NAME "sirfsoc_dma" - -static int sirfsoc_dma_runtime_suspend(struct device *dev); - -/* Convert struct dma_chan to struct sirfsoc_dma_chan */ -static inline -struct sirfsoc_dma_chan *dma_chan_to_sirfsoc_dma_chan(struct dma_chan *c) -{ - return container_of(c, struct sirfsoc_dma_chan, chan); -} - -/* Convert struct dma_chan to struct sirfsoc_dma */ -static inline struct sirfsoc_dma *dma_chan_to_sirfsoc_dma(struct dma_chan *c) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(c); - return container_of(schan, struct sirfsoc_dma, channels[c->chan_id]); -} - -static void sirfsoc_dma_execute_hw_a7v2(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base) -{ - if (sdesc->chain) { - /* DMA v2 HW chain mode */ - writel_relaxed((sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT_ATLAS7) | - (sdesc->chain << - SIRFSOC_DMA_CHAIN_CTRL_BIT_ATLAS7) | - (0x8 << SIRFSOC_DMA_TAB_NUM_ATLAS7) | 0x3, - base + SIRFSOC_DMA_CH_CTRL); - } else { - /* DMA v2 legacy mode */ - writel_relaxed(sdesc->xlen, base + SIRFSOC_DMA_CH_XLEN); - writel_relaxed(sdesc->ylen, base + SIRFSOC_DMA_CH_YLEN); - writel_relaxed(sdesc->width, base + SIRFSOC_DMA_WIDTH_ATLAS7); - writel_relaxed((sdesc->width*((sdesc->ylen+1)>>1)), - base + SIRFSOC_DMA_MUL_ATLAS7); - writel_relaxed((sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT_ATLAS7) | - (sdesc->chain << - SIRFSOC_DMA_CHAIN_CTRL_BIT_ATLAS7) | - 0x3, base + SIRFSOC_DMA_CH_CTRL); - } - writel_relaxed(sdesc->chain ? SIRFSOC_DMA_INT_END_INT_ATLAS7 : - (SIRFSOC_DMA_INT_FINI_INT_ATLAS7 | - SIRFSOC_DMA_INT_LOOP_INT_ATLAS7), - base + SIRFSOC_DMA_INT_EN_ATLAS7); - writel(sdesc->addr, base + SIRFSOC_DMA_CH_ADDR); - if (sdesc->cyclic) - writel(0x10001, base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); -} - -static void sirfsoc_dma_execute_hw_a7v1(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base) -{ - writel_relaxed(1, base + SIRFSOC_DMA_IOBG_SCMD_EN); - writel_relaxed((1 << cid), base + SIRFSOC_DMA_EARLY_RESP_SET); - writel_relaxed(sdesc->width, base + SIRFSOC_DMA_WIDTH_0 + cid * 4); - writel_relaxed(cid | (burst_mode << SIRFSOC_DMA_MODE_CTRL_BIT) | - (sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT), - base + cid * 0x10 + SIRFSOC_DMA_CH_CTRL); - writel_relaxed(sdesc->xlen, base + cid * 0x10 + SIRFSOC_DMA_CH_XLEN); - writel_relaxed(sdesc->ylen, base + cid * 0x10 + SIRFSOC_DMA_CH_YLEN); - writel_relaxed(readl_relaxed(base + SIRFSOC_DMA_INT_EN) | - (1 << cid), base + SIRFSOC_DMA_INT_EN); - writel(sdesc->addr >> 2, base + cid * 0x10 + SIRFSOC_DMA_CH_ADDR); - if (sdesc->cyclic) { - writel((1 << cid) | 1 << (cid + 16) | - readl_relaxed(base + SIRFSOC_DMA_CH_LOOP_CTRL_ATLAS7), - base + SIRFSOC_DMA_CH_LOOP_CTRL_ATLAS7); - } - -} - -static void sirfsoc_dma_execute_hw_a6(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base) -{ - writel_relaxed(sdesc->width, base + SIRFSOC_DMA_WIDTH_0 + cid * 4); - writel_relaxed(cid | (burst_mode << SIRFSOC_DMA_MODE_CTRL_BIT) | - (sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT), - base + cid * 0x10 + SIRFSOC_DMA_CH_CTRL); - writel_relaxed(sdesc->xlen, base + cid * 0x10 + SIRFSOC_DMA_CH_XLEN); - writel_relaxed(sdesc->ylen, base + cid * 0x10 + SIRFSOC_DMA_CH_YLEN); - writel_relaxed(readl_relaxed(base + SIRFSOC_DMA_INT_EN) | - (1 << cid), base + SIRFSOC_DMA_INT_EN); - writel(sdesc->addr >> 2, base + cid * 0x10 + SIRFSOC_DMA_CH_ADDR); - if (sdesc->cyclic) { - writel((1 << cid) | 1 << (cid + 16) | - readl_relaxed(base + SIRFSOC_DMA_CH_LOOP_CTRL), - base + SIRFSOC_DMA_CH_LOOP_CTRL); - } - -} - -/* Execute all queued DMA descriptors */ -static void sirfsoc_dma_execute(struct sirfsoc_dma_chan *schan) -{ - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan); - int cid = schan->chan.chan_id; - struct sirfsoc_dma_desc *sdesc = NULL; - void __iomem *base; - - /* - * lock has been held by functions calling this, so we don't hold - * lock again - */ - base = sdma->base; - sdesc = list_first_entry(&schan->queued, struct sirfsoc_dma_desc, - node); - /* Move the first queued descriptor to active list */ - list_move_tail(&sdesc->node, &schan->active); - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) - cid = 0; - - /* Start the DMA transfer */ - sdma->exec_desc(sdesc, cid, schan->mode, base); - - if (sdesc->cyclic) - schan->happened_cyclic = schan->completed_cyclic = 0; -} - -/* Interrupt handler */ -static irqreturn_t sirfsoc_dma_irq(int irq, void *data) -{ - struct sirfsoc_dma *sdma = data; - struct sirfsoc_dma_chan *schan; - struct sirfsoc_dma_desc *sdesc = NULL; - u32 is; - bool chain; - int ch; - void __iomem *reg; - - switch (sdma->type) { - case SIRFSOC_DMA_VER_A6: - case SIRFSOC_DMA_VER_A7V1: - is = readl(sdma->base + SIRFSOC_DMA_CH_INT); - reg = sdma->base + SIRFSOC_DMA_CH_INT; - while ((ch = fls(is) - 1) >= 0) { - is &= ~(1 << ch); - writel_relaxed(1 << ch, reg); - schan = &sdma->channels[ch]; - spin_lock(&schan->lock); - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, node); - if (!sdesc->cyclic) { - /* Execute queued descriptors */ - list_splice_tail_init(&schan->active, - &schan->completed); - dma_cookie_complete(&sdesc->desc); - if (!list_empty(&schan->queued)) - sirfsoc_dma_execute(schan); - } else - schan->happened_cyclic++; - spin_unlock(&schan->lock); - } - break; - - case SIRFSOC_DMA_VER_A7V2: - is = readl(sdma->base + SIRFSOC_DMA_INT_ATLAS7); - - reg = sdma->base + SIRFSOC_DMA_INT_ATLAS7; - writel_relaxed(SIRFSOC_DMA_INT_ALL_ATLAS7, reg); - schan = &sdma->channels[0]; - spin_lock(&schan->lock); - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, node); - if (!sdesc->cyclic) { - chain = sdesc->chain; - if ((chain && (is & SIRFSOC_DMA_INT_END_INT_ATLAS7)) || - (!chain && - (is & SIRFSOC_DMA_INT_FINI_INT_ATLAS7))) { - /* Execute queued descriptors */ - list_splice_tail_init(&schan->active, - &schan->completed); - dma_cookie_complete(&sdesc->desc); - if (!list_empty(&schan->queued)) - sirfsoc_dma_execute(schan); - } - } else if (sdesc->cyclic && (is & - SIRFSOC_DMA_INT_LOOP_INT_ATLAS7)) - schan->happened_cyclic++; - - spin_unlock(&schan->lock); - break; - - default: - break; - } - - /* Schedule tasklet */ - tasklet_schedule(&sdma->tasklet); - - return IRQ_HANDLED; -} - -/* process completed descriptors */ -static void sirfsoc_dma_process_completed(struct sirfsoc_dma *sdma) -{ - dma_cookie_t last_cookie = 0; - struct sirfsoc_dma_chan *schan; - struct sirfsoc_dma_desc *sdesc; - struct dma_async_tx_descriptor *desc; - unsigned long flags; - unsigned long happened_cyclic; - LIST_HEAD(list); - int i; - - for (i = 0; i < sdma->dma.chancnt; i++) { - schan = &sdma->channels[i]; - - /* Get all completed descriptors */ - spin_lock_irqsave(&schan->lock, flags); - if (!list_empty(&schan->completed)) { - list_splice_tail_init(&schan->completed, &list); - spin_unlock_irqrestore(&schan->lock, flags); - - /* Execute callbacks and run dependencies */ - list_for_each_entry(sdesc, &list, node) { - desc = &sdesc->desc; - - dmaengine_desc_get_callback_invoke(desc, NULL); - last_cookie = desc->cookie; - dma_run_dependencies(desc); - } - - /* Free descriptors */ - spin_lock_irqsave(&schan->lock, flags); - list_splice_tail_init(&list, &schan->free); - schan->chan.completed_cookie = last_cookie; - spin_unlock_irqrestore(&schan->lock, flags); - } else { - if (list_empty(&schan->active)) { - spin_unlock_irqrestore(&schan->lock, flags); - continue; - } - - /* for cyclic channel, desc is always in active list */ - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, node); - - /* cyclic DMA */ - happened_cyclic = schan->happened_cyclic; - spin_unlock_irqrestore(&schan->lock, flags); - - desc = &sdesc->desc; - while (happened_cyclic != schan->completed_cyclic) { - dmaengine_desc_get_callback_invoke(desc, NULL); - schan->completed_cyclic++; - } - } - } -} - -/* DMA Tasklet */ -static void sirfsoc_dma_tasklet(struct tasklet_struct *t) -{ - struct sirfsoc_dma *sdma = from_tasklet(sdma, t, tasklet); - - sirfsoc_dma_process_completed(sdma); -} - -/* Submit descriptor to hardware */ -static dma_cookie_t sirfsoc_dma_tx_submit(struct dma_async_tx_descriptor *txd) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(txd->chan); - struct sirfsoc_dma_desc *sdesc; - unsigned long flags; - dma_cookie_t cookie; - - sdesc = container_of(txd, struct sirfsoc_dma_desc, desc); - - spin_lock_irqsave(&schan->lock, flags); - - /* Move descriptor to queue */ - list_move_tail(&sdesc->node, &schan->queued); - - cookie = dma_cookie_assign(txd); - - spin_unlock_irqrestore(&schan->lock, flags); - - return cookie; -} - -static int sirfsoc_dma_slave_config(struct dma_chan *chan, - struct dma_slave_config *config) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - unsigned long flags; - - if ((config->src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) || - (config->dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)) - return -EINVAL; - - spin_lock_irqsave(&schan->lock, flags); - schan->mode = (config->src_maxburst == 4 ? 1 : 0); - spin_unlock_irqrestore(&schan->lock, flags); - - return 0; -} - -static int sirfsoc_dma_terminate_all(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan); - int cid = schan->chan.chan_id; - unsigned long flags; - - spin_lock_irqsave(&schan->lock, flags); - - switch (sdma->type) { - case SIRFSOC_DMA_VER_A7V1: - writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_INT_EN_CLR); - writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_CH_INT); - writel_relaxed((1 << cid) | 1 << (cid + 16), - sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL_CLR_ATLAS7); - writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_CH_VALID); - break; - case SIRFSOC_DMA_VER_A7V2: - writel_relaxed(0, sdma->base + SIRFSOC_DMA_INT_EN_ATLAS7); - writel_relaxed(SIRFSOC_DMA_INT_ALL_ATLAS7, - sdma->base + SIRFSOC_DMA_INT_ATLAS7); - writel_relaxed(0, sdma->base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); - writel_relaxed(0, sdma->base + SIRFSOC_DMA_VALID_ATLAS7); - break; - case SIRFSOC_DMA_VER_A6: - writel_relaxed(readl_relaxed(sdma->base + SIRFSOC_DMA_INT_EN) & - ~(1 << cid), sdma->base + SIRFSOC_DMA_INT_EN); - writel_relaxed(readl_relaxed(sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL) & - ~((1 << cid) | 1 << (cid + 16)), - sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL); - writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_CH_VALID); - break; - default: - break; - } - - list_splice_tail_init(&schan->active, &schan->free); - list_splice_tail_init(&schan->queued, &schan->free); - - spin_unlock_irqrestore(&schan->lock, flags); - - return 0; -} - -static int sirfsoc_dma_pause_chan(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan); - int cid = schan->chan.chan_id; - unsigned long flags; - - spin_lock_irqsave(&schan->lock, flags); - - switch (sdma->type) { - case SIRFSOC_DMA_VER_A7V1: - writel_relaxed((1 << cid) | 1 << (cid + 16), - sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL_CLR_ATLAS7); - break; - case SIRFSOC_DMA_VER_A7V2: - writel_relaxed(0, sdma->base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); - break; - case SIRFSOC_DMA_VER_A6: - writel_relaxed(readl_relaxed(sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL) & - ~((1 << cid) | 1 << (cid + 16)), - sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL); - break; - - default: - break; - } - - spin_unlock_irqrestore(&schan->lock, flags); - - return 0; -} - -static int sirfsoc_dma_resume_chan(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan); - int cid = schan->chan.chan_id; - unsigned long flags; - - spin_lock_irqsave(&schan->lock, flags); - switch (sdma->type) { - case SIRFSOC_DMA_VER_A7V1: - writel_relaxed((1 << cid) | 1 << (cid + 16), - sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL_ATLAS7); - break; - case SIRFSOC_DMA_VER_A7V2: - writel_relaxed(0x10001, - sdma->base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); - break; - case SIRFSOC_DMA_VER_A6: - writel_relaxed(readl_relaxed(sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL) | - ((1 << cid) | 1 << (cid + 16)), - sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL); - break; - - default: - break; - } - - spin_unlock_irqrestore(&schan->lock, flags); - - return 0; -} - -/* Alloc channel resources */ -static int sirfsoc_dma_alloc_chan_resources(struct dma_chan *chan) -{ - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan); - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma_desc *sdesc; - unsigned long flags; - LIST_HEAD(descs); - int i; - - pm_runtime_get_sync(sdma->dma.dev); - - /* Alloc descriptors for this channel */ - for (i = 0; i < SIRFSOC_DMA_DESCRIPTORS; i++) { - sdesc = kzalloc(sizeof(*sdesc), GFP_KERNEL); - if (!sdesc) { - dev_notice(sdma->dma.dev, "Memory allocation error. " - "Allocated only %u descriptors\n", i); - break; - } - - dma_async_tx_descriptor_init(&sdesc->desc, chan); - sdesc->desc.flags = DMA_CTRL_ACK; - sdesc->desc.tx_submit = sirfsoc_dma_tx_submit; - - list_add_tail(&sdesc->node, &descs); - } - - /* Return error only if no descriptors were allocated */ - if (i == 0) - return -ENOMEM; - - spin_lock_irqsave(&schan->lock, flags); - - list_splice_tail_init(&descs, &schan->free); - spin_unlock_irqrestore(&schan->lock, flags); - - return i; -} - -/* Free channel resources */ -static void sirfsoc_dma_free_chan_resources(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan); - struct sirfsoc_dma_desc *sdesc, *tmp; - unsigned long flags; - LIST_HEAD(descs); - - spin_lock_irqsave(&schan->lock, flags); - - /* Channel must be idle */ - BUG_ON(!list_empty(&schan->prepared)); - BUG_ON(!list_empty(&schan->queued)); - BUG_ON(!list_empty(&schan->active)); - BUG_ON(!list_empty(&schan->completed)); - - /* Move data */ - list_splice_tail_init(&schan->free, &descs); - - spin_unlock_irqrestore(&schan->lock, flags); - - /* Free descriptors */ - list_for_each_entry_safe(sdesc, tmp, &descs, node) - kfree(sdesc); - - pm_runtime_put(sdma->dma.dev); -} - -/* Send pending descriptor to hardware */ -static void sirfsoc_dma_issue_pending(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&schan->lock, flags); - - if (list_empty(&schan->active) && !list_empty(&schan->queued)) - sirfsoc_dma_execute(schan); - - spin_unlock_irqrestore(&schan->lock, flags); -} - -/* Check request completion status */ -static enum dma_status -sirfsoc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan); - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - unsigned long flags; - enum dma_status ret; - struct sirfsoc_dma_desc *sdesc; - int cid = schan->chan.chan_id; - unsigned long dma_pos; - unsigned long dma_request_bytes; - unsigned long residue; - - spin_lock_irqsave(&schan->lock, flags); - - if (list_empty(&schan->active)) { - ret = dma_cookie_status(chan, cookie, txstate); - dma_set_residue(txstate, 0); - spin_unlock_irqrestore(&schan->lock, flags); - return ret; - } - sdesc = list_first_entry(&schan->active, struct sirfsoc_dma_desc, node); - if (sdesc->cyclic) - dma_request_bytes = (sdesc->xlen + 1) * (sdesc->ylen + 1) * - (sdesc->width * SIRFSOC_DMA_WORD_LEN); - else - dma_request_bytes = sdesc->xlen * SIRFSOC_DMA_WORD_LEN; - - ret = dma_cookie_status(chan, cookie, txstate); - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) - cid = 0; - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) { - dma_pos = readl_relaxed(sdma->base + SIRFSOC_DMA_CUR_DATA_ADDR); - } else { - dma_pos = readl_relaxed( - sdma->base + cid * 0x10 + SIRFSOC_DMA_CH_ADDR) << 2; - } - - residue = dma_request_bytes - (dma_pos - sdesc->addr); - dma_set_residue(txstate, residue); - - spin_unlock_irqrestore(&schan->lock, flags); - - return ret; -} - -static struct dma_async_tx_descriptor *sirfsoc_dma_prep_interleaved( - struct dma_chan *chan, struct dma_interleaved_template *xt, - unsigned long flags) -{ - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan); - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma_desc *sdesc = NULL; - unsigned long iflags; - int ret; - - if ((xt->dir != DMA_MEM_TO_DEV) && (xt->dir != DMA_DEV_TO_MEM)) { - ret = -EINVAL; - goto err_dir; - } - - /* Get free descriptor */ - spin_lock_irqsave(&schan->lock, iflags); - if (!list_empty(&schan->free)) { - sdesc = list_first_entry(&schan->free, struct sirfsoc_dma_desc, - node); - list_del(&sdesc->node); - } - spin_unlock_irqrestore(&schan->lock, iflags); - - if (!sdesc) { - /* try to free completed descriptors */ - sirfsoc_dma_process_completed(sdma); - ret = 0; - goto no_desc; - } - - /* Place descriptor in prepared list */ - spin_lock_irqsave(&schan->lock, iflags); - - /* - * Number of chunks in a frame can only be 1 for prima2 - * and ylen (number of frame - 1) must be at least 0 - */ - if ((xt->frame_size == 1) && (xt->numf > 0)) { - sdesc->cyclic = 0; - sdesc->xlen = xt->sgl[0].size / SIRFSOC_DMA_WORD_LEN; - sdesc->width = (xt->sgl[0].size + xt->sgl[0].icg) / - SIRFSOC_DMA_WORD_LEN; - sdesc->ylen = xt->numf - 1; - if (xt->dir == DMA_MEM_TO_DEV) { - sdesc->addr = xt->src_start; - sdesc->dir = 1; - } else { - sdesc->addr = xt->dst_start; - sdesc->dir = 0; - } - - list_add_tail(&sdesc->node, &schan->prepared); - } else { - pr_err("sirfsoc DMA Invalid xfer\n"); - ret = -EINVAL; - goto err_xfer; - } - spin_unlock_irqrestore(&schan->lock, iflags); - - return &sdesc->desc; -err_xfer: - spin_unlock_irqrestore(&schan->lock, iflags); -no_desc: -err_dir: - return ERR_PTR(ret); -} - -static struct dma_async_tx_descriptor * -sirfsoc_dma_prep_cyclic(struct dma_chan *chan, dma_addr_t addr, - size_t buf_len, size_t period_len, - enum dma_transfer_direction direction, unsigned long flags) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma_desc *sdesc = NULL; - unsigned long iflags; - - /* - * we only support cycle transfer with 2 period - * If the X-length is set to 0, it would be the loop mode. - * The DMA address keeps increasing until reaching the end of a loop - * area whose size is defined by (DMA_WIDTH x (Y_LENGTH + 1)). Then - * the DMA address goes back to the beginning of this area. - * In loop mode, the DMA data region is divided into two parts, BUFA - * and BUFB. DMA controller generates interrupts twice in each loop: - * when the DMA address reaches the end of BUFA or the end of the - * BUFB - */ - if (buf_len != 2 * period_len) - return ERR_PTR(-EINVAL); - - /* Get free descriptor */ - spin_lock_irqsave(&schan->lock, iflags); - if (!list_empty(&schan->free)) { - sdesc = list_first_entry(&schan->free, struct sirfsoc_dma_desc, - node); - list_del(&sdesc->node); - } - spin_unlock_irqrestore(&schan->lock, iflags); - - if (!sdesc) - return NULL; - - /* Place descriptor in prepared list */ - spin_lock_irqsave(&schan->lock, iflags); - sdesc->addr = addr; - sdesc->cyclic = 1; - sdesc->xlen = 0; - sdesc->ylen = buf_len / SIRFSOC_DMA_WORD_LEN - 1; - sdesc->width = 1; - list_add_tail(&sdesc->node, &schan->prepared); - spin_unlock_irqrestore(&schan->lock, iflags); - - return &sdesc->desc; -} - -/* - * The DMA controller consists of 16 independent DMA channels. - * Each channel is allocated to a different function - */ -bool sirfsoc_dma_filter_id(struct dma_chan *chan, void *chan_id) -{ - unsigned int ch_nr = (unsigned int) chan_id; - - if (ch_nr == chan->chan_id + - chan->device->dev_id * SIRFSOC_DMA_CHANNELS) - return true; - - return false; -} -EXPORT_SYMBOL(sirfsoc_dma_filter_id); - -#define SIRFSOC_DMA_BUSWIDTHS \ - (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ - BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ - BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) - -static struct dma_chan *of_dma_sirfsoc_xlate(struct of_phandle_args *dma_spec, - struct of_dma *ofdma) -{ - struct sirfsoc_dma *sdma = ofdma->of_dma_data; - unsigned int request = dma_spec->args[0]; - - if (request >= SIRFSOC_DMA_CHANNELS) - return NULL; - - return dma_get_slave_channel(&sdma->channels[request].chan); -} - -static int sirfsoc_dma_probe(struct platform_device *op) -{ - struct device_node *dn = op->dev.of_node; - struct device *dev = &op->dev; - struct dma_device *dma; - struct sirfsoc_dma *sdma; - struct sirfsoc_dma_chan *schan; - struct sirfsoc_dmadata *data; - struct resource res; - ulong regs_start, regs_size; - u32 id; - int ret, i; - - sdma = devm_kzalloc(dev, sizeof(*sdma), GFP_KERNEL); - if (!sdma) - return -ENOMEM; - - data = (struct sirfsoc_dmadata *) - (of_match_device(op->dev.driver->of_match_table, - &op->dev)->data); - sdma->exec_desc = data->exec; - sdma->type = data->type; - - if (of_property_read_u32(dn, "cell-index", &id)) { - dev_err(dev, "Fail to get DMAC index\n"); - return -ENODEV; - } - - sdma->irq = irq_of_parse_and_map(dn, 0); - if (!sdma->irq) { - dev_err(dev, "Error mapping IRQ!\n"); - return -EINVAL; - } - - sdma->clk = devm_clk_get(dev, NULL); - if (IS_ERR(sdma->clk)) { - dev_err(dev, "failed to get a clock.\n"); - return PTR_ERR(sdma->clk); - } - - ret = of_address_to_resource(dn, 0, &res); - if (ret) { - dev_err(dev, "Error parsing memory region!\n"); - goto irq_dispose; - } - - regs_start = res.start; - regs_size = resource_size(&res); - - sdma->base = devm_ioremap(dev, regs_start, regs_size); - if (!sdma->base) { - dev_err(dev, "Error mapping memory region!\n"); - ret = -ENOMEM; - goto irq_dispose; - } - - ret = request_irq(sdma->irq, &sirfsoc_dma_irq, 0, DRV_NAME, sdma); - if (ret) { - dev_err(dev, "Error requesting IRQ!\n"); - ret = -EINVAL; - goto irq_dispose; - } - - dma = &sdma->dma; - dma->dev = dev; - - dma->device_alloc_chan_resources = sirfsoc_dma_alloc_chan_resources; - dma->device_free_chan_resources = sirfsoc_dma_free_chan_resources; - dma->device_issue_pending = sirfsoc_dma_issue_pending; - dma->device_config = sirfsoc_dma_slave_config; - dma->device_pause = sirfsoc_dma_pause_chan; - dma->device_resume = sirfsoc_dma_resume_chan; - dma->device_terminate_all = sirfsoc_dma_terminate_all; - dma->device_tx_status = sirfsoc_dma_tx_status; - dma->device_prep_interleaved_dma = sirfsoc_dma_prep_interleaved; - dma->device_prep_dma_cyclic = sirfsoc_dma_prep_cyclic; - dma->src_addr_widths = SIRFSOC_DMA_BUSWIDTHS; - dma->dst_addr_widths = SIRFSOC_DMA_BUSWIDTHS; - dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); - - INIT_LIST_HEAD(&dma->channels); - dma_cap_set(DMA_SLAVE, dma->cap_mask); - dma_cap_set(DMA_CYCLIC, dma->cap_mask); - dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); - dma_cap_set(DMA_PRIVATE, dma->cap_mask); - - for (i = 0; i < SIRFSOC_DMA_CHANNELS; i++) { - schan = &sdma->channels[i]; - - schan->chan.device = dma; - dma_cookie_init(&schan->chan); - - INIT_LIST_HEAD(&schan->free); - INIT_LIST_HEAD(&schan->prepared); - INIT_LIST_HEAD(&schan->queued); - INIT_LIST_HEAD(&schan->active); - INIT_LIST_HEAD(&schan->completed); - - spin_lock_init(&schan->lock); - list_add_tail(&schan->chan.device_node, &dma->channels); - } - - tasklet_setup(&sdma->tasklet, sirfsoc_dma_tasklet); - - /* Register DMA engine */ - dev_set_drvdata(dev, sdma); - - ret = dma_async_device_register(dma); - if (ret) - goto free_irq; - - /* Device-tree DMA controller registration */ - ret = of_dma_controller_register(dn, of_dma_sirfsoc_xlate, sdma); - if (ret) { - dev_err(dev, "failed to register DMA controller\n"); - goto unreg_dma_dev; - } - - pm_runtime_enable(&op->dev); - dev_info(dev, "initialized SIRFSOC DMAC driver\n"); - - return 0; - -unreg_dma_dev: - dma_async_device_unregister(dma); -free_irq: - free_irq(sdma->irq, sdma); -irq_dispose: - irq_dispose_mapping(sdma->irq); - return ret; -} - -static int sirfsoc_dma_remove(struct platform_device *op) -{ - struct device *dev = &op->dev; - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - - of_dma_controller_free(op->dev.of_node); - dma_async_device_unregister(&sdma->dma); - free_irq(sdma->irq, sdma); - tasklet_kill(&sdma->tasklet); - irq_dispose_mapping(sdma->irq); - pm_runtime_disable(&op->dev); - if (!pm_runtime_status_suspended(&op->dev)) - sirfsoc_dma_runtime_suspend(&op->dev); - - return 0; -} - -static int __maybe_unused sirfsoc_dma_runtime_suspend(struct device *dev) -{ - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - - clk_disable_unprepare(sdma->clk); - return 0; -} - -static int __maybe_unused sirfsoc_dma_runtime_resume(struct device *dev) -{ - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - int ret; - - ret = clk_prepare_enable(sdma->clk); - if (ret < 0) { - dev_err(dev, "clk_enable failed: %d\n", ret); - return ret; - } - return 0; -} - -static int __maybe_unused sirfsoc_dma_pm_suspend(struct device *dev) -{ - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - struct sirfsoc_dma_regs *save = &sdma->regs_save; - struct sirfsoc_dma_chan *schan; - int ch; - int ret; - int count; - u32 int_offset; - - /* - * if we were runtime-suspended before, resume to enable clock - * before accessing register - */ - if (pm_runtime_status_suspended(dev)) { - ret = sirfsoc_dma_runtime_resume(dev); - if (ret < 0) - return ret; - } - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) { - count = 1; - int_offset = SIRFSOC_DMA_INT_EN_ATLAS7; - } else { - count = SIRFSOC_DMA_CHANNELS; - int_offset = SIRFSOC_DMA_INT_EN; - } - - /* - * DMA controller will lose all registers while suspending - * so we need to save registers for active channels - */ - for (ch = 0; ch < count; ch++) { - schan = &sdma->channels[ch]; - if (list_empty(&schan->active)) - continue; - save->ctrl[ch] = readl_relaxed(sdma->base + - ch * 0x10 + SIRFSOC_DMA_CH_CTRL); - } - save->interrupt_en = readl_relaxed(sdma->base + int_offset); - - /* Disable clock */ - sirfsoc_dma_runtime_suspend(dev); - - return 0; -} - -static int __maybe_unused sirfsoc_dma_pm_resume(struct device *dev) -{ - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - struct sirfsoc_dma_regs *save = &sdma->regs_save; - struct sirfsoc_dma_desc *sdesc; - struct sirfsoc_dma_chan *schan; - int ch; - int ret; - int count; - u32 int_offset; - u32 width_offset; - - /* Enable clock before accessing register */ - ret = sirfsoc_dma_runtime_resume(dev); - if (ret < 0) - return ret; - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) { - count = 1; - int_offset = SIRFSOC_DMA_INT_EN_ATLAS7; - width_offset = SIRFSOC_DMA_WIDTH_ATLAS7; - } else { - count = SIRFSOC_DMA_CHANNELS; - int_offset = SIRFSOC_DMA_INT_EN; - width_offset = SIRFSOC_DMA_WIDTH_0; - } - - writel_relaxed(save->interrupt_en, sdma->base + int_offset); - for (ch = 0; ch < count; ch++) { - schan = &sdma->channels[ch]; - if (list_empty(&schan->active)) - continue; - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, - node); - writel_relaxed(sdesc->width, - sdma->base + width_offset + ch * 4); - writel_relaxed(sdesc->xlen, - sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_XLEN); - writel_relaxed(sdesc->ylen, - sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_YLEN); - writel_relaxed(save->ctrl[ch], - sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_CTRL); - if (sdma->type == SIRFSOC_DMA_VER_A7V2) { - writel_relaxed(sdesc->addr, - sdma->base + SIRFSOC_DMA_CH_ADDR); - } else { - writel_relaxed(sdesc->addr >> 2, - sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_ADDR); - - } - } - - /* if we were runtime-suspended before, suspend again */ - if (pm_runtime_status_suspended(dev)) - sirfsoc_dma_runtime_suspend(dev); - - return 0; -} - -static const struct dev_pm_ops sirfsoc_dma_pm_ops = { - SET_RUNTIME_PM_OPS(sirfsoc_dma_runtime_suspend, sirfsoc_dma_runtime_resume, NULL) - SET_SYSTEM_SLEEP_PM_OPS(sirfsoc_dma_pm_suspend, sirfsoc_dma_pm_resume) -}; - -static struct sirfsoc_dmadata sirfsoc_dmadata_a6 = { - .exec = sirfsoc_dma_execute_hw_a6, - .type = SIRFSOC_DMA_VER_A6, -}; - -static struct sirfsoc_dmadata sirfsoc_dmadata_a7v1 = { - .exec = sirfsoc_dma_execute_hw_a7v1, - .type = SIRFSOC_DMA_VER_A7V1, -}; - -static struct sirfsoc_dmadata sirfsoc_dmadata_a7v2 = { - .exec = sirfsoc_dma_execute_hw_a7v2, - .type = SIRFSOC_DMA_VER_A7V2, -}; - -static const struct of_device_id sirfsoc_dma_match[] = { - { .compatible = "sirf,prima2-dmac", .data = &sirfsoc_dmadata_a6,}, - { .compatible = "sirf,atlas7-dmac", .data = &sirfsoc_dmadata_a7v1,}, - { .compatible = "sirf,atlas7-dmac-v2", .data = &sirfsoc_dmadata_a7v2,}, - {}, -}; -MODULE_DEVICE_TABLE(of, sirfsoc_dma_match); - -static struct platform_driver sirfsoc_dma_driver = { - .probe = sirfsoc_dma_probe, - .remove = sirfsoc_dma_remove, - .driver = { - .name = DRV_NAME, - .pm = &sirfsoc_dma_pm_ops, - .of_match_table = sirfsoc_dma_match, - }, -}; - -static __init int sirfsoc_dma_init(void) -{ - return platform_driver_register(&sirfsoc_dma_driver); -} - -static void __exit sirfsoc_dma_exit(void) -{ - platform_driver_unregister(&sirfsoc_dma_driver); -} - -subsys_initcall(sirfsoc_dma_init); -module_exit(sirfsoc_dma_exit); - -MODULE_AUTHOR("Rongjun Ying <rongjun.ying@csr.com>"); -MODULE_AUTHOR("Barry Song <baohua.song@csr.com>"); -MODULE_DESCRIPTION("SIRFSOC DMA control driver"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 4256e55bbf25..265d7c07b348 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -78,7 +78,7 @@ static int dma40_memcpy_channels[] = { DB8500_DMA_MEMCPY_EV_5, }; -/* Default configuration for physcial memcpy */ +/* Default configuration for physical memcpy */ static const struct stedma40_chan_cfg dma40_memcpy_conf_phy = { .mode = STEDMA40_MODE_PHYSICAL, .dir = DMA_MEM_TO_MEM, diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index f474a1232335..96ad21869ba7 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -121,6 +121,11 @@ struct udma_oes_offsets { #define UDMA_FLAG_PDMA_ACC32 BIT(0) #define UDMA_FLAG_PDMA_BURST BIT(1) #define UDMA_FLAG_TDTYPE BIT(2) +#define UDMA_FLAG_BURST_SIZE BIT(3) +#define UDMA_FLAGS_J7_CLASS (UDMA_FLAG_PDMA_ACC32 | \ + UDMA_FLAG_PDMA_BURST | \ + UDMA_FLAG_TDTYPE | \ + UDMA_FLAG_BURST_SIZE) struct udma_match_data { enum k3_dma_type type; @@ -128,6 +133,7 @@ struct udma_match_data { bool enable_memcpy_support; u32 flags; u32 statictr_z_mask; + u8 burst_size[3]; }; struct udma_soc_data { @@ -436,6 +442,18 @@ static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel) } } +static u8 udma_get_chan_tpl_index(struct udma_tpl *tpl_map, int chan_id) +{ + int i; + + for (i = 0; i < tpl_map->levels; i++) { + if (chan_id >= tpl_map->start_idx[i]) + return i; + } + + return 0; +} + static void udma_reset_uchan(struct udma_chan *uc) { memset(&uc->config, 0, sizeof(uc->config)); @@ -1811,13 +1829,21 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct udma_tchan *tchan = uc->tchan; struct udma_rchan *rchan = uc->rchan; - int ret = 0; + u8 burst_size = 0; + int ret; + u8 tpl; /* Non synchronized - mem to mem type of transfer */ int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; + if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) { + tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, tchan->id); + + burst_size = ud->match_data->burst_size[tpl]; + } + req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS; req_tx.nav_id = tisci_rm->tisci_dev_id; req_tx.index = tchan->id; @@ -1825,6 +1851,10 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; req_tx.txcq_qnum = tc_ring; req_tx.tx_atype = ud->atype; + if (burst_size) { + req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; + req_tx.tx_burst_size = burst_size; + } ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); if (ret) { @@ -1839,6 +1869,10 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) req_rx.rxcq_qnum = tc_ring; req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; req_rx.rx_atype = ud->atype; + if (burst_size) { + req_rx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; + req_rx.rx_burst_size = burst_size; + } ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); if (ret) @@ -1854,12 +1888,24 @@ static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; struct udma_bchan *bchan = uc->bchan; - int ret = 0; + u8 burst_size = 0; + int ret; + u8 tpl; + + if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) { + tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, bchan->id); + + burst_size = ud->match_data->burst_size[tpl]; + } req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS; req_tx.nav_id = tisci_rm->tisci_dev_id; req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN; req_tx.index = bchan->id; + if (burst_size) { + req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; + req_tx.tx_burst_size = burst_size; + } ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); if (ret) @@ -1877,7 +1923,7 @@ static int udma_tisci_tx_channel_config(struct udma_chan *uc) int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; u32 mode, fetch_size; - int ret = 0; + int ret; if (uc->config.pkt_mode) { mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; @@ -1918,7 +1964,7 @@ static int bcdma_tisci_tx_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct udma_tchan *tchan = uc->tchan; struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; - int ret = 0; + int ret; req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS; req_tx.nav_id = tisci_rm->tisci_dev_id; @@ -1951,7 +1997,7 @@ static int udma_tisci_rx_channel_config(struct udma_chan *uc) struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; u32 mode, fetch_size; - int ret = 0; + int ret; if (uc->config.pkt_mode) { mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; @@ -2028,7 +2074,7 @@ static int bcdma_tisci_rx_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct udma_rchan *rchan = uc->rchan; struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; - int ret = 0; + int ret; req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS; req_rx.nav_id = tisci_rm->tisci_dev_id; @@ -2048,7 +2094,7 @@ static int pktdma_tisci_rx_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; - int ret = 0; + int ret; req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS; req_rx.nav_id = tisci_rm->tisci_dev_id; @@ -4168,6 +4214,11 @@ static struct udma_match_data am654_main_data = { .psil_base = 0x1000, .enable_memcpy_support = true, .statictr_z_mask = GENMASK(11, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */ + 0, /* No UH Channels */ + }, }; static struct udma_match_data am654_mcu_data = { @@ -4175,38 +4226,63 @@ static struct udma_match_data am654_mcu_data = { .psil_base = 0x6000, .enable_memcpy_support = false, .statictr_z_mask = GENMASK(11, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */ + 0, /* No UH Channels */ + }, }; static struct udma_match_data j721e_main_data = { .type = DMA_TYPE_UDMA, .psil_base = 0x1000, .enable_memcpy_support = true, - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .flags = UDMA_FLAGS_J7_CLASS, .statictr_z_mask = GENMASK(23, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* H Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* UH Channels */ + }, }; static struct udma_match_data j721e_mcu_data = { .type = DMA_TYPE_UDMA, .psil_base = 0x6000, .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */ - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .flags = UDMA_FLAGS_J7_CLASS, .statictr_z_mask = GENMASK(23, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES, /* H Channels */ + 0, /* No UH Channels */ + }, }; static struct udma_match_data am64_bcdma_data = { .type = DMA_TYPE_BCDMA, .psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */ .enable_memcpy_support = true, /* Supported via bchan */ - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .flags = UDMA_FLAGS_J7_CLASS, .statictr_z_mask = GENMASK(23, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + 0, /* No H Channels */ + 0, /* No UH Channels */ + }, }; static struct udma_match_data am64_pktdma_data = { .type = DMA_TYPE_PKTDMA, .psil_base = 0x1000, .enable_memcpy_support = false, /* PKTDMA does not support MEM_TO_MEM */ - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .flags = UDMA_FLAGS_J7_CLASS, .statictr_z_mask = GENMASK(23, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + 0, /* No H Channels */ + 0, /* No UH Channels */ + }, }; static const struct of_device_id udma_of_match[] = { @@ -4306,6 +4382,7 @@ static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2); ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2); ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2); + ud->rflow_cnt = ud->rchan_cnt; break; case DMA_TYPE_PKTDMA: cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30); @@ -5046,6 +5123,34 @@ static void udma_dbg_summary_show(struct seq_file *s, } #endif /* CONFIG_DEBUG_FS */ +static enum dmaengine_alignment udma_get_copy_align(struct udma_dev *ud) +{ + const struct udma_match_data *match_data = ud->match_data; + u8 tpl; + + if (!match_data->enable_memcpy_support) + return DMAENGINE_ALIGN_8_BYTES; + + /* Get the highest TPL level the device supports for memcpy */ + if (ud->bchan_cnt) + tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, 0); + else if (ud->tchan_cnt) + tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, 0); + else + return DMAENGINE_ALIGN_8_BYTES; + + switch (match_data->burst_size[tpl]) { + case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES: + return DMAENGINE_ALIGN_256_BYTES; + case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES: + return DMAENGINE_ALIGN_128_BYTES; + case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES: + fallthrough; + default: + return DMAENGINE_ALIGN_64_BYTES; + } +} + #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ @@ -5202,7 +5307,6 @@ static int udma_probe(struct platform_device *pdev) ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS; ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; - ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES; ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT | DESC_METADATA_ENGINE; if (ud->match_data->enable_memcpy_support && @@ -5284,6 +5388,9 @@ static int udma_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion); } + /* Configure the copy_align to the maximum burst size the device supports */ + ud->ddev.copy_align = udma_get_copy_align(ud); + ret = dma_async_device_register(&ud->ddev); if (ret) { dev_err(dev, "failed to register slave DMA engine: %d\n", ret); diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 79777550a6ff..3aded7861fef 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -800,7 +800,7 @@ xilinx_dma_alloc_tx_descriptor(struct xilinx_dma_chan *chan) { struct xilinx_dma_tx_descriptor *desc; - desc = kzalloc(sizeof(*desc), GFP_KERNEL); + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); if (!desc) return NULL; diff --git a/drivers/dma/zx_dma.c b/drivers/dma/zx_dma.c deleted file mode 100644 index b057582b2fac..000000000000 --- a/drivers/dma/zx_dma.c +++ /dev/null @@ -1,941 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2015 Linaro. - */ -#include <linux/sched.h> -#include <linux/device.h> -#include <linux/dmaengine.h> -#include <linux/dma-mapping.h> -#include <linux/dmapool.h> -#include <linux/init.h> -#include <linux/interrupt.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/platform_device.h> -#include <linux/slab.h> -#include <linux/spinlock.h> -#include <linux/of_device.h> -#include <linux/of.h> -#include <linux/clk.h> -#include <linux/of_dma.h> - -#include "virt-dma.h" - -#define DRIVER_NAME "zx-dma" -#define DMA_ALIGN 4 -#define DMA_MAX_SIZE (0x10000 - 512) -#define LLI_BLOCK_SIZE (4 * PAGE_SIZE) - -#define REG_ZX_SRC_ADDR 0x00 -#define REG_ZX_DST_ADDR 0x04 -#define REG_ZX_TX_X_COUNT 0x08 -#define REG_ZX_TX_ZY_COUNT 0x0c -#define REG_ZX_SRC_ZY_STEP 0x10 -#define REG_ZX_DST_ZY_STEP 0x14 -#define REG_ZX_LLI_ADDR 0x1c -#define REG_ZX_CTRL 0x20 -#define REG_ZX_TC_IRQ 0x800 -#define REG_ZX_SRC_ERR_IRQ 0x804 -#define REG_ZX_DST_ERR_IRQ 0x808 -#define REG_ZX_CFG_ERR_IRQ 0x80c -#define REG_ZX_TC_IRQ_RAW 0x810 -#define REG_ZX_SRC_ERR_IRQ_RAW 0x814 -#define REG_ZX_DST_ERR_IRQ_RAW 0x818 -#define REG_ZX_CFG_ERR_IRQ_RAW 0x81c -#define REG_ZX_STATUS 0x820 -#define REG_ZX_DMA_GRP_PRIO 0x824 -#define REG_ZX_DMA_ARB 0x828 - -#define ZX_FORCE_CLOSE BIT(31) -#define ZX_DST_BURST_WIDTH(x) (((x) & 0x7) << 13) -#define ZX_MAX_BURST_LEN 16 -#define ZX_SRC_BURST_LEN(x) (((x) & 0xf) << 9) -#define ZX_SRC_BURST_WIDTH(x) (((x) & 0x7) << 6) -#define ZX_IRQ_ENABLE_ALL (3 << 4) -#define ZX_DST_FIFO_MODE BIT(3) -#define ZX_SRC_FIFO_MODE BIT(2) -#define ZX_SOFT_REQ BIT(1) -#define ZX_CH_ENABLE BIT(0) - -#define ZX_DMA_BUSWIDTHS \ - (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ - BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ - BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) - -enum zx_dma_burst_width { - ZX_DMA_WIDTH_8BIT = 0, - ZX_DMA_WIDTH_16BIT = 1, - ZX_DMA_WIDTH_32BIT = 2, - ZX_DMA_WIDTH_64BIT = 3, -}; - -struct zx_desc_hw { - u32 saddr; - u32 daddr; - u32 src_x; - u32 src_zy; - u32 src_zy_step; - u32 dst_zy_step; - u32 reserved1; - u32 lli; - u32 ctr; - u32 reserved[7]; /* pack as hardware registers region size */ -} __aligned(32); - -struct zx_dma_desc_sw { - struct virt_dma_desc vd; - dma_addr_t desc_hw_lli; - size_t desc_num; - size_t size; - struct zx_desc_hw *desc_hw; -}; - -struct zx_dma_phy; - -struct zx_dma_chan { - struct dma_slave_config slave_cfg; - int id; /* Request phy chan id */ - u32 ccfg; - u32 cyclic; - struct virt_dma_chan vc; - struct zx_dma_phy *phy; - struct list_head node; - dma_addr_t dev_addr; - enum dma_status status; -}; - -struct zx_dma_phy { - u32 idx; - void __iomem *base; - struct zx_dma_chan *vchan; - struct zx_dma_desc_sw *ds_run; - struct zx_dma_desc_sw *ds_done; -}; - -struct zx_dma_dev { - struct dma_device slave; - void __iomem *base; - spinlock_t lock; /* lock for ch and phy */ - struct list_head chan_pending; - struct zx_dma_phy *phy; - struct zx_dma_chan *chans; - struct clk *clk; - struct dma_pool *pool; - u32 dma_channels; - u32 dma_requests; - int irq; -}; - -#define to_zx_dma(dmadev) container_of(dmadev, struct zx_dma_dev, slave) - -static struct zx_dma_chan *to_zx_chan(struct dma_chan *chan) -{ - return container_of(chan, struct zx_dma_chan, vc.chan); -} - -static void zx_dma_terminate_chan(struct zx_dma_phy *phy, struct zx_dma_dev *d) -{ - u32 val = 0; - - val = readl_relaxed(phy->base + REG_ZX_CTRL); - val &= ~ZX_CH_ENABLE; - val |= ZX_FORCE_CLOSE; - writel_relaxed(val, phy->base + REG_ZX_CTRL); - - val = 0x1 << phy->idx; - writel_relaxed(val, d->base + REG_ZX_TC_IRQ_RAW); - writel_relaxed(val, d->base + REG_ZX_SRC_ERR_IRQ_RAW); - writel_relaxed(val, d->base + REG_ZX_DST_ERR_IRQ_RAW); - writel_relaxed(val, d->base + REG_ZX_CFG_ERR_IRQ_RAW); -} - -static void zx_dma_set_desc(struct zx_dma_phy *phy, struct zx_desc_hw *hw) -{ - writel_relaxed(hw->saddr, phy->base + REG_ZX_SRC_ADDR); - writel_relaxed(hw->daddr, phy->base + REG_ZX_DST_ADDR); - writel_relaxed(hw->src_x, phy->base + REG_ZX_TX_X_COUNT); - writel_relaxed(0, phy->base + REG_ZX_TX_ZY_COUNT); - writel_relaxed(0, phy->base + REG_ZX_SRC_ZY_STEP); - writel_relaxed(0, phy->base + REG_ZX_DST_ZY_STEP); - writel_relaxed(hw->lli, phy->base + REG_ZX_LLI_ADDR); - writel_relaxed(hw->ctr, phy->base + REG_ZX_CTRL); -} - -static u32 zx_dma_get_curr_lli(struct zx_dma_phy *phy) -{ - return readl_relaxed(phy->base + REG_ZX_LLI_ADDR); -} - -static u32 zx_dma_get_chan_stat(struct zx_dma_dev *d) -{ - return readl_relaxed(d->base + REG_ZX_STATUS); -} - -static void zx_dma_init_state(struct zx_dma_dev *d) -{ - /* set same priority */ - writel_relaxed(0x0, d->base + REG_ZX_DMA_ARB); - /* clear all irq */ - writel_relaxed(0xffffffff, d->base + REG_ZX_TC_IRQ_RAW); - writel_relaxed(0xffffffff, d->base + REG_ZX_SRC_ERR_IRQ_RAW); - writel_relaxed(0xffffffff, d->base + REG_ZX_DST_ERR_IRQ_RAW); - writel_relaxed(0xffffffff, d->base + REG_ZX_CFG_ERR_IRQ_RAW); -} - -static int zx_dma_start_txd(struct zx_dma_chan *c) -{ - struct zx_dma_dev *d = to_zx_dma(c->vc.chan.device); - struct virt_dma_desc *vd = vchan_next_desc(&c->vc); - - if (!c->phy) - return -EAGAIN; - - if (BIT(c->phy->idx) & zx_dma_get_chan_stat(d)) - return -EAGAIN; - - if (vd) { - struct zx_dma_desc_sw *ds = - container_of(vd, struct zx_dma_desc_sw, vd); - /* - * fetch and remove request from vc->desc_issued - * so vc->desc_issued only contains desc pending - */ - list_del(&ds->vd.node); - c->phy->ds_run = ds; - c->phy->ds_done = NULL; - /* start dma */ - zx_dma_set_desc(c->phy, ds->desc_hw); - return 0; - } - c->phy->ds_done = NULL; - c->phy->ds_run = NULL; - return -EAGAIN; -} - -static void zx_dma_task(struct zx_dma_dev *d) -{ - struct zx_dma_phy *p; - struct zx_dma_chan *c, *cn; - unsigned pch, pch_alloc = 0; - unsigned long flags; - - /* check new dma request of running channel in vc->desc_issued */ - list_for_each_entry_safe(c, cn, &d->slave.channels, - vc.chan.device_node) { - spin_lock_irqsave(&c->vc.lock, flags); - p = c->phy; - if (p && p->ds_done && zx_dma_start_txd(c)) { - /* No current txd associated with this channel */ - dev_dbg(d->slave.dev, "pchan %u: free\n", p->idx); - /* Mark this channel free */ - c->phy = NULL; - p->vchan = NULL; - } - spin_unlock_irqrestore(&c->vc.lock, flags); - } - - /* check new channel request in d->chan_pending */ - spin_lock_irqsave(&d->lock, flags); - while (!list_empty(&d->chan_pending)) { - c = list_first_entry(&d->chan_pending, - struct zx_dma_chan, node); - p = &d->phy[c->id]; - if (!p->vchan) { - /* remove from d->chan_pending */ - list_del_init(&c->node); - pch_alloc |= 1 << c->id; - /* Mark this channel allocated */ - p->vchan = c; - c->phy = p; - } else { - dev_dbg(d->slave.dev, "pchan %u: busy!\n", c->id); - } - } - spin_unlock_irqrestore(&d->lock, flags); - - for (pch = 0; pch < d->dma_channels; pch++) { - if (pch_alloc & (1 << pch)) { - p = &d->phy[pch]; - c = p->vchan; - if (c) { - spin_lock_irqsave(&c->vc.lock, flags); - zx_dma_start_txd(c); - spin_unlock_irqrestore(&c->vc.lock, flags); - } - } - } -} - -static irqreturn_t zx_dma_int_handler(int irq, void *dev_id) -{ - struct zx_dma_dev *d = (struct zx_dma_dev *)dev_id; - struct zx_dma_phy *p; - struct zx_dma_chan *c; - u32 tc = readl_relaxed(d->base + REG_ZX_TC_IRQ); - u32 serr = readl_relaxed(d->base + REG_ZX_SRC_ERR_IRQ); - u32 derr = readl_relaxed(d->base + REG_ZX_DST_ERR_IRQ); - u32 cfg = readl_relaxed(d->base + REG_ZX_CFG_ERR_IRQ); - u32 i, irq_chan = 0, task = 0; - - while (tc) { - i = __ffs(tc); - tc &= ~BIT(i); - p = &d->phy[i]; - c = p->vchan; - if (c) { - spin_lock(&c->vc.lock); - if (c->cyclic) { - vchan_cyclic_callback(&p->ds_run->vd); - } else { - vchan_cookie_complete(&p->ds_run->vd); - p->ds_done = p->ds_run; - task = 1; - } - spin_unlock(&c->vc.lock); - irq_chan |= BIT(i); - } - } - - if (serr || derr || cfg) - dev_warn(d->slave.dev, "DMA ERR src 0x%x, dst 0x%x, cfg 0x%x\n", - serr, derr, cfg); - - writel_relaxed(irq_chan, d->base + REG_ZX_TC_IRQ_RAW); - writel_relaxed(serr, d->base + REG_ZX_SRC_ERR_IRQ_RAW); - writel_relaxed(derr, d->base + REG_ZX_DST_ERR_IRQ_RAW); - writel_relaxed(cfg, d->base + REG_ZX_CFG_ERR_IRQ_RAW); - - if (task) - zx_dma_task(d); - return IRQ_HANDLED; -} - -static void zx_dma_free_chan_resources(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_dev *d = to_zx_dma(chan->device); - unsigned long flags; - - spin_lock_irqsave(&d->lock, flags); - list_del_init(&c->node); - spin_unlock_irqrestore(&d->lock, flags); - - vchan_free_chan_resources(&c->vc); - c->ccfg = 0; -} - -static enum dma_status zx_dma_tx_status(struct dma_chan *chan, - dma_cookie_t cookie, - struct dma_tx_state *state) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_phy *p; - struct virt_dma_desc *vd; - unsigned long flags; - enum dma_status ret; - size_t bytes = 0; - - ret = dma_cookie_status(&c->vc.chan, cookie, state); - if (ret == DMA_COMPLETE || !state) - return ret; - - spin_lock_irqsave(&c->vc.lock, flags); - p = c->phy; - ret = c->status; - - /* - * If the cookie is on our issue queue, then the residue is - * its total size. - */ - vd = vchan_find_desc(&c->vc, cookie); - if (vd) { - bytes = container_of(vd, struct zx_dma_desc_sw, vd)->size; - } else if ((!p) || (!p->ds_run)) { - bytes = 0; - } else { - struct zx_dma_desc_sw *ds = p->ds_run; - u32 clli = 0, index = 0; - - bytes = 0; - clli = zx_dma_get_curr_lli(p); - index = (clli - ds->desc_hw_lli) / - sizeof(struct zx_desc_hw) + 1; - for (; index < ds->desc_num; index++) { - bytes += ds->desc_hw[index].src_x; - /* end of lli */ - if (!ds->desc_hw[index].lli) - break; - } - } - spin_unlock_irqrestore(&c->vc.lock, flags); - dma_set_residue(state, bytes); - return ret; -} - -static void zx_dma_issue_pending(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_dev *d = to_zx_dma(chan->device); - unsigned long flags; - int issue = 0; - - spin_lock_irqsave(&c->vc.lock, flags); - /* add request to vc->desc_issued */ - if (vchan_issue_pending(&c->vc)) { - spin_lock(&d->lock); - if (!c->phy && list_empty(&c->node)) { - /* if new channel, add chan_pending */ - list_add_tail(&c->node, &d->chan_pending); - issue = 1; - dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc); - } - spin_unlock(&d->lock); - } else { - dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc); - } - spin_unlock_irqrestore(&c->vc.lock, flags); - - if (issue) - zx_dma_task(d); -} - -static void zx_dma_fill_desc(struct zx_dma_desc_sw *ds, dma_addr_t dst, - dma_addr_t src, size_t len, u32 num, u32 ccfg) -{ - if ((num + 1) < ds->desc_num) - ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) * - sizeof(struct zx_desc_hw); - ds->desc_hw[num].saddr = src; - ds->desc_hw[num].daddr = dst; - ds->desc_hw[num].src_x = len; - ds->desc_hw[num].ctr = ccfg; -} - -static struct zx_dma_desc_sw *zx_alloc_desc_resource(int num, - struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_desc_sw *ds; - struct zx_dma_dev *d = to_zx_dma(chan->device); - int lli_limit = LLI_BLOCK_SIZE / sizeof(struct zx_desc_hw); - - if (num > lli_limit) { - dev_dbg(chan->device->dev, "vch %p: sg num %d exceed max %d\n", - &c->vc, num, lli_limit); - return NULL; - } - - ds = kzalloc(sizeof(*ds), GFP_ATOMIC); - if (!ds) - return NULL; - - ds->desc_hw = dma_pool_zalloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli); - if (!ds->desc_hw) { - dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc); - kfree(ds); - return NULL; - } - ds->desc_num = num; - return ds; -} - -static enum zx_dma_burst_width zx_dma_burst_width(enum dma_slave_buswidth width) -{ - switch (width) { - case DMA_SLAVE_BUSWIDTH_1_BYTE: - case DMA_SLAVE_BUSWIDTH_2_BYTES: - case DMA_SLAVE_BUSWIDTH_4_BYTES: - case DMA_SLAVE_BUSWIDTH_8_BYTES: - return ffs(width) - 1; - default: - return ZX_DMA_WIDTH_32BIT; - } -} - -static int zx_pre_config(struct zx_dma_chan *c, enum dma_transfer_direction dir) -{ - struct dma_slave_config *cfg = &c->slave_cfg; - enum zx_dma_burst_width src_width; - enum zx_dma_burst_width dst_width; - u32 maxburst = 0; - - switch (dir) { - case DMA_MEM_TO_MEM: - c->ccfg = ZX_CH_ENABLE | ZX_SOFT_REQ - | ZX_SRC_BURST_LEN(ZX_MAX_BURST_LEN - 1) - | ZX_SRC_BURST_WIDTH(ZX_DMA_WIDTH_32BIT) - | ZX_DST_BURST_WIDTH(ZX_DMA_WIDTH_32BIT); - break; - case DMA_MEM_TO_DEV: - c->dev_addr = cfg->dst_addr; - /* dst len is calculated from src width, len and dst width. - * We need make sure dst len not exceed MAX LEN. - * Trailing single transaction that does not fill a full - * burst also require identical src/dst data width. - */ - dst_width = zx_dma_burst_width(cfg->dst_addr_width); - maxburst = cfg->dst_maxburst; - maxburst = maxburst < ZX_MAX_BURST_LEN ? - maxburst : ZX_MAX_BURST_LEN; - c->ccfg = ZX_DST_FIFO_MODE | ZX_CH_ENABLE - | ZX_SRC_BURST_LEN(maxburst - 1) - | ZX_SRC_BURST_WIDTH(dst_width) - | ZX_DST_BURST_WIDTH(dst_width); - break; - case DMA_DEV_TO_MEM: - c->dev_addr = cfg->src_addr; - src_width = zx_dma_burst_width(cfg->src_addr_width); - maxburst = cfg->src_maxburst; - maxburst = maxburst < ZX_MAX_BURST_LEN ? - maxburst : ZX_MAX_BURST_LEN; - c->ccfg = ZX_SRC_FIFO_MODE | ZX_CH_ENABLE - | ZX_SRC_BURST_LEN(maxburst - 1) - | ZX_SRC_BURST_WIDTH(src_width) - | ZX_DST_BURST_WIDTH(src_width); - break; - default: - return -EINVAL; - } - return 0; -} - -static struct dma_async_tx_descriptor *zx_dma_prep_memcpy( - struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, - size_t len, unsigned long flags) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_desc_sw *ds; - size_t copy = 0; - int num = 0; - - if (!len) - return NULL; - - if (zx_pre_config(c, DMA_MEM_TO_MEM)) - return NULL; - - num = DIV_ROUND_UP(len, DMA_MAX_SIZE); - - ds = zx_alloc_desc_resource(num, chan); - if (!ds) - return NULL; - - ds->size = len; - num = 0; - - do { - copy = min_t(size_t, len, DMA_MAX_SIZE); - zx_dma_fill_desc(ds, dst, src, copy, num++, c->ccfg); - - src += copy; - dst += copy; - len -= copy; - } while (len); - - c->cyclic = 0; - ds->desc_hw[num - 1].lli = 0; /* end of link */ - ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL; - return vchan_tx_prep(&c->vc, &ds->vd, flags); -} - -static struct dma_async_tx_descriptor *zx_dma_prep_slave_sg( - struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen, - enum dma_transfer_direction dir, unsigned long flags, void *context) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_desc_sw *ds; - size_t len, avail, total = 0; - struct scatterlist *sg; - dma_addr_t addr, src = 0, dst = 0; - int num = sglen, i; - - if (!sgl) - return NULL; - - if (zx_pre_config(c, dir)) - return NULL; - - for_each_sg(sgl, sg, sglen, i) { - avail = sg_dma_len(sg); - if (avail > DMA_MAX_SIZE) - num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1; - } - - ds = zx_alloc_desc_resource(num, chan); - if (!ds) - return NULL; - - c->cyclic = 0; - num = 0; - for_each_sg(sgl, sg, sglen, i) { - addr = sg_dma_address(sg); - avail = sg_dma_len(sg); - total += avail; - - do { - len = min_t(size_t, avail, DMA_MAX_SIZE); - - if (dir == DMA_MEM_TO_DEV) { - src = addr; - dst = c->dev_addr; - } else if (dir == DMA_DEV_TO_MEM) { - src = c->dev_addr; - dst = addr; - } - - zx_dma_fill_desc(ds, dst, src, len, num++, c->ccfg); - - addr += len; - avail -= len; - } while (avail); - } - - ds->desc_hw[num - 1].lli = 0; /* end of link */ - ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL; - ds->size = total; - return vchan_tx_prep(&c->vc, &ds->vd, flags); -} - -static struct dma_async_tx_descriptor *zx_dma_prep_dma_cyclic( - struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, - size_t period_len, enum dma_transfer_direction dir, - unsigned long flags) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_desc_sw *ds; - dma_addr_t src = 0, dst = 0; - int num_periods = buf_len / period_len; - int buf = 0, num = 0; - - if (period_len > DMA_MAX_SIZE) { - dev_err(chan->device->dev, "maximum period size exceeded\n"); - return NULL; - } - - if (zx_pre_config(c, dir)) - return NULL; - - ds = zx_alloc_desc_resource(num_periods, chan); - if (!ds) - return NULL; - c->cyclic = 1; - - while (buf < buf_len) { - if (dir == DMA_MEM_TO_DEV) { - src = dma_addr; - dst = c->dev_addr; - } else if (dir == DMA_DEV_TO_MEM) { - src = c->dev_addr; - dst = dma_addr; - } - zx_dma_fill_desc(ds, dst, src, period_len, num++, - c->ccfg | ZX_IRQ_ENABLE_ALL); - dma_addr += period_len; - buf += period_len; - } - - ds->desc_hw[num - 1].lli = ds->desc_hw_lli; - ds->size = buf_len; - return vchan_tx_prep(&c->vc, &ds->vd, flags); -} - -static int zx_dma_config(struct dma_chan *chan, - struct dma_slave_config *cfg) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - - if (!cfg) - return -EINVAL; - - memcpy(&c->slave_cfg, cfg, sizeof(*cfg)); - - return 0; -} - -static int zx_dma_terminate_all(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_dev *d = to_zx_dma(chan->device); - struct zx_dma_phy *p = c->phy; - unsigned long flags; - LIST_HEAD(head); - - dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc); - - /* Prevent this channel being scheduled */ - spin_lock(&d->lock); - list_del_init(&c->node); - spin_unlock(&d->lock); - - /* Clear the tx descriptor lists */ - spin_lock_irqsave(&c->vc.lock, flags); - vchan_get_all_descriptors(&c->vc, &head); - if (p) { - /* vchan is assigned to a pchan - stop the channel */ - zx_dma_terminate_chan(p, d); - c->phy = NULL; - p->vchan = NULL; - p->ds_run = NULL; - p->ds_done = NULL; - } - spin_unlock_irqrestore(&c->vc.lock, flags); - vchan_dma_desc_free_list(&c->vc, &head); - - return 0; -} - -static int zx_dma_transfer_pause(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - u32 val = 0; - - val = readl_relaxed(c->phy->base + REG_ZX_CTRL); - val &= ~ZX_CH_ENABLE; - writel_relaxed(val, c->phy->base + REG_ZX_CTRL); - - return 0; -} - -static int zx_dma_transfer_resume(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - u32 val = 0; - - val = readl_relaxed(c->phy->base + REG_ZX_CTRL); - val |= ZX_CH_ENABLE; - writel_relaxed(val, c->phy->base + REG_ZX_CTRL); - - return 0; -} - -static void zx_dma_free_desc(struct virt_dma_desc *vd) -{ - struct zx_dma_desc_sw *ds = - container_of(vd, struct zx_dma_desc_sw, vd); - struct zx_dma_dev *d = to_zx_dma(vd->tx.chan->device); - - dma_pool_free(d->pool, ds->desc_hw, ds->desc_hw_lli); - kfree(ds); -} - -static const struct of_device_id zx6702_dma_dt_ids[] = { - { .compatible = "zte,zx296702-dma", }, - {} -}; -MODULE_DEVICE_TABLE(of, zx6702_dma_dt_ids); - -static struct dma_chan *zx_of_dma_simple_xlate(struct of_phandle_args *dma_spec, - struct of_dma *ofdma) -{ - struct zx_dma_dev *d = ofdma->of_dma_data; - unsigned int request = dma_spec->args[0]; - struct dma_chan *chan; - struct zx_dma_chan *c; - - if (request >= d->dma_requests) - return NULL; - - chan = dma_get_any_slave_channel(&d->slave); - if (!chan) { - dev_err(d->slave.dev, "get channel fail in %s.\n", __func__); - return NULL; - } - c = to_zx_chan(chan); - c->id = request; - dev_info(d->slave.dev, "zx_dma: pchan %u: alloc vchan %p\n", - c->id, &c->vc); - return chan; -} - -static int zx_dma_probe(struct platform_device *op) -{ - struct zx_dma_dev *d; - int i, ret = 0; - - d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL); - if (!d) - return -ENOMEM; - - d->base = devm_platform_ioremap_resource(op, 0); - if (IS_ERR(d->base)) - return PTR_ERR(d->base); - - of_property_read_u32((&op->dev)->of_node, - "dma-channels", &d->dma_channels); - of_property_read_u32((&op->dev)->of_node, - "dma-requests", &d->dma_requests); - if (!d->dma_requests || !d->dma_channels) - return -EINVAL; - - d->clk = devm_clk_get(&op->dev, NULL); - if (IS_ERR(d->clk)) { - dev_err(&op->dev, "no dma clk\n"); - return PTR_ERR(d->clk); - } - - d->irq = platform_get_irq(op, 0); - ret = devm_request_irq(&op->dev, d->irq, zx_dma_int_handler, - 0, DRIVER_NAME, d); - if (ret) - return ret; - - /* A DMA memory pool for LLIs, align on 32-byte boundary */ - d->pool = dmam_pool_create(DRIVER_NAME, &op->dev, - LLI_BLOCK_SIZE, 32, 0); - if (!d->pool) - return -ENOMEM; - - /* init phy channel */ - d->phy = devm_kcalloc(&op->dev, - d->dma_channels, sizeof(struct zx_dma_phy), GFP_KERNEL); - if (!d->phy) - return -ENOMEM; - - for (i = 0; i < d->dma_channels; i++) { - struct zx_dma_phy *p = &d->phy[i]; - - p->idx = i; - p->base = d->base + i * 0x40; - } - - INIT_LIST_HEAD(&d->slave.channels); - dma_cap_set(DMA_SLAVE, d->slave.cap_mask); - dma_cap_set(DMA_MEMCPY, d->slave.cap_mask); - dma_cap_set(DMA_CYCLIC, d->slave.cap_mask); - dma_cap_set(DMA_PRIVATE, d->slave.cap_mask); - d->slave.dev = &op->dev; - d->slave.device_free_chan_resources = zx_dma_free_chan_resources; - d->slave.device_tx_status = zx_dma_tx_status; - d->slave.device_prep_dma_memcpy = zx_dma_prep_memcpy; - d->slave.device_prep_slave_sg = zx_dma_prep_slave_sg; - d->slave.device_prep_dma_cyclic = zx_dma_prep_dma_cyclic; - d->slave.device_issue_pending = zx_dma_issue_pending; - d->slave.device_config = zx_dma_config; - d->slave.device_terminate_all = zx_dma_terminate_all; - d->slave.device_pause = zx_dma_transfer_pause; - d->slave.device_resume = zx_dma_transfer_resume; - d->slave.copy_align = DMA_ALIGN; - d->slave.src_addr_widths = ZX_DMA_BUSWIDTHS; - d->slave.dst_addr_widths = ZX_DMA_BUSWIDTHS; - d->slave.directions = BIT(DMA_MEM_TO_MEM) | BIT(DMA_MEM_TO_DEV) - | BIT(DMA_DEV_TO_MEM); - d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; - - /* init virtual channel */ - d->chans = devm_kcalloc(&op->dev, - d->dma_requests, sizeof(struct zx_dma_chan), GFP_KERNEL); - if (!d->chans) - return -ENOMEM; - - for (i = 0; i < d->dma_requests; i++) { - struct zx_dma_chan *c = &d->chans[i]; - - c->status = DMA_IN_PROGRESS; - INIT_LIST_HEAD(&c->node); - c->vc.desc_free = zx_dma_free_desc; - vchan_init(&c->vc, &d->slave); - } - - /* Enable clock before accessing registers */ - ret = clk_prepare_enable(d->clk); - if (ret < 0) { - dev_err(&op->dev, "clk_prepare_enable failed: %d\n", ret); - goto zx_dma_out; - } - - zx_dma_init_state(d); - - spin_lock_init(&d->lock); - INIT_LIST_HEAD(&d->chan_pending); - platform_set_drvdata(op, d); - - ret = dma_async_device_register(&d->slave); - if (ret) - goto clk_dis; - - ret = of_dma_controller_register((&op->dev)->of_node, - zx_of_dma_simple_xlate, d); - if (ret) - goto of_dma_register_fail; - - dev_info(&op->dev, "initialized\n"); - return 0; - -of_dma_register_fail: - dma_async_device_unregister(&d->slave); -clk_dis: - clk_disable_unprepare(d->clk); -zx_dma_out: - return ret; -} - -static int zx_dma_remove(struct platform_device *op) -{ - struct zx_dma_chan *c, *cn; - struct zx_dma_dev *d = platform_get_drvdata(op); - - /* explictly free the irq */ - devm_free_irq(&op->dev, d->irq, d); - - dma_async_device_unregister(&d->slave); - of_dma_controller_free((&op->dev)->of_node); - - list_for_each_entry_safe(c, cn, &d->slave.channels, - vc.chan.device_node) { - list_del(&c->vc.chan.device_node); - } - clk_disable_unprepare(d->clk); - - return 0; -} - -#ifdef CONFIG_PM_SLEEP -static int zx_dma_suspend_dev(struct device *dev) -{ - struct zx_dma_dev *d = dev_get_drvdata(dev); - u32 stat = 0; - - stat = zx_dma_get_chan_stat(d); - if (stat) { - dev_warn(d->slave.dev, - "chan %d is running fail to suspend\n", stat); - return -1; - } - clk_disable_unprepare(d->clk); - return 0; -} - -static int zx_dma_resume_dev(struct device *dev) -{ - struct zx_dma_dev *d = dev_get_drvdata(dev); - int ret = 0; - - ret = clk_prepare_enable(d->clk); - if (ret < 0) { - dev_err(d->slave.dev, "clk_prepare_enable failed: %d\n", ret); - return ret; - } - zx_dma_init_state(d); - return 0; -} -#endif - -static SIMPLE_DEV_PM_OPS(zx_dma_pmops, zx_dma_suspend_dev, zx_dma_resume_dev); - -static struct platform_driver zx_pdma_driver = { - .driver = { - .name = DRIVER_NAME, - .pm = &zx_dma_pmops, - .of_match_table = zx6702_dma_dt_ids, - }, - .probe = zx_dma_probe, - .remove = zx_dma_remove, -}; - -module_platform_driver(zx_pdma_driver); - -MODULE_DESCRIPTION("ZTE ZX296702 DMA Driver"); -MODULE_AUTHOR("Jun Nie jun.nie@linaro.org"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/firewire/core-device.c b/drivers/firewire/core-device.c index 80db43a22069..68216988391f 100644 --- a/drivers/firewire/core-device.c +++ b/drivers/firewire/core-device.c @@ -192,7 +192,9 @@ static int fw_unit_remove(struct device *dev) struct fw_driver *driver = container_of(dev->driver, struct fw_driver, driver); - return driver->remove(fw_unit(dev)), 0; + driver->remove(fw_unit(dev)); + + return 0; } static int get_modalias(struct fw_unit *unit, char *buffer, size_t buffer_size) diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index 0205987a4fd4..dc83ea118c67 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -46,14 +46,13 @@ static int coreboot_bus_probe(struct device *dev) static int coreboot_bus_remove(struct device *dev) { - int ret = 0; struct coreboot_device *device = CB_DEV(dev); struct coreboot_driver *driver = CB_DRV(dev->driver); if (driver->remove) - ret = driver->remove(device); + driver->remove(device); - return ret; + return 0; } static struct bus_type coreboot_bus_type = { diff --git a/drivers/firmware/google/coreboot_table.h b/drivers/firmware/google/coreboot_table.h index 7b7b4a6eedda..beb778674acd 100644 --- a/drivers/firmware/google/coreboot_table.h +++ b/drivers/firmware/google/coreboot_table.h @@ -72,7 +72,7 @@ struct coreboot_device { /* A driver for handling devices described in coreboot tables. */ struct coreboot_driver { int (*probe)(struct coreboot_device *); - int (*remove)(struct coreboot_device *); + void (*remove)(struct coreboot_device *); struct device_driver drv; u32 tag; }; diff --git a/drivers/firmware/google/framebuffer-coreboot.c b/drivers/firmware/google/framebuffer-coreboot.c index 916f26adc595..c6dcc1ef93ac 100644 --- a/drivers/firmware/google/framebuffer-coreboot.c +++ b/drivers/firmware/google/framebuffer-coreboot.c @@ -72,13 +72,11 @@ static int framebuffer_probe(struct coreboot_device *dev) return PTR_ERR_OR_ZERO(pdev); } -static int framebuffer_remove(struct coreboot_device *dev) +static void framebuffer_remove(struct coreboot_device *dev) { struct platform_device *pdev = dev_get_drvdata(&dev->dev); platform_device_unregister(pdev); - - return 0; } static struct coreboot_driver framebuffer_driver = { diff --git a/drivers/firmware/google/memconsole-coreboot.c b/drivers/firmware/google/memconsole-coreboot.c index d17e4d6ac9bc..74b5286518ee 100644 --- a/drivers/firmware/google/memconsole-coreboot.c +++ b/drivers/firmware/google/memconsole-coreboot.c @@ -91,11 +91,9 @@ static int memconsole_probe(struct coreboot_device *dev) return memconsole_sysfs_init(); } -static int memconsole_remove(struct coreboot_device *dev) +static void memconsole_remove(struct coreboot_device *dev) { memconsole_exit(); - - return 0; } static struct coreboot_driver memconsole_driver = { diff --git a/drivers/firmware/google/vpd.c b/drivers/firmware/google/vpd.c index d23c5c69ab52..ee6e08c0592b 100644 --- a/drivers/firmware/google/vpd.c +++ b/drivers/firmware/google/vpd.c @@ -298,14 +298,12 @@ static int vpd_probe(struct coreboot_device *dev) return 0; } -static int vpd_remove(struct coreboot_device *dev) +static void vpd_remove(struct coreboot_device *dev) { vpd_section_destroy(&ro_vpd); vpd_section_destroy(&rw_vpd); kobject_put(vpd_kobj); - - return 0; } static struct coreboot_driver vpd_driver = { diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig index 5645226ca3ce..5ff9438b7b46 100644 --- a/drivers/fpga/Kconfig +++ b/drivers/fpga/Kconfig @@ -192,6 +192,17 @@ config FPGA_DFL_AFU to the FPGA infrastructure via a Port. There may be more than one Port/AFU per DFL based FPGA device. +config FPGA_DFL_NIOS_INTEL_PAC_N3000 + tristate "FPGA DFL NIOS Driver for Intel PAC N3000" + depends on FPGA_DFL + select REGMAP + help + This is the driver for the N3000 Nios private feature on Intel + PAC (Programmable Acceleration Card) N3000. It communicates + with the embedded Nios processor to configure the retimers on + the card. It also instantiates the SPI master (spi-altera) for + the card's BMC (Board Management Controller). + config FPGA_DFL_PCI tristate "FPGA DFL PCIe Device Driver" depends on PCI && FPGA_DFL diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile index d8e21dfc6778..18dc9885883a 100644 --- a/drivers/fpga/Makefile +++ b/drivers/fpga/Makefile @@ -44,5 +44,7 @@ dfl-fme-objs += dfl-fme-perf.o dfl-afu-objs := dfl-afu-main.o dfl-afu-region.o dfl-afu-dma-region.o dfl-afu-objs += dfl-afu-error.o +obj-$(CONFIG_FPGA_DFL_NIOS_INTEL_PAC_N3000) += dfl-n3000-nios.o + # Drivers for FPGAs which implement DFL obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o diff --git a/drivers/fpga/dfl-fme-perf.c b/drivers/fpga/dfl-fme-perf.c index 531266287eee..4299145ef347 100644 --- a/drivers/fpga/dfl-fme-perf.c +++ b/drivers/fpga/dfl-fme-perf.c @@ -192,7 +192,7 @@ static struct attribute *fme_perf_cpumask_attrs[] = { NULL, }; -static struct attribute_group fme_perf_cpumask_group = { +static const struct attribute_group fme_perf_cpumask_group = { .attrs = fme_perf_cpumask_attrs, }; @@ -225,7 +225,7 @@ static struct attribute *fme_perf_format_attrs[] = { NULL, }; -static struct attribute_group fme_perf_format_group = { +static const struct attribute_group fme_perf_format_group = { .name = "format", .attrs = fme_perf_format_attrs, }; @@ -239,7 +239,7 @@ static struct attribute *fme_perf_events_attrs_empty[] = { NULL, }; -static struct attribute_group fme_perf_events_group = { +static const struct attribute_group fme_perf_events_group = { .name = "events", .attrs = fme_perf_events_attrs_empty, }; diff --git a/drivers/fpga/dfl-n3000-nios.c b/drivers/fpga/dfl-n3000-nios.c new file mode 100644 index 000000000000..7a95366f6516 --- /dev/null +++ b/drivers/fpga/dfl-n3000-nios.c @@ -0,0 +1,588 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DFL device driver for Nios private feature on Intel PAC (Programmable + * Acceleration Card) N3000 + * + * Copyright (C) 2019-2020 Intel Corporation, Inc. + * + * Authors: + * Wu Hao <hao.wu@intel.com> + * Xu Yilun <yilun.xu@intel.com> + */ +#include <linux/bitfield.h> +#include <linux/dfl.h> +#include <linux/errno.h> +#include <linux/io.h> +#include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/stddef.h> +#include <linux/spi/altera.h> +#include <linux/spi/spi.h> +#include <linux/types.h> + +/* + * N3000 Nios private feature registers, named as NIOS_SPI_XX on spec. + * NS is the abbreviation of NIOS_SPI. + */ +#define N3000_NS_PARAM 0x8 +#define N3000_NS_PARAM_SHIFT_MODE_MSK BIT_ULL(1) +#define N3000_NS_PARAM_SHIFT_MODE_MSB 0 +#define N3000_NS_PARAM_SHIFT_MODE_LSB 1 +#define N3000_NS_PARAM_DATA_WIDTH GENMASK_ULL(7, 2) +#define N3000_NS_PARAM_NUM_CS GENMASK_ULL(13, 8) +#define N3000_NS_PARAM_CLK_POL BIT_ULL(14) +#define N3000_NS_PARAM_CLK_PHASE BIT_ULL(15) +#define N3000_NS_PARAM_PERIPHERAL_ID GENMASK_ULL(47, 32) + +#define N3000_NS_CTRL 0x10 +#define N3000_NS_CTRL_WR_DATA GENMASK_ULL(31, 0) +#define N3000_NS_CTRL_ADDR GENMASK_ULL(44, 32) +#define N3000_NS_CTRL_CMD_MSK GENMASK_ULL(63, 62) +#define N3000_NS_CTRL_CMD_NOP 0 +#define N3000_NS_CTRL_CMD_RD 1 +#define N3000_NS_CTRL_CMD_WR 2 + +#define N3000_NS_STAT 0x18 +#define N3000_NS_STAT_RD_DATA GENMASK_ULL(31, 0) +#define N3000_NS_STAT_RW_VAL BIT_ULL(32) + +/* Nios handshake registers, indirect access */ +#define N3000_NIOS_INIT 0x1000 +#define N3000_NIOS_INIT_DONE BIT(0) +#define N3000_NIOS_INIT_START BIT(1) +/* Mode for retimer A, link 0, the same below */ +#define N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK GENMASK(9, 8) +#define N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK GENMASK(11, 10) +#define N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK GENMASK(13, 12) +#define N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK GENMASK(15, 14) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK GENMASK(17, 16) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK GENMASK(19, 18) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK GENMASK(21, 20) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK GENMASK(23, 22) +#define N3000_NIOS_INIT_REQ_FEC_MODE_NO 0x0 +#define N3000_NIOS_INIT_REQ_FEC_MODE_KR 0x1 +#define N3000_NIOS_INIT_REQ_FEC_MODE_RS 0x2 + +#define N3000_NIOS_FW_VERSION 0x1004 +#define N3000_NIOS_FW_VERSION_PATCH GENMASK(23, 20) +#define N3000_NIOS_FW_VERSION_MINOR GENMASK(27, 24) +#define N3000_NIOS_FW_VERSION_MAJOR GENMASK(31, 28) + +/* The retimers we use on Intel PAC N3000 is Parkvale, abbreviated to PKVL */ +#define N3000_NIOS_PKVL_A_MODE_STS 0x1020 +#define N3000_NIOS_PKVL_B_MODE_STS 0x1024 +#define N3000_NIOS_PKVL_MODE_STS_GROUP_MSK GENMASK(15, 8) +#define N3000_NIOS_PKVL_MODE_STS_GROUP_OK 0x0 +#define N3000_NIOS_PKVL_MODE_STS_ID_MSK GENMASK(7, 0) +/* When GROUP MASK field == GROUP_OK */ +#define N3000_NIOS_PKVL_MODE_ID_RESET 0x0 +#define N3000_NIOS_PKVL_MODE_ID_4X10G 0x1 +#define N3000_NIOS_PKVL_MODE_ID_4X25G 0x2 +#define N3000_NIOS_PKVL_MODE_ID_2X25G 0x3 +#define N3000_NIOS_PKVL_MODE_ID_2X25G_2X10G 0x4 +#define N3000_NIOS_PKVL_MODE_ID_1X25G 0x5 + +#define N3000_NIOS_REGBUS_RETRY_COUNT 10000 /* loop count */ + +#define N3000_NIOS_INIT_TIMEOUT 10000000 /* usec */ +#define N3000_NIOS_INIT_TIME_INTV 100000 /* usec */ + +#define N3000_NIOS_INIT_REQ_FEC_MODE_MSK_ALL \ + (N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK) + +#define N3000_NIOS_INIT_REQ_FEC_MODE_NO_ALL \ + (FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO)) + +#define N3000_NIOS_INIT_REQ_FEC_MODE_KR_ALL \ + (FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR)) + +#define N3000_NIOS_INIT_REQ_FEC_MODE_RS_ALL \ + (FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS)) + +struct n3000_nios { + void __iomem *base; + struct regmap *regmap; + struct device *dev; + struct platform_device *altera_spi; +}; + +static ssize_t nios_fw_version_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct n3000_nios *nn = dev_get_drvdata(dev); + unsigned int val; + int ret; + + ret = regmap_read(nn->regmap, N3000_NIOS_FW_VERSION, &val); + if (ret) + return ret; + + return sysfs_emit(buf, "%x.%x.%x\n", + (u8)FIELD_GET(N3000_NIOS_FW_VERSION_MAJOR, val), + (u8)FIELD_GET(N3000_NIOS_FW_VERSION_MINOR, val), + (u8)FIELD_GET(N3000_NIOS_FW_VERSION_PATCH, val)); +} +static DEVICE_ATTR_RO(nios_fw_version); + +#define IS_MODE_STATUS_OK(mode_stat) \ + (FIELD_GET(N3000_NIOS_PKVL_MODE_STS_GROUP_MSK, (mode_stat)) == \ + N3000_NIOS_PKVL_MODE_STS_GROUP_OK) + +#define IS_RETIMER_FEC_SUPPORTED(retimer_mode) \ + ((retimer_mode) != N3000_NIOS_PKVL_MODE_ID_RESET && \ + (retimer_mode) != N3000_NIOS_PKVL_MODE_ID_4X10G) + +static int get_retimer_mode(struct n3000_nios *nn, unsigned int mode_stat_reg, + unsigned int *retimer_mode) +{ + unsigned int val; + int ret; + + ret = regmap_read(nn->regmap, mode_stat_reg, &val); + if (ret) + return ret; + + if (!IS_MODE_STATUS_OK(val)) + return -EFAULT; + + *retimer_mode = FIELD_GET(N3000_NIOS_PKVL_MODE_STS_ID_MSK, val); + + return 0; +} + +static ssize_t retimer_A_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct n3000_nios *nn = dev_get_drvdata(dev); + unsigned int mode; + int ret; + + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_A_MODE_STS, &mode); + if (ret) + return ret; + + return sysfs_emit(buf, "0x%x\n", mode); +} +static DEVICE_ATTR_RO(retimer_A_mode); + +static ssize_t retimer_B_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct n3000_nios *nn = dev_get_drvdata(dev); + unsigned int mode; + int ret; + + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_B_MODE_STS, &mode); + if (ret) + return ret; + + return sysfs_emit(buf, "0x%x\n", mode); +} +static DEVICE_ATTR_RO(retimer_B_mode); + +static ssize_t fec_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned int val, retimer_a_mode, retimer_b_mode, fec_modes; + struct n3000_nios *nn = dev_get_drvdata(dev); + int ret; + + /* FEC mode setting is not supported in early FW versions */ + ret = regmap_read(nn->regmap, N3000_NIOS_FW_VERSION, &val); + if (ret) + return ret; + + if (FIELD_GET(N3000_NIOS_FW_VERSION_MAJOR, val) < 3) + return sysfs_emit(buf, "not supported\n"); + + /* If no 25G links, FEC mode setting is not supported either */ + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_A_MODE_STS, &retimer_a_mode); + if (ret) + return ret; + + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_B_MODE_STS, &retimer_b_mode); + if (ret) + return ret; + + if (!IS_RETIMER_FEC_SUPPORTED(retimer_a_mode) && + !IS_RETIMER_FEC_SUPPORTED(retimer_b_mode)) + return sysfs_emit(buf, "not supported\n"); + + /* get the valid FEC mode for 25G links */ + ret = regmap_read(nn->regmap, N3000_NIOS_INIT, &val); + if (ret) + return ret; + + /* + * FEC mode should always be the same for all links, as we set them + * in this way. + */ + fec_modes = (val & N3000_NIOS_INIT_REQ_FEC_MODE_MSK_ALL); + if (fec_modes == N3000_NIOS_INIT_REQ_FEC_MODE_NO_ALL) + return sysfs_emit(buf, "no\n"); + else if (fec_modes == N3000_NIOS_INIT_REQ_FEC_MODE_KR_ALL) + return sysfs_emit(buf, "kr\n"); + else if (fec_modes == N3000_NIOS_INIT_REQ_FEC_MODE_RS_ALL) + return sysfs_emit(buf, "rs\n"); + + return -EFAULT; +} +static DEVICE_ATTR_RO(fec_mode); + +static struct attribute *n3000_nios_attrs[] = { + &dev_attr_nios_fw_version.attr, + &dev_attr_retimer_A_mode.attr, + &dev_attr_retimer_B_mode.attr, + &dev_attr_fec_mode.attr, + NULL, +}; +ATTRIBUTE_GROUPS(n3000_nios); + +static int n3000_nios_init_done_check(struct n3000_nios *nn) +{ + unsigned int val, state_a, state_b; + struct device *dev = nn->dev; + int ret, ret2; + + /* + * The SPI is shared by the Nios core inside the FPGA, Nios will use + * this SPI master to do some one time initialization after power up, + * and then release the control to OS. The driver needs to poll on + * INIT_DONE to see when driver could take the control. + * + * Please note that after Nios firmware version 3.0.0, INIT_START is + * introduced, so driver needs to trigger START firstly and then check + * INIT_DONE. + */ + + ret = regmap_read(nn->regmap, N3000_NIOS_FW_VERSION, &val); + if (ret) + return ret; + + /* + * If Nios version register is totally uninitialized(== 0x0), then the + * Nios firmware is missing. So host could take control of SPI master + * safely, but initialization work for Nios is not done. To restore the + * card, we need to reprogram a new Nios firmware via the BMC chip on + * SPI bus. So the driver doesn't error out, it continues to create the + * spi controller device and spi_board_info for BMC. + */ + if (val == 0) { + dev_err(dev, "Nios version reg = 0x%x, skip INIT_DONE check, but the retimer may be uninitialized\n", + val); + return 0; + } + + if (FIELD_GET(N3000_NIOS_FW_VERSION_MAJOR, val) >= 3) { + /* read NIOS_INIT to check if retimer initialization is done */ + ret = regmap_read(nn->regmap, N3000_NIOS_INIT, &val); + if (ret) + return ret; + + /* check if retimers are initialized already */ + if (val & (N3000_NIOS_INIT_DONE | N3000_NIOS_INIT_START)) + goto nios_init_done; + + /* configure FEC mode per module param */ + val = N3000_NIOS_INIT_START; + + /* + * When the retimer is to be set to 10G mode, there is no FEC + * mode setting, so the REQ_FEC_MODE field will be ignored by + * Nios firmware in this case. But we should still fill the FEC + * mode field cause host could not get the retimer working mode + * until the Nios init is done. + * + * For now the driver doesn't support the retimer FEC mode + * switching per user's request. It is always set to Reed + * Solomon FEC. + * + * The driver will set the same FEC mode for all links. + */ + val |= N3000_NIOS_INIT_REQ_FEC_MODE_RS_ALL; + + ret = regmap_write(nn->regmap, N3000_NIOS_INIT, val); + if (ret) + return ret; + } + +nios_init_done: + /* polls on NIOS_INIT_DONE */ + ret = regmap_read_poll_timeout(nn->regmap, N3000_NIOS_INIT, val, + val & N3000_NIOS_INIT_DONE, + N3000_NIOS_INIT_TIME_INTV, + N3000_NIOS_INIT_TIMEOUT); + if (ret) + dev_err(dev, "NIOS_INIT_DONE %s\n", + (ret == -ETIMEDOUT) ? "timed out" : "check error"); + + ret2 = regmap_read(nn->regmap, N3000_NIOS_PKVL_A_MODE_STS, &state_a); + if (ret2) + return ret2; + + ret2 = regmap_read(nn->regmap, N3000_NIOS_PKVL_B_MODE_STS, &state_b); + if (ret2) + return ret2; + + if (!ret) { + /* + * After INIT_DONE is detected, it still needs to check if the + * Nios firmware reports any error during the retimer + * configuration. + */ + if (IS_MODE_STATUS_OK(state_a) && IS_MODE_STATUS_OK(state_b)) + return 0; + + /* + * If the retimer configuration is failed, the Nios firmware + * will still release the spi controller for host to + * communicate with the BMC. It makes possible for people to + * reprogram a new Nios firmware and restore the card. So the + * driver doesn't error out, it continues to create the spi + * controller device and spi_board_info for BMC. + */ + dev_err(dev, "NIOS_INIT_DONE OK, but err on retimer init\n"); + } + + dev_err(nn->dev, "PKVL_A_MODE_STS 0x%x\n", state_a); + dev_err(nn->dev, "PKVL_B_MODE_STS 0x%x\n", state_b); + + return ret; +} + +static struct spi_board_info m10_n3000_info = { + .modalias = "m10-n3000", + .max_speed_hz = 12500000, + .bus_num = 0, + .chip_select = 0, +}; + +static int create_altera_spi_controller(struct n3000_nios *nn) +{ + struct altera_spi_platform_data pdata = { 0 }; + struct platform_device_info pdevinfo = { 0 }; + void __iomem *base = nn->base; + u64 v; + + v = readq(base + N3000_NS_PARAM); + + pdata.mode_bits = SPI_CS_HIGH; + if (FIELD_GET(N3000_NS_PARAM_CLK_POL, v)) + pdata.mode_bits |= SPI_CPOL; + if (FIELD_GET(N3000_NS_PARAM_CLK_PHASE, v)) + pdata.mode_bits |= SPI_CPHA; + + pdata.num_chipselect = FIELD_GET(N3000_NS_PARAM_NUM_CS, v); + pdata.bits_per_word_mask = + SPI_BPW_RANGE_MASK(1, FIELD_GET(N3000_NS_PARAM_DATA_WIDTH, v)); + + pdata.num_devices = 1; + pdata.devices = &m10_n3000_info; + + dev_dbg(nn->dev, "%s cs %u bpm 0x%x mode 0x%x\n", __func__, + pdata.num_chipselect, pdata.bits_per_word_mask, + pdata.mode_bits); + + pdevinfo.name = "subdev_spi_altera"; + pdevinfo.id = PLATFORM_DEVID_AUTO; + pdevinfo.parent = nn->dev; + pdevinfo.data = &pdata; + pdevinfo.size_data = sizeof(pdata); + + nn->altera_spi = platform_device_register_full(&pdevinfo); + return PTR_ERR_OR_ZERO(nn->altera_spi); +} + +static void destroy_altera_spi_controller(struct n3000_nios *nn) +{ + platform_device_unregister(nn->altera_spi); +} + +static int n3000_nios_poll_stat_timeout(void __iomem *base, u64 *v) +{ + int loops; + + /* + * We don't use the time based timeout here for performance. + * + * The regbus read/write is on the critical path of Intel PAC N3000 + * image programing. The time based timeout checking will add too much + * overhead on it. Usually the state changes in 1 or 2 loops on the + * test server, and we set 10000 times loop here for safety. + */ + for (loops = N3000_NIOS_REGBUS_RETRY_COUNT; loops > 0 ; loops--) { + *v = readq(base + N3000_NS_STAT); + if (*v & N3000_NS_STAT_RW_VAL) + break; + cpu_relax(); + } + + return (loops > 0) ? 0 : -ETIMEDOUT; +} + +static int n3000_nios_reg_write(void *context, unsigned int reg, unsigned int val) +{ + struct n3000_nios *nn = context; + u64 v; + int ret; + + v = FIELD_PREP(N3000_NS_CTRL_CMD_MSK, N3000_NS_CTRL_CMD_WR) | + FIELD_PREP(N3000_NS_CTRL_ADDR, reg) | + FIELD_PREP(N3000_NS_CTRL_WR_DATA, val); + writeq(v, nn->base + N3000_NS_CTRL); + + ret = n3000_nios_poll_stat_timeout(nn->base, &v); + if (ret) + dev_err(nn->dev, "fail to write reg 0x%x val 0x%x: %d\n", + reg, val, ret); + + return ret; +} + +static int n3000_nios_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + struct n3000_nios *nn = context; + u64 v; + int ret; + + v = FIELD_PREP(N3000_NS_CTRL_CMD_MSK, N3000_NS_CTRL_CMD_RD) | + FIELD_PREP(N3000_NS_CTRL_ADDR, reg); + writeq(v, nn->base + N3000_NS_CTRL); + + ret = n3000_nios_poll_stat_timeout(nn->base, &v); + if (ret) + dev_err(nn->dev, "fail to read reg 0x%x: %d\n", reg, ret); + else + *val = FIELD_GET(N3000_NS_STAT_RD_DATA, v); + + return ret; +} + +static const struct regmap_config n3000_nios_regbus_cfg = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .fast_io = true, + + .reg_write = n3000_nios_reg_write, + .reg_read = n3000_nios_reg_read, +}; + +static int n3000_nios_probe(struct dfl_device *ddev) +{ + struct device *dev = &ddev->dev; + struct n3000_nios *nn; + int ret; + + nn = devm_kzalloc(dev, sizeof(*nn), GFP_KERNEL); + if (!nn) + return -ENOMEM; + + dev_set_drvdata(&ddev->dev, nn); + + nn->dev = dev; + + nn->base = devm_ioremap_resource(&ddev->dev, &ddev->mmio_res); + if (IS_ERR(nn->base)) + return PTR_ERR(nn->base); + + nn->regmap = devm_regmap_init(dev, NULL, nn, &n3000_nios_regbus_cfg); + if (IS_ERR(nn->regmap)) + return PTR_ERR(nn->regmap); + + ret = n3000_nios_init_done_check(nn); + if (ret) + return ret; + + ret = create_altera_spi_controller(nn); + if (ret) + dev_err(dev, "altera spi controller create failed: %d\n", ret); + + return ret; +} + +static void n3000_nios_remove(struct dfl_device *ddev) +{ + struct n3000_nios *nn = dev_get_drvdata(&ddev->dev); + + destroy_altera_spi_controller(nn); +} + +#define FME_FEATURE_ID_N3000_NIOS 0xd + +static const struct dfl_device_id n3000_nios_ids[] = { + { FME_ID, FME_FEATURE_ID_N3000_NIOS }, + { } +}; +MODULE_DEVICE_TABLE(dfl, n3000_nios_ids); + +static struct dfl_driver n3000_nios_driver = { + .drv = { + .name = "dfl-n3000-nios", + .dev_groups = n3000_nios_groups, + }, + .id_table = n3000_nios_ids, + .probe = n3000_nios_probe, + .remove = n3000_nios_remove, +}; + +module_dfl_driver(n3000_nios_driver); + +MODULE_DESCRIPTION("Driver for Nios private feature on Intel PAC N3000"); +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/fpga/dfl-pci.c b/drivers/fpga/dfl-pci.c index a2203d03c9e2..04e47e266f26 100644 --- a/drivers/fpga/dfl-pci.c +++ b/drivers/fpga/dfl-pci.c @@ -27,6 +27,14 @@ #define DRV_VERSION "0.8" #define DRV_NAME "dfl-pci" +#define PCI_VSEC_ID_INTEL_DFLS 0x43 + +#define PCI_VNDR_DFLS_CNT 0x8 +#define PCI_VNDR_DFLS_RES 0xc + +#define PCI_VNDR_DFLS_RES_BAR_MASK GENMASK(2, 0) +#define PCI_VNDR_DFLS_RES_OFF_MASK GENMASK(31, 3) + struct cci_drvdata { struct dfl_fpga_cdev *cdev; /* container device */ }; @@ -119,49 +127,94 @@ static int *cci_pci_create_irq_table(struct pci_dev *pcidev, unsigned int nvec) return table; } -/* enumerate feature devices under pci device */ -static int cci_enumerate_feature_devs(struct pci_dev *pcidev) +static int find_dfls_by_vsec(struct pci_dev *pcidev, struct dfl_fpga_enum_info *info) { - struct cci_drvdata *drvdata = pci_get_drvdata(pcidev); - int port_num, bar, i, nvec, ret = 0; - struct dfl_fpga_enum_info *info; - struct dfl_fpga_cdev *cdev; + u32 bir, offset, vndr_hdr, dfl_cnt, dfl_res; + int dfl_res_off, i, bars, voff = 0; resource_size_t start, len; - void __iomem *base; - int *irq_table; - u32 offset; - u64 v; - /* allocate enumeration info via pci_dev */ - info = dfl_fpga_enum_info_alloc(&pcidev->dev); - if (!info) - return -ENOMEM; + while ((voff = pci_find_next_ext_capability(pcidev, voff, PCI_EXT_CAP_ID_VNDR))) { + vndr_hdr = 0; + pci_read_config_dword(pcidev, voff + PCI_VNDR_HEADER, &vndr_hdr); - /* add irq info for enumeration if the device support irq */ - nvec = cci_pci_alloc_irq(pcidev); - if (nvec < 0) { - dev_err(&pcidev->dev, "Fail to alloc irq %d.\n", nvec); - ret = nvec; - goto enum_info_free_exit; - } else if (nvec) { - irq_table = cci_pci_create_irq_table(pcidev, nvec); - if (!irq_table) { - ret = -ENOMEM; - goto irq_free_exit; + if (PCI_VNDR_HEADER_ID(vndr_hdr) == PCI_VSEC_ID_INTEL_DFLS && + pcidev->vendor == PCI_VENDOR_ID_INTEL) + break; + } + + if (!voff) { + dev_dbg(&pcidev->dev, "%s no DFL VSEC found\n", __func__); + return -ENODEV; + } + + dfl_cnt = 0; + pci_read_config_dword(pcidev, voff + PCI_VNDR_DFLS_CNT, &dfl_cnt); + if (dfl_cnt > PCI_STD_NUM_BARS) { + dev_err(&pcidev->dev, "%s too many DFLs %d > %d\n", + __func__, dfl_cnt, PCI_STD_NUM_BARS); + return -EINVAL; + } + + dfl_res_off = voff + PCI_VNDR_DFLS_RES; + if (dfl_res_off + (dfl_cnt * sizeof(u32)) > PCI_CFG_SPACE_EXP_SIZE) { + dev_err(&pcidev->dev, "%s DFL VSEC too big for PCIe config space\n", + __func__); + return -EINVAL; + } + + for (i = 0, bars = 0; i < dfl_cnt; i++, dfl_res_off += sizeof(u32)) { + dfl_res = GENMASK(31, 0); + pci_read_config_dword(pcidev, dfl_res_off, &dfl_res); + + bir = dfl_res & PCI_VNDR_DFLS_RES_BAR_MASK; + if (bir >= PCI_STD_NUM_BARS) { + dev_err(&pcidev->dev, "%s bad bir number %d\n", + __func__, bir); + return -EINVAL; } - ret = dfl_fpga_enum_info_add_irq(info, nvec, irq_table); - kfree(irq_table); - if (ret) - goto irq_free_exit; + if (bars & BIT(bir)) { + dev_err(&pcidev->dev, "%s DFL for BAR %d already specified\n", + __func__, bir); + return -EINVAL; + } + + bars |= BIT(bir); + + len = pci_resource_len(pcidev, bir); + offset = dfl_res & PCI_VNDR_DFLS_RES_OFF_MASK; + if (offset >= len) { + dev_err(&pcidev->dev, "%s bad offset %u >= %pa\n", + __func__, offset, &len); + return -EINVAL; + } + + dev_dbg(&pcidev->dev, "%s BAR %d offset 0x%x\n", __func__, bir, offset); + + len -= offset; + + start = pci_resource_start(pcidev, bir) + offset; + + dfl_fpga_enum_info_add_dfl(info, start, len); } - /* start to find Device Feature List in Bar 0 */ + return 0; +} + +/* default method of finding dfls starting at offset 0 of bar 0 */ +static int find_dfls_by_default(struct pci_dev *pcidev, + struct dfl_fpga_enum_info *info) +{ + int port_num, bar, i, ret = 0; + resource_size_t start, len; + void __iomem *base; + u32 offset; + u64 v; + + /* start to find Device Feature List from Bar 0 */ base = cci_pci_ioremap_bar0(pcidev); - if (!base) { - ret = -ENOMEM; - goto irq_free_exit; - } + if (!base) + return -ENOMEM; /* * PF device has FME and Ports/AFUs, and VF device only has one @@ -208,12 +261,54 @@ static int cci_enumerate_feature_devs(struct pci_dev *pcidev) dfl_fpga_enum_info_add_dfl(info, start, len); } else { ret = -ENODEV; - goto irq_free_exit; } /* release I/O mappings for next step enumeration */ pcim_iounmap_regions(pcidev, BIT(0)); + return ret; +} + +/* enumerate feature devices under pci device */ +static int cci_enumerate_feature_devs(struct pci_dev *pcidev) +{ + struct cci_drvdata *drvdata = pci_get_drvdata(pcidev); + struct dfl_fpga_enum_info *info; + struct dfl_fpga_cdev *cdev; + int nvec, ret = 0; + int *irq_table; + + /* allocate enumeration info via pci_dev */ + info = dfl_fpga_enum_info_alloc(&pcidev->dev); + if (!info) + return -ENOMEM; + + /* add irq info for enumeration if the device support irq */ + nvec = cci_pci_alloc_irq(pcidev); + if (nvec < 0) { + dev_err(&pcidev->dev, "Fail to alloc irq %d.\n", nvec); + ret = nvec; + goto enum_info_free_exit; + } else if (nvec) { + irq_table = cci_pci_create_irq_table(pcidev, nvec); + if (!irq_table) { + ret = -ENOMEM; + goto irq_free_exit; + } + + ret = dfl_fpga_enum_info_add_irq(info, nvec, irq_table); + kfree(irq_table); + if (ret) + goto irq_free_exit; + } + + ret = find_dfls_by_vsec(pcidev, info); + if (ret == -ENODEV) + ret = find_dfls_by_default(pcidev, info); + + if (ret) + goto irq_free_exit; + /* start enumeration with prepared enumeration information */ cdev = dfl_fpga_feature_devs_enumerate(info); if (IS_ERR(cdev)) { diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c index b450870b75ed..511b20ff35a3 100644 --- a/drivers/fpga/dfl.c +++ b/drivers/fpga/dfl.c @@ -10,6 +10,7 @@ * Wu Hao <hao.wu@intel.com> * Xiao Guangrong <guangrong.xiao@linux.intel.com> */ +#include <linux/dfl.h> #include <linux/fpga-dfl.h> #include <linux/module.h> #include <linux/uaccess.h> @@ -298,8 +299,7 @@ static int dfl_bus_uevent(struct device *dev, struct kobj_uevent_env *env) { struct dfl_device *ddev = to_dfl_dev(dev); - /* The type has 4 valid bits and feature_id has 12 valid bits */ - return add_uevent_var(env, "MODALIAS=dfl:t%01Xf%03X", + return add_uevent_var(env, "MODALIAS=dfl:t%04Xf%04X", ddev->type, ddev->feature_id); } diff --git a/drivers/fpga/dfl.h b/drivers/fpga/dfl.h index 5dc758f655b7..2b82c96ba56c 100644 --- a/drivers/fpga/dfl.h +++ b/drivers/fpga/dfl.h @@ -22,6 +22,7 @@ #include <linux/interrupt.h> #include <linux/iopoll.h> #include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/mod_devicetable.h> #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/uuid.h> @@ -516,88 +517,4 @@ long dfl_feature_ioctl_set_irq(struct platform_device *pdev, struct dfl_feature *feature, unsigned long arg); -/** - * enum dfl_id_type - define the DFL FIU types - */ -enum dfl_id_type { - FME_ID, - PORT_ID, - DFL_ID_MAX, -}; - -/** - * struct dfl_device_id - dfl device identifier - * @type: contains 4 bits DFL FIU type of the device. See enum dfl_id_type. - * @feature_id: contains 12 bits feature identifier local to its DFL FIU type. - * @driver_data: driver specific data. - */ -struct dfl_device_id { - u8 type; - u16 feature_id; - unsigned long driver_data; -}; - -/** - * struct dfl_device - represent an dfl device on dfl bus - * - * @dev: generic device interface. - * @id: id of the dfl device. - * @type: type of DFL FIU of the device. See enum dfl_id_type. - * @feature_id: 16 bits feature identifier local to its DFL FIU type. - * @mmio_res: mmio resource of this dfl device. - * @irqs: list of Linux IRQ numbers of this dfl device. - * @num_irqs: number of IRQs supported by this dfl device. - * @cdev: pointer to DFL FPGA container device this dfl device belongs to. - * @id_entry: matched id entry in dfl driver's id table. - */ -struct dfl_device { - struct device dev; - int id; - u8 type; - u16 feature_id; - struct resource mmio_res; - int *irqs; - unsigned int num_irqs; - struct dfl_fpga_cdev *cdev; - const struct dfl_device_id *id_entry; -}; - -/** - * struct dfl_driver - represent an dfl device driver - * - * @drv: driver model structure. - * @id_table: pointer to table of device IDs the driver is interested in. - * { } member terminated. - * @probe: mandatory callback for device binding. - * @remove: callback for device unbinding. - */ -struct dfl_driver { - struct device_driver drv; - const struct dfl_device_id *id_table; - - int (*probe)(struct dfl_device *dfl_dev); - void (*remove)(struct dfl_device *dfl_dev); -}; - -#define to_dfl_dev(d) container_of(d, struct dfl_device, dev) -#define to_dfl_drv(d) container_of(d, struct dfl_driver, drv) - -/* - * use a macro to avoid include chaining to get THIS_MODULE. - */ -#define dfl_driver_register(drv) \ - __dfl_driver_register(drv, THIS_MODULE) -int __dfl_driver_register(struct dfl_driver *dfl_drv, struct module *owner); -void dfl_driver_unregister(struct dfl_driver *dfl_drv); - -/* - * module_dfl_driver() - Helper macro for drivers that don't do - * anything special in module init/exit. This eliminates a lot of - * boilerplate. Each module may only use this macro once, and - * calling it replaces module_init() and module_exit(). - */ -#define module_dfl_driver(__dfl_driver) \ - module_driver(__dfl_driver, dfl_driver_register, \ - dfl_driver_unregister) - #endif /* __FPGA_DFL_H */ diff --git a/drivers/fpga/fpga-bridge.c b/drivers/fpga/fpga-bridge.c index 2deccacc3aa7..e9266b2a357f 100644 --- a/drivers/fpga/fpga-bridge.c +++ b/drivers/fpga/fpga-bridge.c @@ -17,7 +17,7 @@ static DEFINE_IDA(fpga_bridge_ida); static struct class *fpga_bridge_class; /* Lock for adding/removing bridges to linked lists*/ -static spinlock_t bridge_list_lock; +static DEFINE_SPINLOCK(bridge_list_lock); /** * fpga_bridge_enable - Enable transactions on the bridge @@ -479,8 +479,6 @@ static void fpga_bridge_dev_release(struct device *dev) static int __init fpga_bridge_dev_init(void) { - spin_lock_init(&bridge_list_lock); - fpga_bridge_class = class_create(THIS_MODULE, "fpga_bridge"); if (IS_ERR(fpga_bridge_class)) return PTR_ERR(fpga_bridge_class); diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index b4a71119a4b0..baf0153b7bca 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -1039,3 +1039,14 @@ void of_gpiochip_remove(struct gpio_chip *chip) { of_node_put(chip->of_node); } + +void of_gpio_dev_init(struct gpio_chip *gc, struct gpio_device *gdev) +{ + /* If the gpiochip has an assigned OF node this takes precedence */ + if (gc->of_node) + gdev->dev.of_node = gc->of_node; + else + gc->of_node = gdev->dev.of_node; + if (gdev->dev.of_node) + gdev->dev.fwnode = of_fwnode_handle(gdev->dev.of_node); +} diff --git a/drivers/gpio/gpiolib-of.h b/drivers/gpio/gpiolib-of.h index ed26664f1537..8af2bc899aab 100644 --- a/drivers/gpio/gpiolib-of.h +++ b/drivers/gpio/gpiolib-of.h @@ -15,6 +15,7 @@ int of_gpiochip_add(struct gpio_chip *gc); void of_gpiochip_remove(struct gpio_chip *gc); int of_gpio_get_count(struct device *dev, const char *con_id); bool of_gpio_need_valid_mask(const struct gpio_chip *gc); +void of_gpio_dev_init(struct gpio_chip *gc, struct gpio_device *gdev); #else static inline struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id, @@ -33,6 +34,10 @@ static inline bool of_gpio_need_valid_mask(const struct gpio_chip *gc) { return false; } +static inline void of_gpio_dev_init(struct gpio_chip *gc, + struct gpio_device *gdev) +{ +} #endif /* CONFIG_OF_GPIO */ extern struct notifier_block gpio_of_notifier; diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 844198cb4e31..adf55db080d8 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -56,8 +56,10 @@ static DEFINE_IDA(gpio_ida); static dev_t gpio_devt; #define GPIO_DEV_MAX 256 /* 256 GPIO chip devices supported */ +static int gpio_bus_match(struct device *dev, struct device_driver *drv); static struct bus_type gpio_bus_type = { .name = "gpio", + .match = gpio_bus_match, }; /* @@ -590,13 +592,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, gdev->dev.of_node = gc->parent->of_node; } -#ifdef CONFIG_OF_GPIO - /* If the gpiochip has an assigned OF node this takes precedence */ - if (gc->of_node) - gdev->dev.of_node = gc->of_node; - else - gc->of_node = gdev->dev.of_node; -#endif + of_gpio_dev_init(gc, gdev); gdev->id = ida_alloc(&gpio_ida, GFP_KERNEL); if (gdev->id < 0) { @@ -4215,6 +4211,41 @@ void gpiod_put_array(struct gpio_descs *descs) } EXPORT_SYMBOL_GPL(gpiod_put_array); + +static int gpio_bus_match(struct device *dev, struct device_driver *drv) +{ + /* + * Only match if the fwnode doesn't already have a proper struct device + * created for it. + */ + if (dev->fwnode && dev->fwnode->dev != dev) + return 0; + return 1; +} + +static int gpio_stub_drv_probe(struct device *dev) +{ + /* + * The DT node of some GPIO chips have a "compatible" property, but + * never have a struct device added and probed by a driver to register + * the GPIO chip with gpiolib. In such cases, fw_devlink=on will cause + * the consumers of the GPIO chip to get probe deferred forever because + * they will be waiting for a device associated with the GPIO chip + * firmware node to get added and bound to a driver. + * + * To allow these consumers to probe, we associate the struct + * gpio_device of the GPIO chip with the firmware node and then simply + * bind it to this stub driver. + */ + return 0; +} + +static struct device_driver gpio_stub_drv = { + .name = "gpio_stub_drv", + .bus = &gpio_bus_type, + .probe = gpio_stub_drv_probe, +}; + static int __init gpiolib_dev_init(void) { int ret; @@ -4226,9 +4257,16 @@ static int __init gpiolib_dev_init(void) return ret; } + if (driver_register(&gpio_stub_drv) < 0) { + pr_err("gpiolib: could not register GPIO stub driver\n"); + bus_unregister(&gpio_bus_type); + return ret; + } + ret = alloc_chrdev_region(&gpio_devt, 0, GPIO_DEV_MAX, GPIOCHIP_NAME); if (ret < 0) { pr_err("gpiolib: failed to allocate char dev region\n"); + driver_unregister(&gpio_stub_drv); bus_unregister(&gpio_bus_type); return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 86452f4f3c82..b6879d97c9c9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1008,6 +1008,12 @@ struct amdgpu_device { bool in_suspend; bool in_hibernate; + /* + * The combination flag in_poweroff_reboot_com used to identify the poweroff + * and reboot opt in the s0i3 system-wide suspend. + */ + bool in_poweroff_reboot_com; + atomic_t in_gpu_reset; enum pp_mp1_state mp1_state; struct rw_semaphore reset_sem; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c index 8155c54392c8..36a741d63ddc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c @@ -903,10 +903,11 @@ void amdgpu_acpi_fini(struct amdgpu_device *adev) */ bool amdgpu_acpi_is_s0ix_supported(struct amdgpu_device *adev) { +#if defined(CONFIG_AMD_PMC) if (acpi_gbl_FADT.flags & ACPI_FADT_LOW_POWER_S0) { if (adev->flags & AMD_IS_APU) return true; } - +#endif return false; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index 0a25fecf488a..43059ead733b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -357,7 +357,7 @@ static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf, while (size) { uint32_t value; - value = RREG32_PCIE(*pos >> 2); + value = RREG32_PCIE(*pos); r = put_user(value, (uint32_t *)buf); if (r) { pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); @@ -424,7 +424,7 @@ static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user return r; } - WREG32_PCIE(*pos >> 2, value); + WREG32_PCIE(*pos, value); result += 4; buf += 4; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 7052dc35d278..6447cd6ca5a8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -2678,7 +2678,8 @@ static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) { int i, r; - if (!amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) { + if (adev->in_poweroff_reboot_com || + !amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) { amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); } @@ -3741,7 +3742,8 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) amdgpu_fence_driver_suspend(adev); - if (!amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) + if (adev->in_poweroff_reboot_com || + !amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) r = amdgpu_device_ip_suspend_phase2(adev); else amdgpu_gfx_state_change_set(adev, sGpuChangeState_D3Entry); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 03c529630c21..4575192d9b08 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -1270,7 +1270,9 @@ amdgpu_pci_shutdown(struct pci_dev *pdev) */ if (!amdgpu_passthrough(adev)) adev->mp1_state = PP_MP1_STATE_UNLOAD; + adev->in_poweroff_reboot_com = true; amdgpu_device_ip_suspend(adev); + adev->in_poweroff_reboot_com = false; adev->mp1_state = PP_MP1_STATE_NONE; } @@ -1312,8 +1314,13 @@ static int amdgpu_pmops_thaw(struct device *dev) static int amdgpu_pmops_poweroff(struct device *dev) { struct drm_device *drm_dev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(drm_dev); + int r; - return amdgpu_device_suspend(drm_dev, true); + adev->in_poweroff_reboot_com = true; + r = amdgpu_device_suspend(drm_dev, true); + adev->in_poweroff_reboot_com = false; + return r; } static int amdgpu_pmops_restore(struct device *dev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 3c37cf1ae8b7..64beb3399604 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -173,8 +173,6 @@ int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags) switch (adev->asic_type) { case CHIP_VEGA20: case CHIP_ARCTURUS: - case CHIP_SIENNA_CICHLID: - case CHIP_NAVY_FLOUNDER: /* enable runpm if runpm=1 */ if (amdgpu_runtime_pm > 0) adev->runpm = true; diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c index 160fa5f59805..c625c5d8ed89 100644 --- a/drivers/gpu/drm/amd/amdgpu/nv.c +++ b/drivers/gpu/drm/amd/amdgpu/nv.c @@ -558,7 +558,8 @@ static bool nv_is_headless_sku(struct pci_dev *pdev) { if ((pdev->device == 0x731E && (pdev->revision == 0xC6 || pdev->revision == 0xC7)) || - (pdev->device == 0x7340 && pdev->revision == 0xC9)) + (pdev->device == 0x7340 && pdev->revision == 0xC9) || + (pdev->device == 0x7360 && pdev->revision == 0xC7)) return true; return false; } @@ -634,7 +635,8 @@ int nv_set_ip_blocks(struct amdgpu_device *adev) if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT && !amdgpu_sriov_vf(adev)) amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); - amdgpu_device_ip_block_add(adev, &vcn_v2_0_ip_block); + if (!nv_is_headless_sku(adev->pdev)) + amdgpu_device_ip_block_add(adev, &vcn_v2_0_ip_block); if (!amdgpu_sriov_vf(adev)) amdgpu_device_ip_block_add(adev, &jpeg_v2_0_ip_block); break; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 94cd5ddd67ef..3e1fd1e7d09f 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -937,7 +937,49 @@ static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_ } #endif +#if defined(CONFIG_DRM_AMD_DC_DCN) +static void event_mall_stutter(struct work_struct *work) +{ + + struct vblank_workqueue *vblank_work = container_of(work, struct vblank_workqueue, mall_work); + struct amdgpu_display_manager *dm = vblank_work->dm; + + mutex_lock(&dm->dc_lock); + + if (vblank_work->enable) + dm->active_vblank_irq_count++; + else + dm->active_vblank_irq_count--; + + + dc_allow_idle_optimizations( + dm->dc, dm->active_vblank_irq_count == 0 ? true : false); + + DRM_DEBUG_DRIVER("Allow idle optimizations (MALL): %d\n", dm->active_vblank_irq_count == 0); + + mutex_unlock(&dm->dc_lock); +} + +static struct vblank_workqueue *vblank_create_workqueue(struct amdgpu_device *adev, struct dc *dc) +{ + + int max_caps = dc->caps.max_links; + struct vblank_workqueue *vblank_work; + int i = 0; + + vblank_work = kcalloc(max_caps, sizeof(*vblank_work), GFP_KERNEL); + if (ZERO_OR_NULL_PTR(vblank_work)) { + kfree(vblank_work); + return NULL; + } + + for (i = 0; i < max_caps; i++) + INIT_WORK(&vblank_work[i].mall_work, event_mall_stutter); + + return vblank_work; +} +#endif static int amdgpu_dm_init(struct amdgpu_device *adev) { struct dc_init_data init_data; @@ -957,6 +999,9 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) mutex_init(&adev->dm.dc_lock); mutex_init(&adev->dm.audio_lock); +#if defined(CONFIG_DRM_AMD_DC_DCN) + spin_lock_init(&adev->dm.vblank_lock); +#endif if(amdgpu_dm_irq_init(adev)) { DRM_ERROR("amdgpu: failed to initialize DM IRQ support.\n"); @@ -1071,6 +1116,17 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) amdgpu_dm_init_color_mod(); +#if defined(CONFIG_DRM_AMD_DC_DCN) + if (adev->dm.dc->caps.max_links > 0) { + adev->dm.vblank_workqueue = vblank_create_workqueue(adev, adev->dm.dc); + + if (!adev->dm.vblank_workqueue) + DRM_ERROR("amdgpu: failed to initialize vblank_workqueue.\n"); + else + DRM_DEBUG_DRIVER("amdgpu: vblank_workqueue init done %p.\n", adev->dm.vblank_workqueue); + } +#endif + #ifdef CONFIG_DRM_AMD_DC_HDCP if (adev->dm.dc->caps.max_links > 0 && adev->asic_type >= CHIP_RAVEN) { adev->dm.hdcp_workqueue = hdcp_create_workqueue(adev, &init_params.cp_psp, adev->dm.dc); @@ -1936,7 +1992,7 @@ static void dm_gpureset_commit_state(struct dc_state *dc_state, dc_commit_updates_for_stream( dm->dc, bundle->surface_updates, dc_state->stream_status->plane_count, - dc_state->streams[k], &bundle->stream_update); + dc_state->streams[k], &bundle->stream_update, dc_state); } cleanup: @@ -1967,7 +2023,8 @@ static void dm_set_dpms_off(struct dc_link *link) stream_update.stream = stream_state; dc_commit_updates_for_stream(stream_state->ctx->dc, NULL, 0, - stream_state, &stream_update); + stream_state, &stream_update, + stream_state->ctx->dc->current_state); mutex_unlock(&adev->dm.dc_lock); } @@ -5374,7 +5431,10 @@ static inline int dm_set_vblank(struct drm_crtc *crtc, bool enable) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); struct amdgpu_device *adev = drm_to_adev(crtc->dev); struct dm_crtc_state *acrtc_state = to_dm_crtc_state(crtc->state); +#if defined(CONFIG_DRM_AMD_DC_DCN) struct amdgpu_display_manager *dm = &adev->dm; + unsigned long flags; +#endif int rc = 0; if (enable) { @@ -5397,22 +5457,15 @@ static inline int dm_set_vblank(struct drm_crtc *crtc, bool enable) if (amdgpu_in_reset(adev)) return 0; - mutex_lock(&dm->dc_lock); - - if (enable) - dm->active_vblank_irq_count++; - else - dm->active_vblank_irq_count--; - #if defined(CONFIG_DRM_AMD_DC_DCN) - dc_allow_idle_optimizations( - adev->dm.dc, dm->active_vblank_irq_count == 0 ? true : false); - - DRM_DEBUG_DRIVER("Allow idle optimizations (MALL): %d\n", dm->active_vblank_irq_count == 0); + spin_lock_irqsave(&dm->vblank_lock, flags); + dm->vblank_workqueue->dm = dm; + dm->vblank_workqueue->otg_inst = acrtc->otg_inst; + dm->vblank_workqueue->enable = enable; + spin_unlock_irqrestore(&dm->vblank_lock, flags); + schedule_work(&dm->vblank_workqueue->mall_work); #endif - mutex_unlock(&dm->dc_lock); - return 0; } @@ -7663,7 +7716,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, struct drm_crtc *pcrtc, bool wait_for_vblank) { - int i; + uint32_t i; uint64_t timestamp_ns; struct drm_plane *plane; struct drm_plane_state *old_plane_state, *new_plane_state; @@ -7704,7 +7757,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, amdgpu_dm_commit_cursors(state); /* update planes when needed */ - for_each_oldnew_plane_in_state_reverse(state, plane, old_plane_state, new_plane_state, i) { + for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) { struct drm_crtc *crtc = new_plane_state->crtc; struct drm_crtc_state *new_crtc_state; struct drm_framebuffer *fb = new_plane_state->fb; @@ -7927,7 +7980,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, bundle->surface_updates, planes_count, acrtc_state->stream, - &bundle->stream_update); + &bundle->stream_update, + dc_state); /** * Enable or disable the interrupts on the backend. @@ -8263,13 +8317,13 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); struct dm_connector_state *dm_old_con_state = to_dm_connector_state(old_con_state); struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - struct dc_surface_update surface_updates[MAX_SURFACES]; + struct dc_surface_update dummy_updates[MAX_SURFACES]; struct dc_stream_update stream_update; struct dc_info_packet hdr_packet; struct dc_stream_status *status = NULL; bool abm_changed, hdr_changed, scaling_changed; - memset(&surface_updates, 0, sizeof(surface_updates)); + memset(&dummy_updates, 0, sizeof(dummy_updates)); memset(&stream_update, 0, sizeof(stream_update)); if (acrtc) { @@ -8326,15 +8380,16 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) * To fix this, DC should permit updating only stream properties. */ for (j = 0; j < status->plane_count; j++) - surface_updates[j].surface = status->plane_states[j]; + dummy_updates[j].surface = status->plane_states[0]; mutex_lock(&dm->dc_lock); dc_commit_updates_for_stream(dm->dc, - surface_updates, + dummy_updates, status->plane_count, dm_new_crtc_state->stream, - &stream_update); + &stream_update, + dc_state); mutex_unlock(&dm->dc_lock); } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index f72930c36c22..8bfe901cf237 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -93,6 +93,20 @@ struct dm_compressor_info { }; /** + * struct vblank_workqueue - Works to be executed in a separate thread during vblank + * @mall_work: work for mall stutter + * @dm: amdgpu display manager device + * @otg_inst: otg instance of which vblank is being set + * @enable: true if enable vblank + */ +struct vblank_workqueue { + struct work_struct mall_work; + struct amdgpu_display_manager *dm; + int otg_inst; + bool enable; +}; + +/** * struct amdgpu_dm_backlight_caps - Information about backlight * * Describe the backlight support for ACPI or eDP AUX. @@ -244,6 +258,15 @@ struct amdgpu_display_manager { struct mutex audio_lock; /** + * @vblank_work_lock: + * + * Guards access to deferred vblank work state. + */ +#if defined(CONFIG_DRM_AMD_DC_DCN) + spinlock_t vblank_lock; +#endif + + /** * @audio_component: * * Used to notify ELD changes to sound driver. @@ -321,6 +344,10 @@ struct amdgpu_display_manager { struct hdcp_workqueue *hdcp_workqueue; #endif +#if defined(CONFIG_DRM_AMD_DC_DCN) + struct vblank_workqueue *vblank_workqueue; +#endif + struct drm_atomic_state *cached_state; struct dc_state *cached_dc_state; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index 5159399f8239..5750818db8f6 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -530,7 +530,7 @@ bool dm_helpers_dp_write_dsc_enable( { uint8_t enable_dsc = enable ? 1 : 0; struct amdgpu_dm_connector *aconnector; - uint8_t ret; + uint8_t ret = 0; if (!stream) return false; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index c9aede2f783d..8f8a13c7cf73 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -2697,7 +2697,8 @@ void dc_commit_updates_for_stream(struct dc *dc, struct dc_surface_update *srf_updates, int surface_count, struct dc_stream_state *stream, - struct dc_stream_update *stream_update) + struct dc_stream_update *stream_update, + struct dc_state *state) { const struct dc_stream_status *stream_status; enum surface_update_type update_type; @@ -2716,12 +2717,6 @@ void dc_commit_updates_for_stream(struct dc *dc, if (update_type >= UPDATE_TYPE_FULL) { - struct dc_plane_state *new_planes[MAX_SURFACES]; - - memset(new_planes, 0, sizeof(new_planes)); - - for (i = 0; i < surface_count; i++) - new_planes[i] = srf_updates[i].surface; /* initialize scratch memory for building context */ context = dc_create_state(dc); @@ -2730,21 +2725,15 @@ void dc_commit_updates_for_stream(struct dc *dc, return; } - dc_resource_state_copy_construct( - dc->current_state, context); + dc_resource_state_copy_construct(state, context); - /*remove old surfaces from context */ - if (!dc_rem_all_planes_for_stream(dc, stream, context)) { - DC_ERROR("Failed to remove streams for new validate context!\n"); - return; - } + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *new_pipe = &context->res_ctx.pipe_ctx[i]; + struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i]; - /* add surface to context */ - if (!dc_add_all_planes_for_stream(dc, stream, new_planes, surface_count, context)) { - DC_ERROR("Failed to add streams for new validate context!\n"); - return; + if (new_pipe->plane_state && new_pipe->plane_state != old_pipe->plane_state) + new_pipe->plane_state->force_full_update = true; } - } diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h index a4f7ec888c67..80b67b860091 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_stream.h +++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h @@ -294,7 +294,8 @@ void dc_commit_updates_for_stream(struct dc *dc, struct dc_surface_update *srf_updates, int surface_count, struct dc_stream_state *stream, - struct dc_stream_update *stream_update); + struct dc_stream_update *stream_update, + struct dc_state *state); /* * Log the current stream state. */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c index 59024653430c..e4701825b5a0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c @@ -480,7 +480,6 @@ unsigned int dcn10_get_dig_frontend(struct link_encoder *enc) break; default: // invalid source select DIG - ASSERT(false); result = ENGINE_ID_UNKNOWN; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c index 9620fb8a27dc..06dc1e2e8383 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c @@ -539,6 +539,8 @@ void dcn30_init_hw(struct dc *dc) fe = dc->links[i]->link_enc->funcs->get_dig_frontend( dc->links[i]->link_enc); + if (fe == ENGINE_ID_UNKNOWN) + continue; for (j = 0; j < dc->res_pool->stream_enc_count; j++) { if (fe == dc->res_pool->stream_enc[j]->id) { diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c b/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c index 1b971265418b..0e0f494fbb5e 100644 --- a/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c +++ b/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c @@ -168,6 +168,11 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = { .ack = NULL }; +static const struct irq_source_info_funcs vupdate_no_lock_irq_info_funcs = { + .set = NULL, + .ack = NULL +}; + #undef BASE_INNER #define BASE_INNER(seg) DMU_BASE__INST0_SEG ## seg @@ -230,6 +235,17 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = { .funcs = &vblank_irq_info_funcs\ } +/* vupdate_no_lock_int_entry maps to DC_IRQ_SOURCE_VUPDATEx, to match semantic + * of DCE's DC_IRQ_SOURCE_VUPDATEx. + */ +#define vupdate_no_lock_int_entry(reg_num)\ + [DC_IRQ_SOURCE_VUPDATE1 + reg_num] = {\ + IRQ_REG_ENTRY(OTG, reg_num,\ + OTG_GLOBAL_SYNC_STATUS, VUPDATE_NO_LOCK_INT_EN,\ + OTG_GLOBAL_SYNC_STATUS, VUPDATE_NO_LOCK_EVENT_CLEAR),\ + .funcs = &vupdate_no_lock_irq_info_funcs\ + } + #define vblank_int_entry(reg_num)\ [DC_IRQ_SOURCE_VBLANK1 + reg_num] = {\ IRQ_REG_ENTRY(OTG, reg_num,\ @@ -338,6 +354,12 @@ irq_source_info_dcn21[DAL_IRQ_SOURCES_NUMBER] = { vupdate_int_entry(3), vupdate_int_entry(4), vupdate_int_entry(5), + vupdate_no_lock_int_entry(0), + vupdate_no_lock_int_entry(1), + vupdate_no_lock_int_entry(2), + vupdate_no_lock_int_entry(3), + vupdate_no_lock_int_entry(4), + vupdate_no_lock_int_entry(5), vblank_int_entry(0), vblank_int_entry(1), vblank_int_entry(2), diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c index 45564a776e9b..9f0d03ae3109 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c @@ -1322,7 +1322,7 @@ static int arcturus_set_power_profile_mode(struct smu_context *smu, CMN2ASIC_MAPPING_WORKLOAD, profile_mode); if (workload_type < 0) { - dev_err(smu->adev->dev, "Unsupported power profile mode %d on arcturus\n", profile_mode); + dev_dbg(smu->adev->dev, "Unsupported power profile mode %d on arcturus\n", profile_mode); return -EINVAL; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c index 90585461a56e..a6211858ead4 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c @@ -78,6 +78,9 @@ MODULE_FIRMWARE("amdgpu/dimgrey_cavefish_smc.bin"); #define PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK 0xC000 #define PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT 0xE +#define mmTHM_BACO_CNTL_ARCT 0xA7 +#define mmTHM_BACO_CNTL_ARCT_BASE_IDX 0 + static int link_width[] = {0, 1, 2, 4, 8, 12, 16}; static int link_speed[] = {25, 50, 80, 160}; @@ -1532,9 +1535,15 @@ int smu_v11_0_baco_set_state(struct smu_context *smu, enum smu_baco_state state) break; default: if (!ras || !ras->supported) { - data = RREG32_SOC15(THM, 0, mmTHM_BACO_CNTL); - data |= 0x80000000; - WREG32_SOC15(THM, 0, mmTHM_BACO_CNTL, data); + if (adev->asic_type == CHIP_ARCTURUS) { + data = RREG32_SOC15(THM, 0, mmTHM_BACO_CNTL_ARCT); + data |= 0x80000000; + WREG32_SOC15(THM, 0, mmTHM_BACO_CNTL_ARCT, data); + } else { + data = RREG32_SOC15(THM, 0, mmTHM_BACO_CNTL); + data |= 0x80000000; + WREG32_SOC15(THM, 0, mmTHM_BACO_CNTL, data); + } ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_EnterBaco, 0, NULL); } else { diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c index 093b01159408..7ddbaecb11c2 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c @@ -810,7 +810,7 @@ static int vangogh_set_power_profile_mode(struct smu_context *smu, long *input, CMN2ASIC_MAPPING_WORKLOAD, profile_mode); if (workload_type < 0) { - dev_err_once(smu->adev->dev, "Unsupported power profile mode %d on VANGOGH\n", + dev_dbg(smu->adev->dev, "Unsupported power profile mode %d on VANGOGH\n", profile_mode); return -EINVAL; } @@ -1685,9 +1685,9 @@ static int vangogh_system_features_control(struct smu_context *smu, bool en) uint32_t feature_mask[2]; int ret = 0; - if (adev->pm.fw_version >= 0x43f1700) + if (adev->pm.fw_version >= 0x43f1700 && !en) ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_RlcPowerNotify, - en ? RLC_STATUS_NORMAL : RLC_STATUS_OFF, NULL); + RLC_STATUS_OFF, NULL); bitmap_zero(feature->enabled, feature->feature_num); bitmap_zero(feature->supported, feature->feature_num); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c index 5faa509f0dba..5493388fcb10 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c @@ -844,7 +844,7 @@ static int renoir_set_power_profile_mode(struct smu_context *smu, long *input, u * TODO: If some case need switch to powersave/default power mode * then can consider enter WORKLOAD_COMPUTE/WORKLOAD_CUSTOM for power saving. */ - dev_err_once(smu->adev->dev, "Unsupported power profile mode %d on RENOIR\n", profile_mode); + dev_dbg(smu->adev->dev, "Unsupported power profile mode %d on RENOIR\n", profile_mode); return -EINVAL; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index bb620fdd4cd2..bcedd4d92e35 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -762,7 +762,7 @@ void smu_cmn_init_soft_gpu_metrics(void *table, uint8_t frev, uint8_t crev) structure_size = sizeof(struct gpu_metrics_v2_0); break; default: - break; + return; } #undef METRICS_VERSION diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c index 6b116bfd747c..7efbccffc2ea 100644 --- a/drivers/gpu/drm/drm_file.c +++ b/drivers/gpu/drm/drm_file.c @@ -775,20 +775,19 @@ void drm_event_cancel_free(struct drm_device *dev, EXPORT_SYMBOL(drm_event_cancel_free); /** - * drm_send_event_locked - send DRM event to file descriptor + * drm_send_event_helper - send DRM event to file descriptor * @dev: DRM device * @e: DRM event to deliver + * @timestamp: timestamp to set for the fence event in kernel's CLOCK_MONOTONIC + * time domain * - * This function sends the event @e, initialized with drm_event_reserve_init(), - * to its associated userspace DRM file. Callers must already hold - * &drm_device.event_lock, see drm_send_event() for the unlocked version. - * - * Note that the core will take care of unlinking and disarming events when the - * corresponding DRM file is closed. Drivers need not worry about whether the - * DRM file for this event still exists and can call this function upon - * completion of the asynchronous work unconditionally. + * This helper function sends the event @e, initialized with + * drm_event_reserve_init(), to its associated userspace DRM file. + * The timestamp variant of dma_fence_signal is used when the caller + * sends a valid timestamp. */ -void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) +void drm_send_event_helper(struct drm_device *dev, + struct drm_pending_event *e, ktime_t timestamp) { assert_spin_locked(&dev->event_lock); @@ -799,7 +798,10 @@ void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) } if (e->fence) { - dma_fence_signal(e->fence); + if (timestamp) + dma_fence_signal_timestamp(e->fence, timestamp); + else + dma_fence_signal(e->fence); dma_fence_put(e->fence); } @@ -814,6 +816,48 @@ void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) wake_up_interruptible_poll(&e->file_priv->event_wait, EPOLLIN | EPOLLRDNORM); } + +/** + * drm_send_event_timestamp_locked - send DRM event to file descriptor + * @dev: DRM device + * @e: DRM event to deliver + * @timestamp: timestamp to set for the fence event in kernel's CLOCK_MONOTONIC + * time domain + * + * This function sends the event @e, initialized with drm_event_reserve_init(), + * to its associated userspace DRM file. Callers must already hold + * &drm_device.event_lock. + * + * Note that the core will take care of unlinking and disarming events when the + * corresponding DRM file is closed. Drivers need not worry about whether the + * DRM file for this event still exists and can call this function upon + * completion of the asynchronous work unconditionally. + */ +void drm_send_event_timestamp_locked(struct drm_device *dev, + struct drm_pending_event *e, ktime_t timestamp) +{ + drm_send_event_helper(dev, e, timestamp); +} +EXPORT_SYMBOL(drm_send_event_timestamp_locked); + +/** + * drm_send_event_locked - send DRM event to file descriptor + * @dev: DRM device + * @e: DRM event to deliver + * + * This function sends the event @e, initialized with drm_event_reserve_init(), + * to its associated userspace DRM file. Callers must already hold + * &drm_device.event_lock, see drm_send_event() for the unlocked version. + * + * Note that the core will take care of unlinking and disarming events when the + * corresponding DRM file is closed. Drivers need not worry about whether the + * DRM file for this event still exists and can call this function upon + * completion of the asynchronous work unconditionally. + */ +void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) +{ + drm_send_event_helper(dev, e, 0); +} EXPORT_SYMBOL(drm_send_event_locked); /** @@ -836,7 +880,7 @@ void drm_send_event(struct drm_device *dev, struct drm_pending_event *e) unsigned long irqflags; spin_lock_irqsave(&dev->event_lock, irqflags); - drm_send_event_locked(dev, e); + drm_send_event_helper(dev, e, 0); spin_unlock_irqrestore(&dev->event_lock, irqflags); } EXPORT_SYMBOL(drm_send_event); diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c index 30912d8f82a5..893165eeddf3 100644 --- a/drivers/gpu/drm/drm_vblank.c +++ b/drivers/gpu/drm/drm_vblank.c @@ -1006,7 +1006,14 @@ static void send_vblank_event(struct drm_device *dev, break; } trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq); - drm_send_event_locked(dev, &e->base); + /* + * Use the same timestamp for any associated fence signal to avoid + * mismatch in timestamps for vsync & fence events triggered by the + * same HW event. Frameworks like SurfaceFlinger in Android expects the + * retire-fence timestamp to match exactly with HW vsync as it uses it + * for its software vsync modeling. + */ + drm_send_event_timestamp_locked(dev, &e->base, now); } /** diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 32bd1fdffffe..2385a7505f5d 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -21,7 +21,6 @@ subdir-ccflags-y += $(call cc-disable-warning, unused-but-set-variable) subdir-ccflags-y += $(call cc-disable-warning, sign-compare) subdir-ccflags-y += $(call cc-disable-warning, sometimes-uninitialized) subdir-ccflags-y += $(call cc-disable-warning, initializer-overrides) -subdir-ccflags-y += $(call cc-disable-warning, uninitialized) subdir-ccflags-y += $(call cc-disable-warning, frame-address) subdir-ccflags-$(CONFIG_DRM_I915_WERROR) += -Werror @@ -298,7 +297,7 @@ obj-$(CONFIG_DRM_I915_GVT_KVMGT) += gvt/kvmgt.o no-header-test := \ display/intel_vbt_defs.h -extra-$(CONFIG_DRM_I915_WERROR) += \ +always-$(CONFIG_DRM_I915_WERROR) += \ $(patsubst %.h,%.hdrtest, $(filter-out $(no-header-test), \ $(shell cd $(srctree)/$(src) && find * -name '*.h'))) diff --git a/drivers/gpu/drm/i915/display/intel_crtc.c b/drivers/gpu/drm/i915/display/intel_crtc.c index 57b0a3ebe908..8e77ca7ddf11 100644 --- a/drivers/gpu/drm/i915/display/intel_crtc.c +++ b/drivers/gpu/drm/i915/display/intel_crtc.c @@ -109,7 +109,6 @@ void intel_crtc_state_reset(struct intel_crtc_state *crtc_state, crtc_state->cpu_transcoder = INVALID_TRANSCODER; crtc_state->master_transcoder = INVALID_TRANSCODER; crtc_state->hsw_workaround_pipe = INVALID_PIPE; - crtc_state->output_format = INTEL_OUTPUT_FORMAT_INVALID; crtc_state->scaler_state.scaler_id = -1; crtc_state->mst_master_transcoder = INVALID_TRANSCODER; } diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 7ea1e5b487b6..8d7aaa68c6f6 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -10211,7 +10211,6 @@ static void snprintf_output_types(char *buf, size_t len, } static const char * const output_format_str[] = { - [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid", [INTEL_OUTPUT_FORMAT_RGB] = "RGB", [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0", [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4", @@ -10220,7 +10219,7 @@ static const char * const output_format_str[] = { static const char *output_formats(enum intel_output_format format) { if (format >= ARRAY_SIZE(output_format_str)) - format = INTEL_OUTPUT_FORMAT_INVALID; + return "invalid"; return output_format_str[format]; } diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index 39397748b4b0..184ecbbcec99 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -830,7 +830,6 @@ struct intel_crtc_wm_state { }; enum intel_output_format { - INTEL_OUTPUT_FORMAT_INVALID, INTEL_OUTPUT_FORMAT_RGB, INTEL_OUTPUT_FORMAT_YCBCR420, INTEL_OUTPUT_FORMAT_YCBCR444, diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index d54ea0e4681d..fef1e857cefc 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -41,6 +41,7 @@ #include "gt/intel_lrc.h" #include "gt/intel_ring.h" #include "gt/intel_gt_requests.h" +#include "gt/shmem_utils.h" #include "gvt.h" #include "i915_pvinfo.h" #include "trace.h" @@ -3094,71 +3095,28 @@ int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) */ void intel_gvt_update_reg_whitelist(struct intel_vgpu *vgpu) { + const unsigned long start = LRC_STATE_PN * PAGE_SIZE; struct intel_gvt *gvt = vgpu->gvt; - struct drm_i915_private *dev_priv = gvt->gt->i915; struct intel_engine_cs *engine; enum intel_engine_id id; - const unsigned long start = LRC_STATE_PN * PAGE_SIZE; - struct i915_request *rq; - struct intel_vgpu_submission *s = &vgpu->submission; - struct i915_request *requests[I915_NUM_ENGINES] = {}; - bool is_ctx_pinned[I915_NUM_ENGINES] = {}; - int ret = 0; if (gvt->is_reg_whitelist_updated) return; - for_each_engine(engine, &dev_priv->gt, id) { - ret = intel_context_pin(s->shadow[id]); - if (ret) { - gvt_vgpu_err("fail to pin shadow ctx\n"); - goto out; - } - is_ctx_pinned[id] = true; - - rq = i915_request_create(s->shadow[id]); - if (IS_ERR(rq)) { - gvt_vgpu_err("fail to alloc default request\n"); - ret = -EIO; - goto out; - } - requests[id] = i915_request_get(rq); - i915_request_add(rq); - } - - if (intel_gt_wait_for_idle(&dev_priv->gt, - I915_GEM_IDLE_TIMEOUT) == -ETIME) { - ret = -EIO; - goto out; - } - /* scan init ctx to update cmd accessible list */ - for_each_engine(engine, &dev_priv->gt, id) { - int size = engine->context_size - PAGE_SIZE; - void *vaddr; + for_each_engine(engine, gvt->gt, id) { struct parser_exec_state s; - struct drm_i915_gem_object *obj; - struct i915_request *rq; - - rq = requests[id]; - GEM_BUG_ON(!i915_request_completed(rq)); - GEM_BUG_ON(!intel_context_is_pinned(rq->context)); - obj = rq->context->state->obj; - - if (!obj) { - ret = -EIO; - goto out; - } + void *vaddr; + int ret; - i915_gem_object_set_cache_coherency(obj, - I915_CACHE_LLC); + if (!engine->default_state) + continue; - vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); + vaddr = shmem_pin_map(engine->default_state); if (IS_ERR(vaddr)) { - gvt_err("failed to pin init ctx obj, ring=%d, err=%lx\n", - id, PTR_ERR(vaddr)); - ret = PTR_ERR(vaddr); - goto out; + gvt_err("failed to map %s->default state, err:%zd\n", + engine->name, PTR_ERR(vaddr)); + return; } s.buf_type = RING_BUFFER_CTX; @@ -3166,9 +3124,9 @@ void intel_gvt_update_reg_whitelist(struct intel_vgpu *vgpu) s.vgpu = vgpu; s.engine = engine; s.ring_start = 0; - s.ring_size = size; + s.ring_size = engine->context_size - start; s.ring_head = 0; - s.ring_tail = size; + s.ring_tail = s.ring_size; s.rb_va = vaddr + start; s.workload = NULL; s.is_ctx_wa = false; @@ -3176,29 +3134,18 @@ void intel_gvt_update_reg_whitelist(struct intel_vgpu *vgpu) /* skipping the first RING_CTX_SIZE(0x50) dwords */ ret = ip_gma_set(&s, RING_CTX_SIZE); - if (ret) { - i915_gem_object_unpin_map(obj); - goto out; + if (ret == 0) { + ret = command_scan(&s, 0, s.ring_size, 0, s.ring_size); + if (ret) + gvt_err("Scan init ctx error\n"); } - ret = command_scan(&s, 0, size, 0, size); + shmem_unpin_map(engine->default_state, vaddr); if (ret) - gvt_err("Scan init ctx error\n"); - - i915_gem_object_unpin_map(obj); + return; } -out: - if (!ret) - gvt->is_reg_whitelist_updated = true; - - for (id = 0; id < I915_NUM_ENGINES ; id++) { - if (requests[id]) - i915_request_put(requests[id]); - - if (is_ctx_pinned[id]) - intel_context_unpin(s->shadow[id]); - } + gvt->is_reg_whitelist_updated = true; } int intel_gvt_scan_engine_context(struct intel_vgpu_workload *workload) diff --git a/drivers/gpu/drm/i915/gvt/execlist.c b/drivers/gpu/drm/i915/gvt/execlist.c index 158873f269b1..c8dcda6d4f0d 100644 --- a/drivers/gpu/drm/i915/gvt/execlist.c +++ b/drivers/gpu/drm/i915/gvt/execlist.c @@ -522,12 +522,11 @@ static void init_vgpu_execlist(struct intel_vgpu *vgpu, static void clean_execlist(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask) { - struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915; - struct intel_engine_cs *engine; struct intel_vgpu_submission *s = &vgpu->submission; + struct intel_engine_cs *engine; intel_engine_mask_t tmp; - for_each_engine_masked(engine, &dev_priv->gt, engine_mask, tmp) { + for_each_engine_masked(engine, vgpu->gvt->gt, engine_mask, tmp) { kfree(s->ring_scan_buffer[engine->id]); s->ring_scan_buffer[engine->id] = NULL; s->ring_scan_buffer_size[engine->id] = 0; @@ -537,11 +536,10 @@ static void clean_execlist(struct intel_vgpu *vgpu, static void reset_execlist(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask) { - struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915; struct intel_engine_cs *engine; intel_engine_mask_t tmp; - for_each_engine_masked(engine, &dev_priv->gt, engine_mask, tmp) + for_each_engine_masked(engine, vgpu->gvt->gt, engine_mask, tmp) init_vgpu_execlist(vgpu, engine); } diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index 43f31c2eab14..fc735692f21f 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -412,7 +412,9 @@ static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) if (!wa_ctx->indirect_ctx.obj) return; + i915_gem_object_lock(wa_ctx->indirect_ctx.obj, NULL); i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj); + i915_gem_object_unlock(wa_ctx->indirect_ctx.obj); i915_gem_object_put(wa_ctx->indirect_ctx.obj); wa_ctx->indirect_ctx.obj = NULL; @@ -520,6 +522,7 @@ static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload) struct intel_gvt *gvt = workload->vgpu->gvt; const int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd; struct intel_vgpu_shadow_bb *bb; + struct i915_gem_ww_ctx ww; int ret; list_for_each_entry(bb, &workload->shadow_bb, list) { @@ -544,10 +547,19 @@ static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload) * directly */ if (!bb->ppgtt) { - bb->vma = i915_gem_object_ggtt_pin(bb->obj, - NULL, 0, 0, 0); + i915_gem_ww_ctx_init(&ww, false); +retry: + i915_gem_object_lock(bb->obj, &ww); + + bb->vma = i915_gem_object_ggtt_pin_ww(bb->obj, &ww, + NULL, 0, 0, 0); if (IS_ERR(bb->vma)) { ret = PTR_ERR(bb->vma); + if (ret == -EDEADLK) { + ret = i915_gem_ww_ctx_backoff(&ww); + if (!ret) + goto retry; + } goto err; } @@ -561,13 +573,15 @@ static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload) 0); if (ret) goto err; - } - /* No one is going to touch shadow bb from now on. */ - i915_gem_object_flush_map(bb->obj); + /* No one is going to touch shadow bb from now on. */ + i915_gem_object_flush_map(bb->obj); + i915_gem_object_unlock(bb->obj); + } } return 0; err: + i915_gem_ww_ctx_fini(&ww); release_shadow_batch_buffer(workload); return ret; } @@ -594,14 +608,29 @@ static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) unsigned char *per_ctx_va = (unsigned char *)wa_ctx->indirect_ctx.shadow_va + wa_ctx->indirect_ctx.size; + struct i915_gem_ww_ctx ww; + int ret; if (wa_ctx->indirect_ctx.size == 0) return 0; - vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL, - 0, CACHELINE_BYTES, 0); - if (IS_ERR(vma)) - return PTR_ERR(vma); + i915_gem_ww_ctx_init(&ww, false); +retry: + i915_gem_object_lock(wa_ctx->indirect_ctx.obj, &ww); + + vma = i915_gem_object_ggtt_pin_ww(wa_ctx->indirect_ctx.obj, &ww, NULL, + 0, CACHELINE_BYTES, 0); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + if (ret == -EDEADLK) { + ret = i915_gem_ww_ctx_backoff(&ww); + if (!ret) + goto retry; + } + return ret; + } + + i915_gem_object_unlock(wa_ctx->indirect_ctx.obj); /* FIXME: we are not tracking our pinned VMA leaving it * up to the core to fix up the stray pin_count upon @@ -635,12 +664,14 @@ static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload) list_for_each_entry_safe(bb, pos, &workload->shadow_bb, list) { if (bb->obj) { + i915_gem_object_lock(bb->obj, NULL); if (bb->va && !IS_ERR(bb->va)) i915_gem_object_unpin_map(bb->obj); if (bb->vma && !IS_ERR(bb->vma)) i915_vma_unpin(bb->vma); + i915_gem_object_unlock(bb->obj); i915_gem_object_put(bb->obj); } list_del(&bb->list); @@ -1015,13 +1046,12 @@ void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask) { struct intel_vgpu_submission *s = &vgpu->submission; - struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915; struct intel_engine_cs *engine; struct intel_vgpu_workload *pos, *n; intel_engine_mask_t tmp; /* free the unsubmited workloads in the queues. */ - for_each_engine_masked(engine, &dev_priv->gt, engine_mask, tmp) { + for_each_engine_masked(engine, vgpu->gvt->gt, engine_mask, tmp) { list_for_each_entry_safe(pos, n, &s->workload_q_head[engine->id], list) { list_del_init(&pos->list); diff --git a/drivers/gpu/drm/msm/adreno/a5xx.xml.h b/drivers/gpu/drm/msm/adreno/a5xx.xml.h index 346cc6ff3a36..7b9fcfe95c04 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx.xml.h +++ b/drivers/gpu/drm/msm/adreno/a5xx.xml.h @@ -2367,6 +2367,8 @@ static inline uint32_t A5XX_VSC_RESOLVE_CNTL_Y(uint32_t val) #define REG_A5XX_UCHE_ADDR_MODE_CNTL 0x00000e80 +#define REG_A5XX_UCHE_MODE_CNTL 0x00000e81 + #define REG_A5XX_UCHE_SVM_CNTL 0x00000e82 #define REG_A5XX_UCHE_WRITE_THRU_BASE_LO 0x00000e87 diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c index a5af223eaf50..7e553d3efeb2 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c @@ -222,7 +222,7 @@ static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) a5xx_preempt_trigger(gpu); } -static const struct { +static const struct adreno_five_hwcg_regs { u32 offset; u32 value; } a5xx_hwcg[] = { @@ -318,16 +318,124 @@ static const struct { {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222} +}, a50x_hwcg[] = { + {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, + {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, + {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, + {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00FFFFF4}, + {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, + {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, + {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, + {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, + {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, + {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, + {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, + {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, + {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, +}, a512_hwcg[] = { + {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, + {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, + {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, + {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, + {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, + {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, + {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, + {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, + {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, + {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, + {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, + {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, + {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, + {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, + {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, + {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, + {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, + {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, + {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, + {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, + {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, + {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, + {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, + {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, + {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, }; void a5xx_set_hwcg(struct msm_gpu *gpu, bool state) { struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); - unsigned int i; + const struct adreno_five_hwcg_regs *regs; + unsigned int i, sz; + + if (adreno_is_a508(adreno_gpu)) { + regs = a50x_hwcg; + sz = ARRAY_SIZE(a50x_hwcg); + } else if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu)) { + regs = a512_hwcg; + sz = ARRAY_SIZE(a512_hwcg); + } else { + regs = a5xx_hwcg; + sz = ARRAY_SIZE(a5xx_hwcg); + } - for (i = 0; i < ARRAY_SIZE(a5xx_hwcg); i++) - gpu_write(gpu, a5xx_hwcg[i].offset, - state ? a5xx_hwcg[i].value : 0); + for (i = 0; i < sz; i++) + gpu_write(gpu, regs[i].offset, + state ? regs[i].value : 0); if (adreno_is_a540(adreno_gpu)) { gpu_write(gpu, REG_A5XX_RBBM_CLOCK_DELAY_GPMU, state ? 0x00000770 : 0); @@ -538,11 +646,13 @@ static int a5xx_hw_init(struct msm_gpu *gpu) { struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); + u32 regbit; int ret; gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003); - if (adreno_is_a540(adreno_gpu)) + if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) || + adreno_is_a540(adreno_gpu)) gpu_write(gpu, REG_A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009); /* Make all blocks contribute to the GPU BUSY perf counter */ @@ -604,29 +714,48 @@ static int a5xx_hw_init(struct msm_gpu *gpu) 0x00100000 + adreno_gpu->gmem - 1); gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000); - if (adreno_is_a510(adreno_gpu)) { + if (adreno_is_a508(adreno_gpu) || adreno_is_a510(adreno_gpu)) { gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x20); - gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x20); + if (adreno_is_a508(adreno_gpu)) + gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400); + else + gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x20); gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030); gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A); - gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, - (0x200 << 11 | 0x200 << 22)); } else { gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40); if (adreno_is_a530(adreno_gpu)) gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40); - if (adreno_is_a540(adreno_gpu)) + else gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400); gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060); gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16); + } + + if (adreno_is_a508(adreno_gpu)) + gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, + (0x100 << 11 | 0x100 << 22)); + else if (adreno_is_a509(adreno_gpu) || adreno_is_a510(adreno_gpu) || + adreno_is_a512(adreno_gpu)) + gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, + (0x200 << 11 | 0x200 << 22)); + else gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, (0x400 << 11 | 0x300 << 22)); - } if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI) gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8)); - gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0xc0200100); + /* + * Disable the RB sampler datapath DP2 clock gating optimization + * for 1-SP GPUs, as it is enabled by default. + */ + if (adreno_is_a508(adreno_gpu) || adreno_is_a509(adreno_gpu) || + adreno_is_a512(adreno_gpu)) + gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, 0, (1 << 9)); + + /* Disable UCHE global filter as SP can invalidate/flush independently */ + gpu_write(gpu, REG_A5XX_UCHE_MODE_CNTL, BIT(29)); /* Enable USE_RETENTION_FLOPS */ gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000); @@ -653,10 +782,20 @@ static int a5xx_hw_init(struct msm_gpu *gpu) gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F); /* Set the highest bank bit */ - gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, 2 << 7); - gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, 2 << 1); if (adreno_is_a540(adreno_gpu)) - gpu_write(gpu, REG_A5XX_UCHE_DBG_ECO_CNTL_2, 2); + regbit = 2; + else + regbit = 1; + + gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, regbit << 7); + gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, regbit << 1); + + if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) || + adreno_is_a540(adreno_gpu)) + gpu_write(gpu, REG_A5XX_UCHE_DBG_ECO_CNTL_2, regbit); + + /* Disable All flat shading optimization (ALLFLATOPTDIS) */ + gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, (1 << 10)); /* Protect registers from the CP */ gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007); @@ -688,12 +827,14 @@ static int a5xx_hw_init(struct msm_gpu *gpu) /* VPC */ gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8)); - gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 4)); + gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 16)); /* UCHE */ gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16)); - if (adreno_is_a530(adreno_gpu) || adreno_is_a510(adreno_gpu)) + if (adreno_is_a508(adreno_gpu) || adreno_is_a509(adreno_gpu) || + adreno_is_a510(adreno_gpu) || adreno_is_a512(adreno_gpu) || + adreno_is_a530(adreno_gpu)) gpu_write(gpu, REG_A5XX_CP_PROTECT(17), ADRENO_PROTECT_RW(0x10000, 0x8000)); @@ -735,7 +876,8 @@ static int a5xx_hw_init(struct msm_gpu *gpu) if (ret) return ret; - if (!adreno_is_a510(adreno_gpu)) + if (!(adreno_is_a508(adreno_gpu) || adreno_is_a509(adreno_gpu) || + adreno_is_a510(adreno_gpu) || adreno_is_a512(adreno_gpu))) a5xx_gpmu_ucode_init(gpu); ret = a5xx_ucode_init(gpu); @@ -1168,7 +1310,8 @@ static int a5xx_pm_resume(struct msm_gpu *gpu) if (ret) return ret; - if (adreno_is_a510(adreno_gpu)) { + /* Adreno 508, 509, 510, 512 needs manual RBBM sus/res control */ + if (!(adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))) { /* Halt the sp_input_clk at HM level */ gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0x00000055); a5xx_set_hwcg(gpu, true); @@ -1210,8 +1353,8 @@ static int a5xx_pm_suspend(struct msm_gpu *gpu) u32 mask = 0xf; int i, ret; - /* A510 has 3 XIN ports in VBIF */ - if (adreno_is_a510(adreno_gpu)) + /* A508, A510 have 3 XIN ports in VBIF */ + if (adreno_is_a508(adreno_gpu) || adreno_is_a510(adreno_gpu)) mask = 0x7; /* Clear the VBIF pipe before shutting down */ @@ -1223,10 +1366,12 @@ static int a5xx_pm_suspend(struct msm_gpu *gpu) /* * Reset the VBIF before power collapse to avoid issue with FIFO - * entries + * entries on Adreno A510 and A530 (the others will tend to lock up) */ - gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000); - gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000); + if (adreno_is_a510(adreno_gpu) || adreno_is_a530(adreno_gpu)) { + gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000); + gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000); + } ret = msm_gpu_pm_suspend(gpu); if (ret) diff --git a/drivers/gpu/drm/msm/adreno/a5xx_power.c b/drivers/gpu/drm/msm/adreno/a5xx_power.c index f176a6f3eff6..5ccc9da455a1 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx_power.c +++ b/drivers/gpu/drm/msm/adreno/a5xx_power.c @@ -298,7 +298,7 @@ int a5xx_power_init(struct msm_gpu *gpu) int ret; /* Not all A5xx chips have a GPMU */ - if (adreno_is_a510(adreno_gpu)) + if (!(adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))) return 0; /* Set up the limits management */ @@ -330,7 +330,7 @@ void a5xx_gpmu_ucode_init(struct msm_gpu *gpu) unsigned int *data, *ptr, *cmds; unsigned int cmds_size; - if (adreno_is_a510(adreno_gpu)) + if (!(adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))) return; if (a5xx_gpu->gpmu_bo) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index 05e0ef58fe32..71c917f909af 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -245,37 +245,66 @@ static int a6xx_gmu_hfi_start(struct a6xx_gmu *gmu) return ret; } +struct a6xx_gmu_oob_bits { + int set, ack, set_new, ack_new; + const char *name; +}; + +/* These are the interrupt / ack bits for each OOB request that are set + * in a6xx_gmu_set_oob and a6xx_clear_oob + */ +static const struct a6xx_gmu_oob_bits a6xx_gmu_oob_bits[] = { + [GMU_OOB_GPU_SET] = { + .name = "GPU_SET", + .set = 16, + .ack = 24, + .set_new = 30, + .ack_new = 31, + }, + + [GMU_OOB_PERFCOUNTER_SET] = { + .name = "PERFCOUNTER", + .set = 17, + .ack = 25, + .set_new = 28, + .ack_new = 30, + }, + + [GMU_OOB_BOOT_SLUMBER] = { + .name = "BOOT_SLUMBER", + .set = 22, + .ack = 30, + }, + + [GMU_OOB_DCVS_SET] = { + .name = "GPU_DCVS", + .set = 23, + .ack = 31, + }, +}; + /* Trigger a OOB (out of band) request to the GMU */ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) { int ret; u32 val; int request, ack; - const char *name; - switch (state) { - case GMU_OOB_GPU_SET: - if (gmu->legacy) { - request = GMU_OOB_GPU_SET_REQUEST; - ack = GMU_OOB_GPU_SET_ACK; - } else { - request = GMU_OOB_GPU_SET_REQUEST_NEW; - ack = GMU_OOB_GPU_SET_ACK_NEW; - } - name = "GPU_SET"; - break; - case GMU_OOB_BOOT_SLUMBER: - request = GMU_OOB_BOOT_SLUMBER_REQUEST; - ack = GMU_OOB_BOOT_SLUMBER_ACK; - name = "BOOT_SLUMBER"; - break; - case GMU_OOB_DCVS_SET: - request = GMU_OOB_DCVS_REQUEST; - ack = GMU_OOB_DCVS_ACK; - name = "GPU_DCVS"; - break; - default: + if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits)) return -EINVAL; + + if (gmu->legacy) { + request = a6xx_gmu_oob_bits[state].set; + ack = a6xx_gmu_oob_bits[state].ack; + } else { + request = a6xx_gmu_oob_bits[state].set_new; + ack = a6xx_gmu_oob_bits[state].ack_new; + if (!request || !ack) { + DRM_DEV_ERROR(gmu->dev, + "Invalid non-legacy GMU request %s\n", + a6xx_gmu_oob_bits[state].name); + return -EINVAL; + } } /* Trigger the equested OOB operation */ @@ -288,7 +317,7 @@ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) if (ret) DRM_DEV_ERROR(gmu->dev, "Timeout waiting for GMU OOB set %s: 0x%x\n", - name, + a6xx_gmu_oob_bits[state].name, gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO)); /* Clear the acknowledge interrupt */ @@ -300,27 +329,17 @@ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) /* Clear a pending OOB state in the GMU */ void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) { - if (!gmu->legacy) { - WARN_ON(state != GMU_OOB_GPU_SET); - gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, - 1 << GMU_OOB_GPU_SET_CLEAR_NEW); + int bit; + + if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits)) return; - } - switch (state) { - case GMU_OOB_GPU_SET: - gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, - 1 << GMU_OOB_GPU_SET_CLEAR); - break; - case GMU_OOB_BOOT_SLUMBER: - gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, - 1 << GMU_OOB_BOOT_SLUMBER_CLEAR); - break; - case GMU_OOB_DCVS_SET: - gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, - 1 << GMU_OOB_DCVS_CLEAR); - break; - } + if (gmu->legacy) + bit = a6xx_gmu_oob_bits[state].ack; + else + bit = a6xx_gmu_oob_bits[state].ack_new; + + gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, bit); } /* Enable CPU control of SPTP power power collapse */ diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h index c6d2bced8e5d..71dfa60070cc 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h @@ -153,44 +153,27 @@ static inline void gmu_write_rscc(struct a6xx_gmu *gmu, u32 offset, u32 value) */ enum a6xx_gmu_oob_state { + /* + * Let the GMU know that a boot or slumber operation has started. The value in + * REG_A6XX_GMU_BOOT_SLUMBER_OPTION lets the GMU know which operation we are + * doing + */ GMU_OOB_BOOT_SLUMBER = 0, + /* + * Let the GMU know to not turn off any GPU registers while the CPU is in a + * critical section + */ GMU_OOB_GPU_SET, + /* + * Set a new power level for the GPU when the CPU is doing frequency scaling + */ GMU_OOB_DCVS_SET, + /* + * Used to keep the GPU on for CPU-side reads of performance counters. + */ + GMU_OOB_PERFCOUNTER_SET, }; -/* These are the interrupt / ack bits for each OOB request that are set - * in a6xx_gmu_set_oob and a6xx_clear_oob - */ - -/* - * Let the GMU know that a boot or slumber operation has started. The value in - * REG_A6XX_GMU_BOOT_SLUMBER_OPTION lets the GMU know which operation we are - * doing - */ -#define GMU_OOB_BOOT_SLUMBER_REQUEST 22 -#define GMU_OOB_BOOT_SLUMBER_ACK 30 -#define GMU_OOB_BOOT_SLUMBER_CLEAR 30 - -/* - * Set a new power level for the GPU when the CPU is doing frequency scaling - */ -#define GMU_OOB_DCVS_REQUEST 23 -#define GMU_OOB_DCVS_ACK 31 -#define GMU_OOB_DCVS_CLEAR 31 - -/* - * Let the GMU know to not turn off any GPU registers while the CPU is in a - * critical section - */ -#define GMU_OOB_GPU_SET_REQUEST 16 -#define GMU_OOB_GPU_SET_ACK 24 -#define GMU_OOB_GPU_SET_CLEAR 24 - -#define GMU_OOB_GPU_SET_REQUEST_NEW 30 -#define GMU_OOB_GPU_SET_ACK_NEW 31 -#define GMU_OOB_GPU_SET_CLEAR_NEW 31 - - void a6xx_hfi_init(struct a6xx_gmu *gmu); int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state); void a6xx_hfi_stop(struct a6xx_gmu *gmu); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index 130661898546..ba8e9d3cf0fe 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -10,6 +10,7 @@ #include <linux/bitfield.h> #include <linux/devfreq.h> +#include <linux/nvmem-consumer.h> #include <linux/soc/qcom/llcc-qcom.h> #define GPU_PAS_ID 13 @@ -1117,7 +1118,7 @@ static void a6xx_llc_slices_init(struct platform_device *pdev, a6xx_gpu->llc_slice = llcc_slice_getd(LLCC_GPU); a6xx_gpu->htw_llc_slice = llcc_slice_getd(LLCC_GPUHTW); - if (IS_ERR(a6xx_gpu->llc_slice) && IS_ERR(a6xx_gpu->htw_llc_slice)) + if (IS_ERR_OR_NULL(a6xx_gpu->llc_slice) && IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice)) a6xx_gpu->llc_mmio = ERR_PTR(-EINVAL); } @@ -1169,14 +1170,18 @@ static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value) { struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + static DEFINE_MUTEX(perfcounter_oob); + + mutex_lock(&perfcounter_oob); /* Force the GPU power on so we can read this register */ - a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); + a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET); *value = gpu_read64(gpu, REG_A6XX_RBBM_PERFCTR_CP_0_LO, REG_A6XX_RBBM_PERFCTR_CP_0_HI); - a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); + a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET); + mutex_unlock(&perfcounter_oob); return 0; } @@ -1208,6 +1213,10 @@ static void a6xx_destroy(struct msm_gpu *gpu) a6xx_gmu_remove(a6xx_gpu); adreno_gpu_cleanup(adreno_gpu); + + if (a6xx_gpu->opp_table) + dev_pm_opp_put_supported_hw(a6xx_gpu->opp_table); + kfree(a6xx_gpu); } @@ -1240,6 +1249,50 @@ static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu) } static struct msm_gem_address_space * +a6xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + struct iommu_domain *iommu; + struct msm_mmu *mmu; + struct msm_gem_address_space *aspace; + u64 start, size; + + iommu = iommu_domain_alloc(&platform_bus_type); + if (!iommu) + return NULL; + + /* + * This allows GPU to set the bus attributes required to use system + * cache on behalf of the iommu page table walker. + */ + if (!IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice)) + adreno_set_llc_attributes(iommu); + + mmu = msm_iommu_new(&pdev->dev, iommu); + if (IS_ERR(mmu)) { + iommu_domain_free(iommu); + return ERR_CAST(mmu); + } + + /* + * Use the aperture start or SZ_16M, whichever is greater. This will + * ensure that we align with the allocated pagetable range while still + * allowing room in the lower 32 bits for GMEM and whatnot + */ + start = max_t(u64, SZ_16M, iommu->geometry.aperture_start); + size = iommu->geometry.aperture_end - start + 1; + + aspace = msm_gem_address_space_create(mmu, "gpu", + start & GENMASK_ULL(48, 0), size); + + if (IS_ERR(aspace) && !IS_ERR(mmu)) + mmu->funcs->destroy(mmu); + + return aspace; +} + +static struct msm_gem_address_space * a6xx_create_private_address_space(struct msm_gpu *gpu) { struct msm_mmu *mmu; @@ -1264,6 +1317,78 @@ static uint32_t a6xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) return ring->memptrs->rptr = gpu_read(gpu, REG_A6XX_CP_RB_RPTR); } +static u32 a618_get_speed_bin(u32 fuse) +{ + if (fuse == 0) + return 0; + else if (fuse == 169) + return 1; + else if (fuse == 174) + return 2; + + return UINT_MAX; +} + +static u32 fuse_to_supp_hw(struct device *dev, u32 revn, u32 fuse) +{ + u32 val = UINT_MAX; + + if (revn == 618) + val = a618_get_speed_bin(fuse); + + if (val == UINT_MAX) { + DRM_DEV_ERROR(dev, + "missing support for speed-bin: %u. Some OPPs may not be supported by hardware", + fuse); + return UINT_MAX; + } + + return (1 << val); +} + +static int a6xx_set_supported_hw(struct device *dev, struct a6xx_gpu *a6xx_gpu, + u32 revn) +{ + struct opp_table *opp_table; + struct nvmem_cell *cell; + u32 supp_hw = UINT_MAX; + void *buf; + + cell = nvmem_cell_get(dev, "speed_bin"); + /* + * -ENOENT means that the platform doesn't support speedbin which is + * fine + */ + if (PTR_ERR(cell) == -ENOENT) + return 0; + else if (IS_ERR(cell)) { + DRM_DEV_ERROR(dev, + "failed to read speed-bin. Some OPPs may not be supported by hardware"); + goto done; + } + + buf = nvmem_cell_read(cell, NULL); + if (IS_ERR(buf)) { + nvmem_cell_put(cell); + DRM_DEV_ERROR(dev, + "failed to read speed-bin. Some OPPs may not be supported by hardware"); + goto done; + } + + supp_hw = fuse_to_supp_hw(dev, revn, *((u32 *) buf)); + + kfree(buf); + nvmem_cell_put(cell); + +done: + opp_table = dev_pm_opp_set_supported_hw(dev, &supp_hw, 1); + if (IS_ERR(opp_table)) + return PTR_ERR(opp_table); + + a6xx_gpu->opp_table = opp_table; + return 0; +} + static const struct adreno_gpu_funcs funcs = { .base = { .get_param = adreno_get_param, @@ -1285,7 +1410,7 @@ static const struct adreno_gpu_funcs funcs = { .gpu_state_get = a6xx_gpu_state_get, .gpu_state_put = a6xx_gpu_state_put, #endif - .create_address_space = adreno_iommu_create_address_space, + .create_address_space = a6xx_create_address_space, .create_private_address_space = a6xx_create_private_address_space, .get_rptr = a6xx_get_rptr, }, @@ -1325,6 +1450,12 @@ struct msm_gpu *a6xx_gpu_init(struct drm_device *dev) a6xx_llc_slices_init(pdev, a6xx_gpu); + ret = a6xx_set_supported_hw(&pdev->dev, a6xx_gpu, info->revn); + if (ret) { + a6xx_destroy(&(a6xx_gpu->base.base)); + return ERR_PTR(ret); + } + ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1); if (ret) { a6xx_destroy(&(a6xx_gpu->base.base)); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h index e793d329e77b..ce0610c5256f 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h @@ -33,6 +33,8 @@ struct a6xx_gpu { void *llc_slice; void *htw_llc_slice; bool have_mmu500; + + struct opp_table *opp_table; }; #define to_a6xx_gpu(x) container_of(x, struct a6xx_gpu, base) diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c index 12e75ba360f9..600d445fabe8 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_device.c +++ b/drivers/gpu/drm/msm/adreno/adreno_device.c @@ -134,6 +134,41 @@ static const struct adreno_info gpulist[] = { .inactive_period = DRM_MSM_INACTIVE_PERIOD, .init = a4xx_gpu_init, }, { + .rev = ADRENO_REV(5, 0, 8, ANY_ID), + .revn = 508, + .name = "A508", + .fw = { + [ADRENO_FW_PM4] = "a530_pm4.fw", + [ADRENO_FW_PFP] = "a530_pfp.fw", + }, + .gmem = (SZ_128K + SZ_8K), + /* + * Increase inactive period to 250 to avoid bouncing + * the GDSC which appears to make it grumpy + */ + .inactive_period = 250, + .quirks = ADRENO_QUIRK_LMLOADKILL_DISABLE, + .init = a5xx_gpu_init, + .zapfw = "a508_zap.mdt", + }, { + .rev = ADRENO_REV(5, 0, 9, ANY_ID), + .revn = 509, + .name = "A509", + .fw = { + [ADRENO_FW_PM4] = "a530_pm4.fw", + [ADRENO_FW_PFP] = "a530_pfp.fw", + }, + .gmem = (SZ_256K + SZ_16K), + /* + * Increase inactive period to 250 to avoid bouncing + * the GDSC which appears to make it grumpy + */ + .inactive_period = 250, + .quirks = ADRENO_QUIRK_LMLOADKILL_DISABLE, + .init = a5xx_gpu_init, + /* Adreno 509 uses the same ZAP as 512 */ + .zapfw = "a512_zap.mdt", + }, { .rev = ADRENO_REV(5, 1, 0, ANY_ID), .revn = 510, .name = "A510", @@ -149,6 +184,23 @@ static const struct adreno_info gpulist[] = { .inactive_period = 250, .init = a5xx_gpu_init, }, { + .rev = ADRENO_REV(5, 1, 2, ANY_ID), + .revn = 512, + .name = "A512", + .fw = { + [ADRENO_FW_PM4] = "a530_pm4.fw", + [ADRENO_FW_PFP] = "a530_pfp.fw", + }, + .gmem = (SZ_256K + SZ_16K), + /* + * Increase inactive period to 250 to avoid bouncing + * the GDSC which appears to make it grumpy + */ + .inactive_period = 250, + .quirks = ADRENO_QUIRK_LMLOADKILL_DISABLE, + .init = a5xx_gpu_init, + .zapfw = "a512_zap.mdt", + }, { .rev = ADRENO_REV(5, 3, 0, 2), .revn = 530, .name = "A530", @@ -168,7 +220,7 @@ static const struct adreno_info gpulist[] = { .init = a5xx_gpu_init, .zapfw = "a530_zap.mdt", }, { - .rev = ADRENO_REV(5, 4, 0, 2), + .rev = ADRENO_REV(5, 4, 0, ANY_ID), .revn = 540, .name = "A540", .fw = { diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index f09175698827..0f184c3dd9d9 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -186,11 +186,18 @@ int adreno_zap_shader_load(struct msm_gpu *gpu, u32 pasid) return zap_shader_load_mdt(gpu, adreno_gpu->info->zapfw, pasid); } +void adreno_set_llc_attributes(struct iommu_domain *iommu) +{ + struct io_pgtable_domain_attr pgtbl_cfg; + + pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_ARM_OUTER_WBWA; + iommu_domain_set_attr(iommu, DOMAIN_ATTR_IO_PGTABLE_CFG, &pgtbl_cfg); +} + struct msm_gem_address_space * adreno_iommu_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev) { - struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); struct iommu_domain *iommu; struct msm_mmu *mmu; struct msm_gem_address_space *aspace; @@ -200,20 +207,6 @@ adreno_iommu_create_address_space(struct msm_gpu *gpu, if (!iommu) return NULL; - - if (adreno_is_a6xx(adreno_gpu)) { - struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); - struct io_pgtable_domain_attr pgtbl_cfg; - /* - * This allows GPU to set the bus attributes required to use system - * cache on behalf of the iommu page table walker. - */ - if (!IS_ERR(a6xx_gpu->htw_llc_slice)) { - pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_ARM_OUTER_WBWA; - iommu_domain_set_attr(iommu, DOMAIN_ATTR_IO_PGTABLE_CFG, &pgtbl_cfg); - } - } - mmu = msm_iommu_new(&pdev->dev, iommu); if (IS_ERR(mmu)) { iommu_domain_free(iommu); diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h index b3d9a333591b..ccac275aa7a2 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h @@ -197,11 +197,26 @@ static inline int adreno_is_a430(struct adreno_gpu *gpu) return gpu->revn == 430; } +static inline int adreno_is_a508(struct adreno_gpu *gpu) +{ + return gpu->revn == 508; +} + +static inline int adreno_is_a509(struct adreno_gpu *gpu) +{ + return gpu->revn == 509; +} + static inline int adreno_is_a510(struct adreno_gpu *gpu) { return gpu->revn == 510; } +static inline int adreno_is_a512(struct adreno_gpu *gpu) +{ + return gpu->revn == 512; +} + static inline int adreno_is_a530(struct adreno_gpu *gpu) { return gpu->revn == 530; @@ -212,11 +227,6 @@ static inline int adreno_is_a540(struct adreno_gpu *gpu) return gpu->revn == 540; } -static inline bool adreno_is_a6xx(struct adreno_gpu *gpu) -{ - return ((gpu->revn < 700 && gpu->revn > 599)); -} - static inline int adreno_is_a618(struct adreno_gpu *gpu) { return gpu->revn == 618; @@ -278,6 +288,8 @@ struct msm_gem_address_space * adreno_iommu_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev); +void adreno_set_llc_attributes(struct iommu_domain *iommu); + /* * For a5xx and a6xx targets load the zap shader that is used to pull the GPU * out of secure mode diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c index 5a056c1191df..b2be39b9144e 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c @@ -4,8 +4,10 @@ */ #define pr_fmt(fmt) "[drm:%s:%d] " fmt, __func__, __LINE__ +#include <linux/delay.h> #include "dpu_encoder_phys.h" #include "dpu_hw_interrupts.h" +#include "dpu_hw_pingpong.h" #include "dpu_core_irq.h" #include "dpu_formats.h" #include "dpu_trace.h" @@ -35,6 +37,8 @@ #define DPU_ENC_WR_PTR_START_TIMEOUT_US 20000 +#define DPU_ENC_MAX_POLL_TIMEOUT_US 2000 + static bool dpu_encoder_phys_cmd_is_master(struct dpu_encoder_phys *phys_enc) { return (phys_enc->split_role != ENC_ROLE_SLAVE) ? true : false; @@ -368,15 +372,12 @@ static void dpu_encoder_phys_cmd_tearcheck_config( tc_cfg.vsync_count = vsync_hz / (mode->vtotal * drm_mode_vrefresh(mode)); - /* enable external TE after kickoff to avoid premature autorefresh */ - tc_cfg.hw_vsync_mode = 0; - /* - * By setting sync_cfg_height to near max register value, we essentially - * disable dpu hw generated TE signal, since hw TE will arrive first. - * Only caveat is if due to error, we hit wrap-around. + * Set the sync_cfg_height to twice vtotal so that if we lose a + * TE event coming from the display TE pin we won't stall immediately */ - tc_cfg.sync_cfg_height = 0xFFF0; + tc_cfg.hw_vsync_mode = 1; + tc_cfg.sync_cfg_height = mode->vtotal * 2; tc_cfg.vsync_init_val = mode->vdisplay; tc_cfg.sync_threshold_start = DEFAULT_TEARCHECK_SYNC_THRESH_START; tc_cfg.sync_threshold_continue = DEFAULT_TEARCHECK_SYNC_THRESH_CONTINUE; @@ -580,6 +581,69 @@ static void dpu_encoder_phys_cmd_prepare_for_kickoff( atomic_read(&phys_enc->pending_kickoff_cnt)); } +static bool dpu_encoder_phys_cmd_is_ongoing_pptx( + struct dpu_encoder_phys *phys_enc) +{ + struct dpu_hw_pp_vsync_info info; + + if (!phys_enc) + return false; + + phys_enc->hw_pp->ops.get_vsync_info(phys_enc->hw_pp, &info); + if (info.wr_ptr_line_count > 0 && + info.wr_ptr_line_count < phys_enc->cached_mode.vdisplay) + return true; + + return false; +} + +static void dpu_encoder_phys_cmd_prepare_commit( + struct dpu_encoder_phys *phys_enc) +{ + struct dpu_encoder_phys_cmd *cmd_enc = + to_dpu_encoder_phys_cmd(phys_enc); + int trial = 0; + + if (!phys_enc) + return; + if (!phys_enc->hw_pp) + return; + if (!dpu_encoder_phys_cmd_is_master(phys_enc)) + return; + + /* If autorefresh is already disabled, we have nothing to do */ + if (!phys_enc->hw_pp->ops.get_autorefresh(phys_enc->hw_pp, NULL)) + return; + + /* + * If autorefresh is enabled, disable it and make sure it is safe to + * proceed with current frame commit/push. Sequence fallowed is, + * 1. Disable TE + * 2. Disable autorefresh config + * 4. Poll for frame transfer ongoing to be false + * 5. Enable TE back + */ + _dpu_encoder_phys_cmd_connect_te(phys_enc, false); + phys_enc->hw_pp->ops.setup_autorefresh(phys_enc->hw_pp, 0, false); + + do { + udelay(DPU_ENC_MAX_POLL_TIMEOUT_US); + if ((trial * DPU_ENC_MAX_POLL_TIMEOUT_US) + > (KICKOFF_TIMEOUT_MS * USEC_PER_MSEC)) { + DPU_ERROR_CMDENC(cmd_enc, + "disable autorefresh failed\n"); + break; + } + + trial++; + } while (dpu_encoder_phys_cmd_is_ongoing_pptx(phys_enc)); + + _dpu_encoder_phys_cmd_connect_te(phys_enc, true); + + DPU_DEBUG_CMDENC(to_dpu_encoder_phys_cmd(phys_enc), + "disabled autorefresh\n"); +} + static int _dpu_encoder_phys_cmd_wait_for_ctl_start( struct dpu_encoder_phys *phys_enc) { @@ -621,20 +685,15 @@ static int dpu_encoder_phys_cmd_wait_for_tx_complete( static int dpu_encoder_phys_cmd_wait_for_commit_done( struct dpu_encoder_phys *phys_enc) { - int rc = 0; struct dpu_encoder_phys_cmd *cmd_enc; cmd_enc = to_dpu_encoder_phys_cmd(phys_enc); /* only required for master controller */ - if (dpu_encoder_phys_cmd_is_master(phys_enc)) - rc = _dpu_encoder_phys_cmd_wait_for_ctl_start(phys_enc); - - /* required for both controllers */ - if (!rc && cmd_enc->serialize_wait4pp) - dpu_encoder_phys_cmd_prepare_for_kickoff(phys_enc); + if (!dpu_encoder_phys_cmd_is_master(phys_enc)) + return 0; - return rc; + return _dpu_encoder_phys_cmd_wait_for_ctl_start(phys_enc); } static int dpu_encoder_phys_cmd_wait_for_vblank( @@ -681,6 +740,7 @@ static void dpu_encoder_phys_cmd_trigger_start( static void dpu_encoder_phys_cmd_init_ops( struct dpu_encoder_phys_ops *ops) { + ops->prepare_commit = dpu_encoder_phys_cmd_prepare_commit; ops->is_master = dpu_encoder_phys_cmd_is_master; ops->mode_set = dpu_encoder_phys_cmd_mode_set; ops->mode_fixup = dpu_encoder_phys_cmd_mode_fixup; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c index 90393fe9e59c..189f3533525c 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c @@ -12,14 +12,17 @@ #define VIG_MASK \ (BIT(DPU_SSPP_SRC) | BIT(DPU_SSPP_QOS) |\ - BIT(DPU_SSPP_CSC_10BIT) | BIT(DPU_SSPP_CDP) | BIT(DPU_SSPP_QOS_8LVL) |\ + BIT(DPU_SSPP_CSC_10BIT) | BIT(DPU_SSPP_CDP) |\ BIT(DPU_SSPP_TS_PREFILL) | BIT(DPU_SSPP_EXCL_RECT)) #define VIG_SDM845_MASK \ - (VIG_MASK | BIT(DPU_SSPP_SCALER_QSEED3)) + (VIG_MASK | BIT(DPU_SSPP_QOS_8LVL) | BIT(DPU_SSPP_SCALER_QSEED3)) #define VIG_SC7180_MASK \ - (VIG_MASK | BIT(DPU_SSPP_SCALER_QSEED4)) + (VIG_MASK | BIT(DPU_SSPP_QOS_8LVL) | BIT(DPU_SSPP_SCALER_QSEED4)) + +#define VIG_SM8250_MASK \ + (VIG_MASK | BIT(DPU_SSPP_SCALER_QSEED3LITE)) #define DMA_SDM845_MASK \ (BIT(DPU_SSPP_SRC) | BIT(DPU_SSPP_QOS) | BIT(DPU_SSPP_QOS_8LVL) |\ @@ -185,7 +188,7 @@ static const struct dpu_caps sm8150_dpu_caps = { static const struct dpu_caps sm8250_dpu_caps = { .max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH, .max_mixer_blendstages = 0xb, - .qseed_type = DPU_SSPP_SCALER_QSEED3, /* TODO: qseed3 lite */ + .qseed_type = DPU_SSPP_SCALER_QSEED3LITE, .smart_dma_rev = DPU_SSPP_SMART_DMA_V2, /* TODO: v2.5 */ .ubwc_version = DPU_HW_UBWC_VER_40, .has_src_split = true, @@ -444,6 +447,34 @@ static const struct dpu_sspp_cfg sc7180_sspp[] = { sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_CURSOR1), }; +static const struct dpu_sspp_sub_blks sm8250_vig_sblk_0 = + _VIG_SBLK("0", 5, DPU_SSPP_SCALER_QSEED3LITE); +static const struct dpu_sspp_sub_blks sm8250_vig_sblk_1 = + _VIG_SBLK("1", 6, DPU_SSPP_SCALER_QSEED3LITE); +static const struct dpu_sspp_sub_blks sm8250_vig_sblk_2 = + _VIG_SBLK("2", 7, DPU_SSPP_SCALER_QSEED3LITE); +static const struct dpu_sspp_sub_blks sm8250_vig_sblk_3 = + _VIG_SBLK("3", 8, DPU_SSPP_SCALER_QSEED3LITE); + +static const struct dpu_sspp_cfg sm8250_sspp[] = { + SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, VIG_SM8250_MASK, + sm8250_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0), + SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, VIG_SM8250_MASK, + sm8250_vig_sblk_1, 4, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1), + SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, VIG_SM8250_MASK, + sm8250_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2), + SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, VIG_SM8250_MASK, + sm8250_vig_sblk_3, 12, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3), + SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, DMA_SDM845_MASK, + sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0), + SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, DMA_SDM845_MASK, + sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1), + SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, DMA_CURSOR_SDM845_MASK, + sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_CURSOR0), + SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000, DMA_CURSOR_SDM845_MASK, + sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_CURSOR1), +}; + /************************************************************* * MIXER sub blocks config *************************************************************/ @@ -532,23 +563,28 @@ static const struct dpu_dspp_sub_blks sm8150_dspp_sblk = { .len = 0x90, .version = 0x40000}, }; -#define DSPP_BLK(_name, _id, _base, _sblk) \ +#define DSPP_BLK(_name, _id, _base, _mask, _sblk) \ {\ .name = _name, .id = _id, \ .base = _base, .len = 0x1800, \ - .features = DSPP_SC7180_MASK, \ + .features = _mask, \ .sblk = _sblk \ } static const struct dpu_dspp_cfg sc7180_dspp[] = { - DSPP_BLK("dspp_0", DSPP_0, 0x54000, &sc7180_dspp_sblk), + DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK, + &sc7180_dspp_sblk), }; static const struct dpu_dspp_cfg sm8150_dspp[] = { - DSPP_BLK("dspp_0", DSPP_0, 0x54000, &sm8150_dspp_sblk), - DSPP_BLK("dspp_1", DSPP_1, 0x56000, &sm8150_dspp_sblk), - DSPP_BLK("dspp_2", DSPP_2, 0x58000, &sm8150_dspp_sblk), - DSPP_BLK("dspp_3", DSPP_3, 0x5a000, &sm8150_dspp_sblk), + DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK, + &sm8150_dspp_sblk), + DSPP_BLK("dspp_1", DSPP_1, 0x56000, DSPP_SC7180_MASK, + &sm8150_dspp_sblk), + DSPP_BLK("dspp_2", DSPP_2, 0x58000, DSPP_SC7180_MASK, + &sm8150_dspp_sblk), + DSPP_BLK("dspp_3", DSPP_3, 0x5a000, DSPP_SC7180_MASK, + &sm8150_dspp_sblk), }; /************************************************************* @@ -624,33 +660,33 @@ static const struct dpu_merge_3d_cfg sm8150_merge_3d[] = { /************************************************************* * INTF sub blocks config *************************************************************/ -#define INTF_BLK(_name, _id, _base, _type, _ctrl_id, _features) \ +#define INTF_BLK(_name, _id, _base, _type, _ctrl_id, _progfetch, _features) \ {\ .name = _name, .id = _id, \ .base = _base, .len = 0x280, \ .features = _features, \ .type = _type, \ .controller_id = _ctrl_id, \ - .prog_fetch_lines_worst_case = 24 \ + .prog_fetch_lines_worst_case = _progfetch \ } static const struct dpu_intf_cfg sdm845_intf[] = { - INTF_BLK("intf_0", INTF_0, 0x6A000, INTF_DP, 0, INTF_SDM845_MASK), - INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0, INTF_SDM845_MASK), - INTF_BLK("intf_2", INTF_2, 0x6B000, INTF_DSI, 1, INTF_SDM845_MASK), - INTF_BLK("intf_3", INTF_3, 0x6B800, INTF_DP, 1, INTF_SDM845_MASK), + INTF_BLK("intf_0", INTF_0, 0x6A000, INTF_DP, 0, 24, INTF_SDM845_MASK), + INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0, 24, INTF_SDM845_MASK), + INTF_BLK("intf_2", INTF_2, 0x6B000, INTF_DSI, 1, 24, INTF_SDM845_MASK), + INTF_BLK("intf_3", INTF_3, 0x6B800, INTF_DP, 1, 24, INTF_SDM845_MASK), }; static const struct dpu_intf_cfg sc7180_intf[] = { - INTF_BLK("intf_0", INTF_0, 0x6A000, INTF_DP, 0, INTF_SC7180_MASK), - INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0, INTF_SC7180_MASK), + INTF_BLK("intf_0", INTF_0, 0x6A000, INTF_DP, 0, 24, INTF_SC7180_MASK), + INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0, 24, INTF_SC7180_MASK), }; static const struct dpu_intf_cfg sm8150_intf[] = { - INTF_BLK("intf_0", INTF_0, 0x6A000, INTF_DP, 0, INTF_SC7180_MASK), - INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0, INTF_SC7180_MASK), - INTF_BLK("intf_2", INTF_2, 0x6B000, INTF_DSI, 1, INTF_SC7180_MASK), - INTF_BLK("intf_3", INTF_3, 0x6B800, INTF_DP, 1, INTF_SC7180_MASK), + INTF_BLK("intf_0", INTF_0, 0x6A000, INTF_DP, 0, 24, INTF_SC7180_MASK), + INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0, 24, INTF_SC7180_MASK), + INTF_BLK("intf_2", INTF_2, 0x6B000, INTF_DSI, 1, 24, INTF_SC7180_MASK), + INTF_BLK("intf_3", INTF_3, 0x6B800, INTF_DP, 1, 24, INTF_SC7180_MASK), }; /************************************************************* @@ -969,9 +1005,8 @@ static void sm8250_cfg_init(struct dpu_mdss_cfg *dpu_cfg) .mdp = sm8250_mdp, .ctl_count = ARRAY_SIZE(sm8150_ctl), .ctl = sm8150_ctl, - /* TODO: sspp qseed version differs from 845 */ - .sspp_count = ARRAY_SIZE(sdm845_sspp), - .sspp = sdm845_sspp, + .sspp_count = ARRAY_SIZE(sm8250_sspp), + .sspp = sm8250_sspp, .mixer_count = ARRAY_SIZE(sm8150_lm), .mixer = sm8150_lm, .dspp_count = ARRAY_SIZE(sm8150_dspp), diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h index eaef99db2d2f..ea4647d21a20 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h @@ -95,6 +95,7 @@ enum { * @DPU_SSPP_SRC Src and fetch part of the pipes, * @DPU_SSPP_SCALER_QSEED2, QSEED2 algorithm support * @DPU_SSPP_SCALER_QSEED3, QSEED3 alogorithm support + * @DPU_SSPP_SCALER_QSEED3LITE, QSEED3 Lite alogorithm support * @DPU_SSPP_SCALER_QSEED4, QSEED4 algorithm support * @DPU_SSPP_SCALER_RGB, RGB Scaler, supported by RGB pipes * @DPU_SSPP_CSC, Support of Color space converion @@ -114,6 +115,7 @@ enum { DPU_SSPP_SRC = 0x1, DPU_SSPP_SCALER_QSEED2, DPU_SSPP_SCALER_QSEED3, + DPU_SSPP_SCALER_QSEED3LITE, DPU_SSPP_SCALER_QSEED4, DPU_SSPP_SCALER_RGB, DPU_SSPP_CSC, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.c index bea4ab5c58c5..245a7a62b5c6 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.c @@ -23,6 +23,7 @@ #define PP_WR_PTR_IRQ 0x024 #define PP_OUT_LINE_COUNT 0x028 #define PP_LINE_COUNT 0x02C +#define PP_AUTOREFRESH_CONFIG 0x030 #define PP_FBC_MODE 0x034 #define PP_FBC_BUDGET_CTL 0x038 @@ -120,6 +121,29 @@ static int dpu_hw_pp_setup_te_config(struct dpu_hw_pingpong *pp, return 0; } +static void dpu_hw_pp_setup_autorefresh_config(struct dpu_hw_pingpong *pp, + u32 frame_count, bool enable) +{ + DPU_REG_WRITE(&pp->hw, PP_AUTOREFRESH_CONFIG, + enable ? (BIT(31) | frame_count) : 0); +} + +/* + * dpu_hw_pp_get_autorefresh_config - Get autorefresh config from HW + * @pp: DPU pingpong structure + * @frame_count: Used to return the current frame count from hw + * + * Returns: True if autorefresh enabled, false if disabled. + */ +static bool dpu_hw_pp_get_autorefresh_config(struct dpu_hw_pingpong *pp, + u32 *frame_count) +{ + u32 val = DPU_REG_READ(&pp->hw, PP_AUTOREFRESH_CONFIG); + if (frame_count != NULL) + *frame_count = val & 0xffff; + return !!((val & BIT(31)) >> 31); +} + static int dpu_hw_pp_poll_timeout_wr_ptr(struct dpu_hw_pingpong *pp, u32 timeout_us) { @@ -228,6 +252,8 @@ static void _setup_pingpong_ops(struct dpu_hw_pingpong *c, c->ops.enable_tearcheck = dpu_hw_pp_enable_te; c->ops.connect_external_te = dpu_hw_pp_connect_external_te; c->ops.get_vsync_info = dpu_hw_pp_get_vsync_info; + c->ops.setup_autorefresh = dpu_hw_pp_setup_autorefresh_config; + c->ops.get_autorefresh = dpu_hw_pp_get_autorefresh_config; c->ops.poll_timeout_wr_ptr = dpu_hw_pp_poll_timeout_wr_ptr; c->ops.get_line_count = dpu_hw_pp_get_line_count; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.h index 6902b9b95c8e..845b9ce80e31 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.h @@ -63,6 +63,8 @@ struct dpu_hw_dither_cfg { * @setup_tearcheck : program tear check values * @enable_tearcheck : enables tear check * @get_vsync_info : retries timing info of the panel + * @setup_autorefresh : configure and enable the autorefresh config + * @get_autorefresh : retrieve autorefresh config from hardware * @setup_dither : function to program the dither hw block * @get_line_count: obtain current vertical line counter */ @@ -95,6 +97,18 @@ struct dpu_hw_pingpong_ops { struct dpu_hw_pp_vsync_info *info); /** + * configure and enable the autorefresh config + */ + void (*setup_autorefresh)(struct dpu_hw_pingpong *pp, + u32 frame_count, bool enable); + + /** + * retrieve autorefresh config from hardware + */ + bool (*get_autorefresh)(struct dpu_hw_pingpong *pp, + u32 *frame_count); + + /** * poll until write pointer transmission starts * @Return: 0 on success, -ETIMEDOUT on timeout */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c index 2c2ca5335aa8..34d81aa16041 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c @@ -673,6 +673,7 @@ static void _setup_layer_ops(struct dpu_hw_pipe *c, c->ops.setup_multirect = dpu_hw_sspp_setup_multirect; if (test_bit(DPU_SSPP_SCALER_QSEED3, &features) || + test_bit(DPU_SSPP_SCALER_QSEED3LITE, &features) || test_bit(DPU_SSPP_SCALER_QSEED4, &features)) { c->ops.setup_scaler = _dpu_hw_sspp_setup_scaler3; c->ops.get_scaler_ver = _dpu_hw_sspp_get_scaler3_ver; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h index 85b018a9b03c..fdfd4b46e2c6 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h @@ -28,6 +28,7 @@ struct dpu_hw_pipe; #define DPU_SSPP_SCALER ((1UL << DPU_SSPP_SCALER_RGB) | \ (1UL << DPU_SSPP_SCALER_QSEED2) | \ (1UL << DPU_SSPP_SCALER_QSEED3) | \ + (1UL << DPU_SSPP_SCALER_QSEED3LITE) | \ (1UL << DPU_SSPP_SCALER_QSEED4)) /** diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c index 84e9875994a8..f94584c982cd 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c @@ -59,6 +59,19 @@ static u32 dpu_hw_util_log_mask = DPU_DBG_MASK_NONE; #define QSEED3_SEP_LUT_SIZE \ (QSEED3_LUT_SIZE * QSEED3_SEPARABLE_LUTS * sizeof(u32)) +/* DPU_SCALER_QSEED3LITE */ +#define QSEED3LITE_COEF_LUT_Y_SEP_BIT 4 +#define QSEED3LITE_COEF_LUT_UV_SEP_BIT 5 +#define QSEED3LITE_COEF_LUT_CTRL 0x4C +#define QSEED3LITE_COEF_LUT_SWAP_BIT 0 +#define QSEED3LITE_DIR_FILTER_WEIGHT 0x60 +#define QSEED3LITE_FILTERS 2 +#define QSEED3LITE_SEPARABLE_LUTS 10 +#define QSEED3LITE_LUT_SIZE 33 +#define QSEED3LITE_SEP_LUT_SIZE \ + (QSEED3LITE_LUT_SIZE * QSEED3LITE_SEPARABLE_LUTS * sizeof(u32)) + + void dpu_reg_write(struct dpu_hw_blk_reg_map *c, u32 reg_off, u32 val, @@ -156,6 +169,57 @@ static void _dpu_hw_setup_scaler3_lut(struct dpu_hw_blk_reg_map *c, } +static void _dpu_hw_setup_scaler3lite_lut(struct dpu_hw_blk_reg_map *c, + struct dpu_hw_scaler3_cfg *scaler3_cfg, u32 offset) +{ + int j, filter; + int config_lut = 0x0; + unsigned long lut_flags; + u32 lut_addr, lut_offset; + u32 *lut[QSEED3LITE_FILTERS] = {NULL, NULL}; + static const uint32_t off_tbl[QSEED3_FILTERS] = { 0x000, 0x200 }; + + DPU_REG_WRITE(c, QSEED3LITE_DIR_FILTER_WEIGHT + offset, scaler3_cfg->dir_weight); + + if (!scaler3_cfg->sep_lut) + return; + + lut_flags = (unsigned long) scaler3_cfg->lut_flag; + if (test_bit(QSEED3_COEF_LUT_Y_SEP_BIT, &lut_flags) && + (scaler3_cfg->y_rgb_sep_lut_idx < QSEED3LITE_SEPARABLE_LUTS) && + (scaler3_cfg->sep_len == QSEED3LITE_SEP_LUT_SIZE)) { + lut[0] = scaler3_cfg->sep_lut + + scaler3_cfg->y_rgb_sep_lut_idx * QSEED3LITE_LUT_SIZE; + config_lut = 1; + } + if (test_bit(QSEED3_COEF_LUT_UV_SEP_BIT, &lut_flags) && + (scaler3_cfg->uv_sep_lut_idx < QSEED3LITE_SEPARABLE_LUTS) && + (scaler3_cfg->sep_len == QSEED3LITE_SEP_LUT_SIZE)) { + lut[1] = scaler3_cfg->sep_lut + + scaler3_cfg->uv_sep_lut_idx * QSEED3LITE_LUT_SIZE; + config_lut = 1; + } + + if (config_lut) { + for (filter = 0; filter < QSEED3LITE_FILTERS; filter++) { + if (!lut[filter]) + continue; + lut_offset = 0; + lut_addr = QSEED3_COEF_LUT + offset + off_tbl[filter]; + for (j = 0; j < QSEED3LITE_LUT_SIZE; j++) { + DPU_REG_WRITE(c, + lut_addr, + (lut[filter])[lut_offset++]); + lut_addr += 4; + } + } + } + + if (test_bit(QSEED3_COEF_LUT_SWAP_BIT, &lut_flags)) + DPU_REG_WRITE(c, QSEED3_COEF_LUT_CTRL + offset, BIT(0)); + +} + static void _dpu_hw_setup_scaler3_de(struct dpu_hw_blk_reg_map *c, struct dpu_hw_scaler3_de_cfg *de_cfg, u32 offset) { @@ -242,9 +306,12 @@ void dpu_hw_setup_scaler3(struct dpu_hw_blk_reg_map *c, op_mode |= BIT(8); } - if (scaler3_cfg->lut_flag) - _dpu_hw_setup_scaler3_lut(c, scaler3_cfg, - scaler_offset); + if (scaler3_cfg->lut_flag) { + if (scaler_version < 0x2004) + _dpu_hw_setup_scaler3_lut(c, scaler3_cfg, scaler_offset); + else + _dpu_hw_setup_scaler3lite_lut(c, scaler3_cfg, scaler_offset); + } if (scaler_version == 0x1002) { phase_init = diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h index 234eb7d65753..ff3cffde84cd 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h @@ -97,6 +97,7 @@ struct dpu_hw_scaler3_de_cfg { * @ cir_lut: pointer to circular filter LUT * @ sep_lut: pointer to separable filter LUT * @ de: detail enhancer configuration + * @ dir_weight: Directional weight */ struct dpu_hw_scaler3_cfg { u32 enable; @@ -137,6 +138,8 @@ struct dpu_hw_scaler3_cfg { * Detail enhancer settings */ struct dpu_hw_scaler3_de_cfg de; + + u32 dir_weight; }; /** diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.c index cf867f3f7c36..b757054e1c23 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.c @@ -30,7 +30,7 @@ #define VBIF_XIN_HALT_CTRL0 0x0200 #define VBIF_XIN_HALT_CTRL1 0x0204 #define VBIF_XINL_QOS_RP_REMAP_000 0x0550 -#define VBIF_XINL_QOS_LVL_REMAP_000 0x0590 +#define VBIF_XINL_QOS_LVL_REMAP_000(v) (v < DPU_HW_VER_400 ? 0x570 : 0x0590) static void dpu_hw_clear_errors(struct dpu_hw_vbif *vbif, u32 *pnd_errors, u32 *src_errors) @@ -156,18 +156,19 @@ static void dpu_hw_set_qos_remap(struct dpu_hw_vbif *vbif, u32 xin_id, u32 level, u32 remap_level) { struct dpu_hw_blk_reg_map *c; - u32 reg_val, reg_val_lvl, mask, reg_high, reg_shift; + u32 reg_lvl, reg_val, reg_val_lvl, mask, reg_high, reg_shift; if (!vbif) return; c = &vbif->hw; + reg_lvl = VBIF_XINL_QOS_LVL_REMAP_000(c->hwversion); reg_high = ((xin_id & 0x8) >> 3) * 4 + (level * 8); reg_shift = (xin_id & 0x7) * 4; reg_val = DPU_REG_READ(c, VBIF_XINL_QOS_RP_REMAP_000 + reg_high); - reg_val_lvl = DPU_REG_READ(c, VBIF_XINL_QOS_LVL_REMAP_000 + reg_high); + reg_val_lvl = DPU_REG_READ(c, reg_lvl + reg_high); mask = 0x7 << reg_shift; @@ -178,7 +179,7 @@ static void dpu_hw_set_qos_remap(struct dpu_hw_vbif *vbif, reg_val_lvl |= (remap_level << reg_shift) & mask; DPU_REG_WRITE(c, VBIF_XINL_QOS_RP_REMAP_000 + reg_high, reg_val); - DPU_REG_WRITE(c, VBIF_XINL_QOS_LVL_REMAP_000 + reg_high, reg_val_lvl); + DPU_REG_WRITE(c, reg_lvl + reg_high, reg_val_lvl); } static void dpu_hw_set_write_gather_en(struct dpu_hw_vbif *vbif, u32 xin_id) diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c index 374b0e8471e6..5a8e3e1fc48c 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c @@ -749,7 +749,7 @@ static void _dpu_kms_set_encoder_mode(struct msm_kms *kms, case DRM_MODE_ENCODER_TMDS: info.num_of_h_tiles = 1; break; - }; + } rc = dpu_encoder_setup(encoder->dev, encoder, &info); if (rc) diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c index bc0231a50132..f898a8f67b7f 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c @@ -1465,6 +1465,7 @@ static int _dpu_plane_init_debugfs(struct drm_plane *plane) pdpu->debugfs_root, &pdpu->debugfs_src); if (cfg->features & BIT(DPU_SSPP_SCALER_QSEED3) || + cfg->features & BIT(DPU_SSPP_SCALER_QSEED3LITE) || cfg->features & BIT(DPU_SSPP_SCALER_QSEED2) || cfg->features & BIT(DPU_SSPP_SCALER_QSEED4)) { dpu_debugfs_setup_regset32(&pdpu->debugfs_scaler, diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c index df10c1ac7591..94ce62a26daf 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c @@ -177,7 +177,7 @@ static const struct mdp5_cfg_hw msm8x74v2_config = { [3] = INTF_HDMI, }, }, - .max_clk = 200000000, + .max_clk = 320000000, }; static const struct mdp5_cfg_hw apq8084_config = { diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c index 0c8f9f88301f..f5d71b274079 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c @@ -1180,7 +1180,7 @@ static void mdp5_crtc_pp_done_irq(struct mdp_irq *irq, uint32_t irqstatus) struct mdp5_crtc *mdp5_crtc = container_of(irq, struct mdp5_crtc, pp_done); - complete(&mdp5_crtc->pp_completion); + complete_all(&mdp5_crtc->pp_completion); } static void mdp5_crtc_wait_for_pp_done(struct drm_crtc *crtc) diff --git a/drivers/gpu/drm/msm/dp/dp_aux.c b/drivers/gpu/drm/msm/dp/dp_aux.c index 19b35ae3e927..1c6e1d2b947c 100644 --- a/drivers/gpu/drm/msm/dp/dp_aux.c +++ b/drivers/gpu/drm/msm/dp/dp_aux.c @@ -336,7 +336,6 @@ static ssize_t dp_aux_transfer(struct drm_dp_aux *dp_aux, ssize_t ret; int const aux_cmd_native_max = 16; int const aux_cmd_i2c_max = 128; - int const retry_count = 5; struct dp_aux_private *aux = container_of(dp_aux, struct dp_aux_private, dp_aux); @@ -378,12 +377,6 @@ static ssize_t dp_aux_transfer(struct drm_dp_aux *dp_aux, ret = dp_aux_cmd_fifo_tx(aux, msg); if (ret < 0) { - if (aux->native) { - aux->retry_cnt++; - if (!(aux->retry_cnt % retry_count)) - dp_catalog_aux_update_cfg(aux->catalog); - dp_catalog_aux_reset(aux->catalog); - } usleep_range(400, 500); /* at least 400us to next try */ goto unlock_exit; } diff --git a/drivers/gpu/drm/msm/dp/dp_catalog.c b/drivers/gpu/drm/msm/dp/dp_catalog.c index 44f0c57798d0..b1a9b1b98f5f 100644 --- a/drivers/gpu/drm/msm/dp/dp_catalog.c +++ b/drivers/gpu/drm/msm/dp/dp_catalog.c @@ -190,6 +190,18 @@ int dp_catalog_aux_clear_hw_interrupts(struct dp_catalog *dp_catalog) return 0; } +/** + * dp_catalog_aux_reset() - reset AUX controller + * + * @aux: DP catalog structure + * + * return: void + * + * This function reset AUX controller + * + * NOTE: reset AUX controller will also clear any pending HPD related interrupts + * + */ void dp_catalog_aux_reset(struct dp_catalog *dp_catalog) { u32 aux_ctrl; @@ -483,6 +495,18 @@ int dp_catalog_ctrl_set_pattern(struct dp_catalog *dp_catalog, return 0; } +/** + * dp_catalog_ctrl_reset() - reset DP controller + * + * @dp_catalog: DP catalog structure + * + * return: void + * + * This function reset the DP controller + * + * NOTE: reset DP controller will also clear any pending HPD related interrupts + * + */ void dp_catalog_ctrl_reset(struct dp_catalog *dp_catalog) { u32 sw_reset; diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c index 36b39c381b3f..1390f3547fde 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.c +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c @@ -631,7 +631,7 @@ static void _dp_ctrl_calc_tu(struct dp_tu_calc_input *in, tu = kzalloc(sizeof(*tu), GFP_KERNEL); if (!tu) - return + return; dp_panel_update_tu_timings(in, tu); @@ -1158,7 +1158,7 @@ static int dp_ctrl_link_rate_down_shift(struct dp_ctrl_private *ctrl) default: ret = -EINVAL; break; - }; + } if (!ret) DRM_DEBUG_DP("new rate=0x%x\n", ctrl->link->link_params.rate); @@ -1296,7 +1296,6 @@ static int dp_ctrl_setup_main_link(struct dp_ctrl_private *ctrl, * transitioned to PUSH_IDLE. In order to start transmitting * a link training pattern, we have to first do soft reset. */ - dp_catalog_ctrl_reset(ctrl->catalog); ret = dp_ctrl_link_train(ctrl, cr, training_step); @@ -1365,7 +1364,7 @@ static int dp_ctrl_enable_stream_clocks(struct dp_ctrl_private *ctrl) return ret; } -int dp_ctrl_host_init(struct dp_ctrl *dp_ctrl, bool flip) +int dp_ctrl_host_init(struct dp_ctrl *dp_ctrl, bool flip, bool reset) { struct dp_ctrl_private *ctrl; struct dp_io *dp_io; @@ -1382,6 +1381,9 @@ int dp_ctrl_host_init(struct dp_ctrl *dp_ctrl, bool flip) ctrl->dp_ctrl.orientation = flip; + if (reset) + dp_catalog_ctrl_reset(ctrl->catalog); + dp_catalog_ctrl_phy_reset(ctrl->catalog); phy_init(phy); dp_catalog_ctrl_enable_irq(ctrl->catalog, true); @@ -1496,7 +1498,6 @@ static int dp_ctrl_link_maintenance(struct dp_ctrl_private *ctrl) int training_step = DP_TRAINING_NONE; dp_ctrl_push_idle(&ctrl->dp_ctrl); - dp_catalog_ctrl_reset(ctrl->catalog); ctrl->dp_ctrl.pixel_rate = ctrl->panel->dp_mode.drm_mode.clock; @@ -1785,14 +1786,14 @@ int dp_ctrl_on_stream(struct dp_ctrl *dp_ctrl) * Set up transfer unit values and set controller state to send * video. */ + reinit_completion(&ctrl->video_comp); + dp_ctrl_configure_source_params(ctrl); dp_catalog_ctrl_config_msa(ctrl->catalog, ctrl->link->link_params.rate, ctrl->dp_ctrl.pixel_rate, dp_ctrl_use_fixed_nvid(ctrl)); - reinit_completion(&ctrl->video_comp); - dp_ctrl_setup_tr_unit(ctrl); dp_catalog_ctrl_state_ctrl(ctrl->catalog, DP_STATE_CTRL_SEND_VIDEO); diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.h b/drivers/gpu/drm/msm/dp/dp_ctrl.h index f60ba93c8678..a836bd358447 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.h +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.h @@ -19,7 +19,7 @@ struct dp_ctrl { u32 pixel_rate; }; -int dp_ctrl_host_init(struct dp_ctrl *dp_ctrl, bool flip); +int dp_ctrl_host_init(struct dp_ctrl *dp_ctrl, bool flip, bool reset); void dp_ctrl_host_deinit(struct dp_ctrl *dp_ctrl); int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl); int dp_ctrl_on_stream(struct dp_ctrl *dp_ctrl); diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index 3bc7ed21de28..5a39da6e1eaf 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -350,7 +350,7 @@ end: return rc; } -static void dp_display_host_init(struct dp_display_private *dp) +static void dp_display_host_init(struct dp_display_private *dp, int reset) { bool flip = false; @@ -365,7 +365,7 @@ static void dp_display_host_init(struct dp_display_private *dp) dp_display_set_encoder_mode(dp); dp_power_init(dp->power, flip); - dp_ctrl_host_init(dp->ctrl, flip); + dp_ctrl_host_init(dp->ctrl, flip, reset); dp_aux_init(dp->aux); dp->core_initialized = true; } @@ -403,7 +403,7 @@ static int dp_display_usbpd_configure_cb(struct device *dev) goto end; } - dp_display_host_init(dp); + dp_display_host_init(dp, false); /* * set sink to normal operation mode -- D0 @@ -651,8 +651,8 @@ static int dp_hpd_unplug_handle(struct dp_display_private *dp, u32 data) dp_add_event(dp, EV_DISCONNECT_PENDING_TIMEOUT, 0, DP_TIMEOUT_5_SECOND); /* signal the disconnect event early to ensure proper teardown */ - dp_display_handle_plugged_change(g_dp_display, false); reinit_completion(&dp->audio_comp); + dp_display_handle_plugged_change(g_dp_display, false); dp_catalog_hpd_config_intr(dp->catalog, DP_DP_HPD_PLUG_INT_MASK | DP_DP_IRQ_HPD_INT_MASK, true); @@ -700,6 +700,13 @@ static int dp_irq_hpd_handle(struct dp_display_private *dp, u32 data) return 0; } + if (state == ST_CONNECT_PENDING || state == ST_DISCONNECT_PENDING) { + /* wait until ST_CONNECTED */ + dp_add_event(dp, EV_IRQ_HPD_INT, 0, 1); /* delay = 1 */ + mutex_unlock(&dp->event_mutex); + return 0; + } + ret = dp_display_usbpd_attention_cb(&dp->pdev->dev); if (ret == -ECONNRESET) { /* cable unplugged */ dp->core_initialized = false; @@ -890,6 +897,9 @@ static int dp_display_disable(struct dp_display_private *dp, u32 data) /* wait only if audio was enabled */ if (dp_display->audio_enabled) { + /* signal the disconnect event */ + reinit_completion(&dp->audio_comp); + dp_display_handle_plugged_change(dp_display, false); if (!wait_for_completion_timeout(&dp->audio_comp, HZ * 5)) DRM_ERROR("audio comp timeout\n"); @@ -1002,7 +1012,7 @@ int dp_display_get_test_bpp(struct msm_dp *dp) static void dp_display_config_hpd(struct dp_display_private *dp) { - dp_display_host_init(dp); + dp_display_host_init(dp, true); dp_catalog_ctrl_hpd_config(dp->catalog); /* Enable interrupt first time @@ -1256,7 +1266,7 @@ static int dp_pm_resume(struct device *dev) dp->hpd_state = ST_DISCONNECTED; /* turn on dp ctrl/phy */ - dp_display_host_init(dp); + dp_display_host_init(dp, true); dp_catalog_ctrl_hpd_config(dp->catalog); @@ -1439,7 +1449,7 @@ int msm_dp_display_enable(struct msm_dp *dp, struct drm_encoder *encoder) state = dp_display->hpd_state; if (state == ST_DISPLAY_OFF) - dp_display_host_init(dp_display); + dp_display_host_init(dp_display, true); dp_display_enable(dp_display, 0); diff --git a/drivers/gpu/drm/msm/dp/dp_panel.c b/drivers/gpu/drm/msm/dp/dp_panel.c index d1780bcac8cc..9cc816663668 100644 --- a/drivers/gpu/drm/msm/dp/dp_panel.c +++ b/drivers/gpu/drm/msm/dp/dp_panel.c @@ -409,7 +409,6 @@ int dp_panel_timing_cfg(struct dp_panel *dp_panel) int dp_panel_init_panel_info(struct dp_panel *dp_panel) { - int rc = 0; struct drm_display_mode *drm_mode; drm_mode = &dp_panel->dp_mode.drm_mode; @@ -436,7 +435,7 @@ int dp_panel_init_panel_info(struct dp_panel *dp_panel) min_t(u32, dp_panel->dp_mode.bpp, 30)); DRM_DEBUG_DP("updated bpp = %d\n", dp_panel->dp_mode.bpp); - return rc; + return 0; } struct dp_panel *dp_panel_get(struct dp_panel_in *in) diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c index 1afb7c579dbb..eca86bf448f7 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c @@ -139,7 +139,7 @@ const struct msm_dsi_phy_cfg dsi_phy_20nm_cfgs = { .disable = dsi_20nm_phy_disable, .init = msm_dsi_phy_init_common, }, - .io_start = { 0xfd998300, 0xfd9a0300 }, + .io_start = { 0xfd998500, 0xfd9a0500 }, .num_dsi_phy = 2, }; diff --git a/drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c b/drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c index e4e9bf04b736..de3b802ccd3d 100644 --- a/drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c +++ b/drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c @@ -172,9 +172,7 @@ static void dsi_pll_calc_dec_frac(struct dsi_pll_10nm *pll) multiplier = 1 << config->frac_bits; dec_multiple = div_u64(pll_freq * multiplier, divider); - div_u64_rem(dec_multiple, multiplier, &frac); - - dec = div_u64(dec_multiple, multiplier); + dec = div_u64_rem(dec_multiple, multiplier, &frac); if (pll_freq <= 1900000000UL) regs->pll_prop_gain_rate = 8; @@ -306,7 +304,8 @@ static void dsi_pll_commit(struct dsi_pll_10nm *pll) reg->frac_div_start_mid); pll_write(base + REG_DSI_10nm_PHY_PLL_FRAC_DIV_START_HIGH_1, reg->frac_div_start_high); - pll_write(base + REG_DSI_10nm_PHY_PLL_PLL_LOCKDET_RATE_1, 0x40); + pll_write(base + REG_DSI_10nm_PHY_PLL_PLL_LOCKDET_RATE_1, + reg->pll_lockdet_rate); pll_write(base + REG_DSI_10nm_PHY_PLL_PLL_LOCK_DELAY, 0x06); pll_write(base + REG_DSI_10nm_PHY_PLL_CMODE, 0x10); pll_write(base + REG_DSI_10nm_PHY_PLL_CLOCK_INVERTERS, @@ -345,6 +344,7 @@ static int dsi_pll_10nm_vco_set_rate(struct clk_hw *hw, unsigned long rate, static int dsi_pll_10nm_lock_status(struct dsi_pll_10nm *pll) { + struct device *dev = &pll->pdev->dev; int rc; u32 status = 0; u32 const delay_us = 100; @@ -357,8 +357,8 @@ static int dsi_pll_10nm_lock_status(struct dsi_pll_10nm *pll) delay_us, timeout_us); if (rc) - pr_err("DSI PLL(%d) lock failed, status=0x%08x\n", - pll->id, status); + DRM_DEV_ERROR(dev, "DSI PLL(%d) lock failed, status=0x%08x\n", + pll->id, status); return rc; } @@ -405,6 +405,7 @@ static int dsi_pll_10nm_vco_prepare(struct clk_hw *hw) { struct msm_dsi_pll *pll = hw_clk_to_pll(hw); struct dsi_pll_10nm *pll_10nm = to_pll_10nm(pll); + struct device *dev = &pll_10nm->pdev->dev; int rc; dsi_pll_enable_pll_bias(pll_10nm); @@ -413,7 +414,7 @@ static int dsi_pll_10nm_vco_prepare(struct clk_hw *hw) rc = dsi_pll_10nm_vco_set_rate(hw,pll_10nm->vco_current_rate, 0); if (rc) { - pr_err("vco_set_rate failed, rc=%d\n", rc); + DRM_DEV_ERROR(dev, "vco_set_rate failed, rc=%d\n", rc); return rc; } @@ -430,7 +431,7 @@ static int dsi_pll_10nm_vco_prepare(struct clk_hw *hw) /* Check for PLL lock */ rc = dsi_pll_10nm_lock_status(pll_10nm); if (rc) { - pr_err("PLL(%d) lock failed\n", pll_10nm->id); + DRM_DEV_ERROR(dev, "PLL(%d) lock failed\n", pll_10nm->id); goto error; } @@ -483,6 +484,7 @@ static unsigned long dsi_pll_10nm_vco_recalc_rate(struct clk_hw *hw, { struct msm_dsi_pll *pll = hw_clk_to_pll(hw); struct dsi_pll_10nm *pll_10nm = to_pll_10nm(pll); + struct dsi_pll_config *config = &pll_10nm->pll_configuration; void __iomem *base = pll_10nm->mmio; u64 ref_clk = pll_10nm->vco_ref_clk_rate; u64 vco_rate = 0x0; @@ -503,9 +505,8 @@ static unsigned long dsi_pll_10nm_vco_recalc_rate(struct clk_hw *hw, /* * TODO: * 1. Assumes prescaler is disabled - * 2. Multiplier is 2^18. it should be 2^(num_of_frac_bits) */ - multiplier = 1 << 18; + multiplier = 1 << config->frac_bits; pll_freq = dec * (ref_clk * 2); tmp64 = (ref_clk * 2 * frac); pll_freq += div_u64(tmp64, multiplier); diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 108c405e03dd..94525ac76d4e 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -788,9 +788,10 @@ static int msm_ioctl_gem_info_iova(struct drm_device *dev, struct drm_file *file, struct drm_gem_object *obj, uint64_t *iova) { + struct msm_drm_private *priv = dev->dev_private; struct msm_file_private *ctx = file->driver_priv; - if (!ctx->aspace) + if (!priv->gpu) return -EINVAL; /* diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index a588b0388d27..f091c1e164fa 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -987,8 +987,7 @@ void msm_gem_free_object(struct drm_gem_object *obj) /* Don't drop the pages for imported dmabuf, as they are not * ours, just free the array we allocated: */ - if (msm_obj->pages) - kvfree(msm_obj->pages); + kvfree(msm_obj->pages); put_iova_vmas(obj); diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c index d04c349d8112..5480852bdeda 100644 --- a/drivers/gpu/drm/msm/msm_gem_submit.c +++ b/drivers/gpu/drm/msm/msm_gem_submit.c @@ -198,6 +198,8 @@ static int submit_lookup_cmds(struct msm_gem_submit *submit, submit->cmd[i].idx = submit_cmd.submit_idx; submit->cmd[i].nr_relocs = submit_cmd.nr_relocs; + userptr = u64_to_user_ptr(submit_cmd.relocs); + sz = array_size(submit_cmd.nr_relocs, sizeof(struct drm_msm_gem_submit_reloc)); /* check for overflow: */ diff --git a/drivers/gpu/drm/msm/msm_kms.h b/drivers/gpu/drm/msm/msm_kms.h index d8151a89e163..4735251a394d 100644 --- a/drivers/gpu/drm/msm/msm_kms.h +++ b/drivers/gpu/drm/msm/msm_kms.h @@ -157,6 +157,7 @@ struct msm_kms { * from the crtc's pending_timer close to end of the frame: */ struct mutex commit_lock[MAX_CRTCS]; + struct lock_class_key commit_lock_keys[MAX_CRTCS]; unsigned pending_crtc_mask; struct msm_pending_timer pending_timers[MAX_CRTCS]; }; @@ -166,8 +167,11 @@ static inline int msm_kms_init(struct msm_kms *kms, { unsigned i, ret; - for (i = 0; i < ARRAY_SIZE(kms->commit_lock); i++) - mutex_init(&kms->commit_lock[i]); + for (i = 0; i < ARRAY_SIZE(kms->commit_lock); i++) { + lockdep_register_key(&kms->commit_lock_keys[i]); + __mutex_init(&kms->commit_lock[i], "&kms->commit_lock[i]", + &kms->commit_lock_keys[i]); + } kms->funcs = funcs; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 69da601f1754..e771bd519ee2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -261,6 +261,9 @@ gk104_fifo_pbdma = { struct nvkm_engine * gk104_fifo_id_engine(struct nvkm_fifo *base, int engi) { + if (engi == GK104_FIFO_ENGN_SW) + return nvkm_device_engine(base->engine.subdev.device, NVKM_ENGINE_SW, 0); + return gk104_fifo(base)->engine[engi].engine; } diff --git a/drivers/gpu/drm/panel/panel-elida-kd35t133.c b/drivers/gpu/drm/panel/panel-elida-kd35t133.c index bc36aa3c1123..fe5ac3ef9018 100644 --- a/drivers/gpu/drm/panel/panel-elida-kd35t133.c +++ b/drivers/gpu/drm/panel/panel-elida-kd35t133.c @@ -265,7 +265,8 @@ static int kd35t133_probe(struct mipi_dsi_device *dsi) dsi->lanes = 1; dsi->format = MIPI_DSI_FMT_RGB888; dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | - MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET; + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET | + MIPI_DSI_CLOCK_NON_CONTINUOUS; drm_panel_init(&ctx->panel, &dsi->dev, &kd35t133_funcs, DRM_MODE_CONNECTOR_DSI); diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c index fb5f6a5e81d7..1864467f1063 100644 --- a/drivers/gpu/drm/qxl/qxl_drv.c +++ b/drivers/gpu/drm/qxl/qxl_drv.c @@ -141,7 +141,7 @@ static void qxl_drm_release(struct drm_device *dev) /* * TODO: qxl_device_fini() call should be in qxl_pci_remove(), - * reodering qxl_modeset_fini() + qxl_device_fini() calls is + * reordering qxl_modeset_fini() + qxl_device_fini() calls is * non-trivial though. */ qxl_modeset_fini(qdev); diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h index 4a2099cb582e..857d97cdc67c 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h @@ -17,9 +17,20 @@ #define NUM_YUV2YUV_COEFFICIENTS 12 +/* AFBC supports a number of configurable modes. Relevant to us is block size + * (16x16 or 32x8), storage modifiers (SPARSE, SPLIT), and the YUV-like + * colourspace transform (YTR). 16x16 SPARSE mode is always used. SPLIT mode + * could be enabled via the hreg_block_split register, but is not currently + * handled. The colourspace transform is implicitly always assumed by the + * decoder, so consumers must use this transform as well. + * + * Failure to match modifiers will cause errors displaying AFBC buffers + * produced by conformant AFBC producers, including Mesa. + */ #define ROCKCHIP_AFBC_MOD \ DRM_FORMAT_MOD_ARM_AFBC( \ AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_SPARSE \ + | AFBC_FORMAT_MOD_YTR \ ) enum vop_data_format { diff --git a/drivers/gpu/drm/tilcdc/Makefile b/drivers/gpu/drm/tilcdc/Makefile index 662bf3a348c9..f5190477de72 100644 --- a/drivers/gpu/drm/tilcdc/Makefile +++ b/drivers/gpu/drm/tilcdc/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -ifeq (, $(findstring -W,$(EXTRA_CFLAGS))) +ifeq (, $(findstring -W,$(KCFLAGS))) ccflags-y += -Werror endif diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index b65f4b12f986..20a25660b35b 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -959,8 +959,10 @@ static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, return ret; /* move to the bounce domain */ ret = ttm_bo_handle_move_mem(bo, &hop_mem, false, ctx, NULL); - if (ret) + if (ret) { + ttm_resource_free(bo, &hop_mem); return ret; + } return 0; } @@ -991,18 +993,19 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, * stop and the driver will be called to make * the second hop. */ -bounce: ret = ttm_bo_mem_space(bo, placement, &mem, ctx); if (ret) return ret; +bounce: ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx, &hop); if (ret == -EMULTIHOP) { ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop); if (ret) - return ret; + goto out; /* try and move to final place now. */ goto bounce; } +out: if (ret) ttm_resource_free(bo, &mem); return ret; diff --git a/drivers/greybus/es2.c b/drivers/greybus/es2.c index 1df6ab5d339d..48ad154df3a7 100644 --- a/drivers/greybus/es2.c +++ b/drivers/greybus/es2.c @@ -567,12 +567,9 @@ static int cport_enable(struct gb_host_device *hd, u16 cport_id, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, cport_id, 0, req, sizeof(*req), ES2_USB_CTRL_TIMEOUT); - if (ret != sizeof(*req)) { + if (ret < 0) { dev_err(&udev->dev, "failed to set cport flags for port %d\n", cport_id); - if (ret >= 0) - ret = -EIO; - goto out; } @@ -961,12 +958,10 @@ static int arpc_send(struct es2_ap_dev *es2, struct arpc *rpc, int timeout) 0, 0, rpc->req, le16_to_cpu(rpc->req->size), ES2_USB_CTRL_TIMEOUT); - if (retval != le16_to_cpu(rpc->req->size)) { + if (retval < 0) { dev_err(&udev->dev, "failed to send ARPC request %d: %d\n", rpc->req->type, retval); - if (retval > 0) - retval = -EIO; return retval; } diff --git a/drivers/greybus/greybus_trace.h b/drivers/greybus/greybus_trace.h index 1bc9f1275c65..616a3bd61aa6 100644 --- a/drivers/greybus/greybus_trace.h +++ b/drivers/greybus/greybus_trace.h @@ -40,7 +40,7 @@ DECLARE_EVENT_CLASS(gb_message, __entry->result = message->header->result; ), - TP_printk("size=%hu operation_id=0x%04x type=0x%02x result=0x%02x", + TP_printk("size=%u operation_id=0x%04x type=0x%02x result=0x%02x", __entry->size, __entry->operation_id, __entry->type, __entry->result) ); @@ -317,7 +317,7 @@ DECLARE_EVENT_CLASS(gb_interface, __entry->mode_switch = intf->mode_switch; ), - TP_printk("intf_id=%hhu device_id=%hhu module_id=%hhu D=%d J=%d A=%d E=%d M=%d", + TP_printk("intf_id=%u device_id=%u module_id=%u D=%d J=%d A=%d E=%d M=%d", __entry->id, __entry->device_id, __entry->module_id, __entry->disconnected, __entry->ejected, __entry->active, __entry->enabled, __entry->mode_switch) @@ -391,7 +391,7 @@ DECLARE_EVENT_CLASS(gb_module, __entry->disconnected = module->disconnected; ), - TP_printk("hd_bus_id=%d module_id=%hhu num_interfaces=%zu disconnected=%d", + TP_printk("hd_bus_id=%d module_id=%u num_interfaces=%zu disconnected=%d", __entry->hd_bus_id, __entry->module_id, __entry->num_interfaces, __entry->disconnected) ); diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index b279c6130bc2..f042c7466930 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -1217,4 +1217,6 @@ source "drivers/hid/intel-ish-hid/Kconfig" source "drivers/hid/amd-sfh-hid/Kconfig" +source "drivers/hid/surface-hid/Kconfig" + endmenu diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 6e24c37132ec..a664a495fc44 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -146,3 +146,5 @@ obj-$(CONFIG_INTEL_ISH_HID) += intel-ish-hid/ obj-$(INTEL_ISH_FIRMWARE_DOWNLOADER) += intel-ish-hid/ obj-$(CONFIG_AMD_SFH_HID) += amd-sfh-hid/ + +obj-$(CONFIG_SURFACE_HID_CORE) += surface-hid/ diff --git a/drivers/hid/surface-hid/Kconfig b/drivers/hid/surface-hid/Kconfig new file mode 100644 index 000000000000..7ce9b5d641eb --- /dev/null +++ b/drivers/hid/surface-hid/Kconfig @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0+ +menu "Surface System Aggregator Module HID support" + depends on SURFACE_AGGREGATOR + depends on INPUT + +config SURFACE_HID + tristate "HID transport driver for Surface System Aggregator Module" + depends on SURFACE_AGGREGATOR_REGISTRY + select SURFACE_HID_CORE + help + Driver to support integrated HID devices on newer Microsoft Surface + models. + + This driver provides support for the HID transport protocol provided + by the Surface Aggregator Module (i.e. the embedded controller) on + 7th-generation Microsoft Surface devices, i.e. Surface Book 3 and + Surface Laptop 3. On those models, it is mainly used to connect the + integrated touchpad and keyboard. + + Say M or Y here, if you want support for integrated HID devices, i.e. + integrated touchpad and keyboard, on 7th generation Microsoft Surface + models. + +config SURFACE_KBD + tristate "HID keyboard transport driver for Surface System Aggregator Module" + select SURFACE_HID_CORE + help + Driver to support HID keyboards on Surface Laptop 1 and 2 devices. + + This driver provides support for the HID transport protocol provided + by the Surface Aggregator Module (i.e. the embedded controller) on + Microsoft Surface Laptops 1 and 2. It is used to connect the + integrated keyboard on those devices. + + Say M or Y here, if you want support for the integrated keyboard on + Microsoft Surface Laptops 1 and 2. + +endmenu + +config SURFACE_HID_CORE + tristate + select HID diff --git a/drivers/hid/surface-hid/Makefile b/drivers/hid/surface-hid/Makefile new file mode 100644 index 000000000000..4ae11cf09b25 --- /dev/null +++ b/drivers/hid/surface-hid/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0+ +# +# Makefile - Surface System Aggregator Module (SSAM) HID transport driver. +# +obj-$(CONFIG_SURFACE_HID_CORE) += surface_hid_core.o +obj-$(CONFIG_SURFACE_HID) += surface_hid.o +obj-$(CONFIG_SURFACE_KBD) += surface_kbd.o diff --git a/drivers/hid/surface-hid/surface_hid.c b/drivers/hid/surface-hid/surface_hid.c new file mode 100644 index 000000000000..3477b31611ae --- /dev/null +++ b/drivers/hid/surface-hid/surface_hid.c @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Surface System Aggregator Module (SSAM) HID transport driver for the + * generic HID interface (HID/TC=0x15 subsystem). Provides support for + * integrated HID devices on Surface Laptop 3, Book 3, and later. + * + * Copyright (C) 2019-2021 Blaž Hrastnik <blaz@mxxn.io>, + * Maximilian Luz <luzmaximilian@gmail.com> + */ + +#include <asm/unaligned.h> +#include <linux/hid.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/types.h> + +#include <linux/surface_aggregator/controller.h> +#include <linux/surface_aggregator/device.h> + +#include "surface_hid_core.h" + + +/* -- SAM interface. -------------------------------------------------------- */ + +struct surface_hid_buffer_slice { + __u8 entry; + __le32 offset; + __le32 length; + __u8 end; + __u8 data[]; +} __packed; + +static_assert(sizeof(struct surface_hid_buffer_slice) == 10); + +enum surface_hid_cid { + SURFACE_HID_CID_OUTPUT_REPORT = 0x01, + SURFACE_HID_CID_GET_FEATURE_REPORT = 0x02, + SURFACE_HID_CID_SET_FEATURE_REPORT = 0x03, + SURFACE_HID_CID_GET_DESCRIPTOR = 0x04, +}; + +static int ssam_hid_get_descriptor(struct surface_hid_device *shid, u8 entry, u8 *buf, size_t len) +{ + u8 buffer[sizeof(struct surface_hid_buffer_slice) + 0x76]; + struct surface_hid_buffer_slice *slice; + struct ssam_request rqst; + struct ssam_response rsp; + u32 buffer_len, offset, length; + int status; + + /* + * Note: The 0x76 above has been chosen because that's what's used by + * the Windows driver. Together with the header, this leads to a 128 + * byte payload in total. + */ + + buffer_len = ARRAY_SIZE(buffer) - sizeof(struct surface_hid_buffer_slice); + + rqst.target_category = shid->uid.category; + rqst.target_id = shid->uid.target; + rqst.command_id = SURFACE_HID_CID_GET_DESCRIPTOR; + rqst.instance_id = shid->uid.instance; + rqst.flags = SSAM_REQUEST_HAS_RESPONSE; + rqst.length = sizeof(struct surface_hid_buffer_slice); + rqst.payload = buffer; + + rsp.capacity = ARRAY_SIZE(buffer); + rsp.pointer = buffer; + + slice = (struct surface_hid_buffer_slice *)buffer; + slice->entry = entry; + slice->end = 0; + + offset = 0; + length = buffer_len; + + while (!slice->end && offset < len) { + put_unaligned_le32(offset, &slice->offset); + put_unaligned_le32(length, &slice->length); + + rsp.length = 0; + + status = ssam_retry(ssam_request_sync_onstack, shid->ctrl, &rqst, &rsp, + sizeof(*slice)); + if (status) + return status; + + offset = get_unaligned_le32(&slice->offset); + length = get_unaligned_le32(&slice->length); + + /* Don't mess stuff up in case we receive garbage. */ + if (length > buffer_len || offset > len) + return -EPROTO; + + if (offset + length > len) + length = len - offset; + + memcpy(buf + offset, &slice->data[0], length); + + offset += length; + length = buffer_len; + } + + if (offset != len) { + dev_err(shid->dev, "unexpected descriptor length: got %u, expected %zu\n", + offset, len); + return -EPROTO; + } + + return 0; +} + +static int ssam_hid_set_raw_report(struct surface_hid_device *shid, u8 rprt_id, bool feature, + u8 *buf, size_t len) +{ + struct ssam_request rqst; + u8 cid; + + if (feature) + cid = SURFACE_HID_CID_SET_FEATURE_REPORT; + else + cid = SURFACE_HID_CID_OUTPUT_REPORT; + + rqst.target_category = shid->uid.category; + rqst.target_id = shid->uid.target; + rqst.instance_id = shid->uid.instance; + rqst.command_id = cid; + rqst.flags = 0; + rqst.length = len; + rqst.payload = buf; + + buf[0] = rprt_id; + + return ssam_retry(ssam_request_sync, shid->ctrl, &rqst, NULL); +} + +static int ssam_hid_get_raw_report(struct surface_hid_device *shid, u8 rprt_id, u8 *buf, size_t len) +{ + struct ssam_request rqst; + struct ssam_response rsp; + + rqst.target_category = shid->uid.category; + rqst.target_id = shid->uid.target; + rqst.instance_id = shid->uid.instance; + rqst.command_id = SURFACE_HID_CID_GET_FEATURE_REPORT; + rqst.flags = 0; + rqst.length = sizeof(rprt_id); + rqst.payload = &rprt_id; + + rsp.capacity = len; + rsp.length = 0; + rsp.pointer = buf; + + return ssam_retry(ssam_request_sync_onstack, shid->ctrl, &rqst, &rsp, sizeof(rprt_id)); +} + +static u32 ssam_hid_event_fn(struct ssam_event_notifier *nf, const struct ssam_event *event) +{ + struct surface_hid_device *shid = container_of(nf, struct surface_hid_device, notif); + + if (event->command_id != 0x00) + return 0; + + hid_input_report(shid->hid, HID_INPUT_REPORT, (u8 *)&event->data[0], event->length, 0); + return SSAM_NOTIF_HANDLED; +} + + +/* -- Transport driver. ----------------------------------------------------- */ + +static int shid_output_report(struct surface_hid_device *shid, u8 rprt_id, u8 *buf, size_t len) +{ + int status; + + status = ssam_hid_set_raw_report(shid, rprt_id, false, buf, len); + return status >= 0 ? len : status; +} + +static int shid_get_feature_report(struct surface_hid_device *shid, u8 rprt_id, u8 *buf, size_t len) +{ + int status; + + status = ssam_hid_get_raw_report(shid, rprt_id, buf, len); + return status >= 0 ? len : status; +} + +static int shid_set_feature_report(struct surface_hid_device *shid, u8 rprt_id, u8 *buf, size_t len) +{ + int status; + + status = ssam_hid_set_raw_report(shid, rprt_id, true, buf, len); + return status >= 0 ? len : status; +} + + +/* -- Driver setup. --------------------------------------------------------- */ + +static int surface_hid_probe(struct ssam_device *sdev) +{ + struct surface_hid_device *shid; + + shid = devm_kzalloc(&sdev->dev, sizeof(*shid), GFP_KERNEL); + if (!shid) + return -ENOMEM; + + shid->dev = &sdev->dev; + shid->ctrl = sdev->ctrl; + shid->uid = sdev->uid; + + shid->notif.base.priority = 1; + shid->notif.base.fn = ssam_hid_event_fn; + shid->notif.event.reg = SSAM_EVENT_REGISTRY_REG; + shid->notif.event.id.target_category = sdev->uid.category; + shid->notif.event.id.instance = sdev->uid.instance; + shid->notif.event.mask = SSAM_EVENT_MASK_STRICT; + shid->notif.event.flags = 0; + + shid->ops.get_descriptor = ssam_hid_get_descriptor; + shid->ops.output_report = shid_output_report; + shid->ops.get_feature_report = shid_get_feature_report; + shid->ops.set_feature_report = shid_set_feature_report; + + ssam_device_set_drvdata(sdev, shid); + return surface_hid_device_add(shid); +} + +static void surface_hid_remove(struct ssam_device *sdev) +{ + surface_hid_device_destroy(ssam_device_get_drvdata(sdev)); +} + +static const struct ssam_device_id surface_hid_match[] = { + { SSAM_SDEV(HID, 0x02, SSAM_ANY_IID, 0x00) }, + { }, +}; +MODULE_DEVICE_TABLE(ssam, surface_hid_match); + +static struct ssam_device_driver surface_hid_driver = { + .probe = surface_hid_probe, + .remove = surface_hid_remove, + .match_table = surface_hid_match, + .driver = { + .name = "surface_hid", + .pm = &surface_hid_pm_ops, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; +module_ssam_device_driver(surface_hid_driver); + +MODULE_AUTHOR("Blaž Hrastnik <blaz@mxxn.io>"); +MODULE_AUTHOR("Maximilian Luz <luzmaximilian@gmail.com>"); +MODULE_DESCRIPTION("HID transport driver for Surface System Aggregator Module"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/surface-hid/surface_hid_core.c b/drivers/hid/surface-hid/surface_hid_core.c new file mode 100644 index 000000000000..7b27ec392232 --- /dev/null +++ b/drivers/hid/surface-hid/surface_hid_core.c @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Common/core components for the Surface System Aggregator Module (SSAM) HID + * transport driver. Provides support for integrated HID devices on Microsoft + * Surface models. + * + * Copyright (C) 2019-2021 Maximilian Luz <luzmaximilian@gmail.com> + */ + +#include <asm/unaligned.h> +#include <linux/hid.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/types.h> +#include <linux/usb/ch9.h> + +#include <linux/surface_aggregator/controller.h> + +#include "surface_hid_core.h" + + +/* -- Device descriptor access. --------------------------------------------- */ + +static int surface_hid_load_hid_descriptor(struct surface_hid_device *shid) +{ + int status; + + status = shid->ops.get_descriptor(shid, SURFACE_HID_DESC_HID, + (u8 *)&shid->hid_desc, sizeof(shid->hid_desc)); + if (status) + return status; + + if (shid->hid_desc.desc_len != sizeof(shid->hid_desc)) { + dev_err(shid->dev, "unexpected HID descriptor length: got %u, expected %zu\n", + shid->hid_desc.desc_len, sizeof(shid->hid_desc)); + return -EPROTO; + } + + if (shid->hid_desc.desc_type != HID_DT_HID) { + dev_err(shid->dev, "unexpected HID descriptor type: got %#04x, expected %#04x\n", + shid->hid_desc.desc_type, HID_DT_HID); + return -EPROTO; + } + + if (shid->hid_desc.num_descriptors != 1) { + dev_err(shid->dev, "unexpected number of descriptors: got %u, expected 1\n", + shid->hid_desc.num_descriptors); + return -EPROTO; + } + + if (shid->hid_desc.report_desc_type != HID_DT_REPORT) { + dev_err(shid->dev, "unexpected report descriptor type: got %#04x, expected %#04x\n", + shid->hid_desc.report_desc_type, HID_DT_REPORT); + return -EPROTO; + } + + return 0; +} + +static int surface_hid_load_device_attributes(struct surface_hid_device *shid) +{ + int status; + + status = shid->ops.get_descriptor(shid, SURFACE_HID_DESC_ATTRS, + (u8 *)&shid->attrs, sizeof(shid->attrs)); + if (status) + return status; + + if (get_unaligned_le32(&shid->attrs.length) != sizeof(shid->attrs)) { + dev_err(shid->dev, "unexpected attribute length: got %u, expected %zu\n", + get_unaligned_le32(&shid->attrs.length), sizeof(shid->attrs)); + return -EPROTO; + } + + return 0; +} + + +/* -- Transport driver (common). -------------------------------------------- */ + +static int surface_hid_start(struct hid_device *hid) +{ + struct surface_hid_device *shid = hid->driver_data; + + return ssam_notifier_register(shid->ctrl, &shid->notif); +} + +static void surface_hid_stop(struct hid_device *hid) +{ + struct surface_hid_device *shid = hid->driver_data; + + /* Note: This call will log errors for us, so ignore them here. */ + ssam_notifier_unregister(shid->ctrl, &shid->notif); +} + +static int surface_hid_open(struct hid_device *hid) +{ + return 0; +} + +static void surface_hid_close(struct hid_device *hid) +{ +} + +static int surface_hid_parse(struct hid_device *hid) +{ + struct surface_hid_device *shid = hid->driver_data; + size_t len = get_unaligned_le16(&shid->hid_desc.report_desc_len); + u8 *buf; + int status; + + buf = kzalloc(len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + status = shid->ops.get_descriptor(shid, SURFACE_HID_DESC_REPORT, buf, len); + if (!status) + status = hid_parse_report(hid, buf, len); + + kfree(buf); + return status; +} + +static int surface_hid_raw_request(struct hid_device *hid, unsigned char reportnum, u8 *buf, + size_t len, unsigned char rtype, int reqtype) +{ + struct surface_hid_device *shid = hid->driver_data; + + if (rtype == HID_OUTPUT_REPORT && reqtype == HID_REQ_SET_REPORT) + return shid->ops.output_report(shid, reportnum, buf, len); + + else if (rtype == HID_FEATURE_REPORT && reqtype == HID_REQ_GET_REPORT) + return shid->ops.get_feature_report(shid, reportnum, buf, len); + + else if (rtype == HID_FEATURE_REPORT && reqtype == HID_REQ_SET_REPORT) + return shid->ops.set_feature_report(shid, reportnum, buf, len); + + return -EIO; +} + +static struct hid_ll_driver surface_hid_ll_driver = { + .start = surface_hid_start, + .stop = surface_hid_stop, + .open = surface_hid_open, + .close = surface_hid_close, + .parse = surface_hid_parse, + .raw_request = surface_hid_raw_request, +}; + + +/* -- Common device setup. -------------------------------------------------- */ + +int surface_hid_device_add(struct surface_hid_device *shid) +{ + int status; + + status = surface_hid_load_hid_descriptor(shid); + if (status) + return status; + + status = surface_hid_load_device_attributes(shid); + if (status) + return status; + + shid->hid = hid_allocate_device(); + if (IS_ERR(shid->hid)) + return PTR_ERR(shid->hid); + + shid->hid->dev.parent = shid->dev; + shid->hid->bus = BUS_HOST; + shid->hid->vendor = cpu_to_le16(shid->attrs.vendor); + shid->hid->product = cpu_to_le16(shid->attrs.product); + shid->hid->version = cpu_to_le16(shid->hid_desc.hid_version); + shid->hid->country = shid->hid_desc.country_code; + + snprintf(shid->hid->name, sizeof(shid->hid->name), "Microsoft Surface %04X:%04X", + shid->hid->vendor, shid->hid->product); + + strscpy(shid->hid->phys, dev_name(shid->dev), sizeof(shid->hid->phys)); + + shid->hid->driver_data = shid; + shid->hid->ll_driver = &surface_hid_ll_driver; + + status = hid_add_device(shid->hid); + if (status) + hid_destroy_device(shid->hid); + + return status; +} +EXPORT_SYMBOL_GPL(surface_hid_device_add); + +void surface_hid_device_destroy(struct surface_hid_device *shid) +{ + hid_destroy_device(shid->hid); +} +EXPORT_SYMBOL_GPL(surface_hid_device_destroy); + + +/* -- PM ops. --------------------------------------------------------------- */ + +#ifdef CONFIG_PM_SLEEP + +static int surface_hid_suspend(struct device *dev) +{ + struct surface_hid_device *d = dev_get_drvdata(dev); + + if (d->hid->driver && d->hid->driver->suspend) + return d->hid->driver->suspend(d->hid, PMSG_SUSPEND); + + return 0; +} + +static int surface_hid_resume(struct device *dev) +{ + struct surface_hid_device *d = dev_get_drvdata(dev); + + if (d->hid->driver && d->hid->driver->resume) + return d->hid->driver->resume(d->hid); + + return 0; +} + +static int surface_hid_freeze(struct device *dev) +{ + struct surface_hid_device *d = dev_get_drvdata(dev); + + if (d->hid->driver && d->hid->driver->suspend) + return d->hid->driver->suspend(d->hid, PMSG_FREEZE); + + return 0; +} + +static int surface_hid_poweroff(struct device *dev) +{ + struct surface_hid_device *d = dev_get_drvdata(dev); + + if (d->hid->driver && d->hid->driver->suspend) + return d->hid->driver->suspend(d->hid, PMSG_HIBERNATE); + + return 0; +} + +static int surface_hid_restore(struct device *dev) +{ + struct surface_hid_device *d = dev_get_drvdata(dev); + + if (d->hid->driver && d->hid->driver->reset_resume) + return d->hid->driver->reset_resume(d->hid); + + return 0; +} + +const struct dev_pm_ops surface_hid_pm_ops = { + .freeze = surface_hid_freeze, + .thaw = surface_hid_resume, + .suspend = surface_hid_suspend, + .resume = surface_hid_resume, + .poweroff = surface_hid_poweroff, + .restore = surface_hid_restore, +}; +EXPORT_SYMBOL_GPL(surface_hid_pm_ops); + +#else /* CONFIG_PM_SLEEP */ + +const struct dev_pm_ops surface_hid_pm_ops = { }; +EXPORT_SYMBOL_GPL(surface_hid_pm_ops); + +#endif /* CONFIG_PM_SLEEP */ + +MODULE_AUTHOR("Maximilian Luz <luzmaximilian@gmail.com>"); +MODULE_DESCRIPTION("HID transport driver core for Surface System Aggregator Module"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/surface-hid/surface_hid_core.h b/drivers/hid/surface-hid/surface_hid_core.h new file mode 100644 index 000000000000..4b1a7b57e035 --- /dev/null +++ b/drivers/hid/surface-hid/surface_hid_core.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Common/core components for the Surface System Aggregator Module (SSAM) HID + * transport driver. Provides support for integrated HID devices on Microsoft + * Surface models. + * + * Copyright (C) 2019-2021 Maximilian Luz <luzmaximilian@gmail.com> + */ + +#ifndef SURFACE_HID_CORE_H +#define SURFACE_HID_CORE_H + +#include <linux/hid.h> +#include <linux/pm.h> +#include <linux/types.h> + +#include <linux/surface_aggregator/controller.h> +#include <linux/surface_aggregator/device.h> + +enum surface_hid_descriptor_entry { + SURFACE_HID_DESC_HID = 0, + SURFACE_HID_DESC_REPORT = 1, + SURFACE_HID_DESC_ATTRS = 2, +}; + +struct surface_hid_descriptor { + __u8 desc_len; /* = 9 */ + __u8 desc_type; /* = HID_DT_HID */ + __le16 hid_version; + __u8 country_code; + __u8 num_descriptors; /* = 1 */ + + __u8 report_desc_type; /* = HID_DT_REPORT */ + __le16 report_desc_len; +} __packed; + +static_assert(sizeof(struct surface_hid_descriptor) == 9); + +struct surface_hid_attributes { + __le32 length; + __le16 vendor; + __le16 product; + __le16 version; + __u8 _unknown[22]; +} __packed; + +static_assert(sizeof(struct surface_hid_attributes) == 32); + +struct surface_hid_device; + +struct surface_hid_device_ops { + int (*get_descriptor)(struct surface_hid_device *shid, u8 entry, u8 *buf, size_t len); + int (*output_report)(struct surface_hid_device *shid, u8 rprt_id, u8 *buf, size_t len); + int (*get_feature_report)(struct surface_hid_device *shid, u8 rprt_id, u8 *buf, size_t len); + int (*set_feature_report)(struct surface_hid_device *shid, u8 rprt_id, u8 *buf, size_t len); +}; + +struct surface_hid_device { + struct device *dev; + struct ssam_controller *ctrl; + struct ssam_device_uid uid; + + struct surface_hid_descriptor hid_desc; + struct surface_hid_attributes attrs; + + struct ssam_event_notifier notif; + struct hid_device *hid; + + struct surface_hid_device_ops ops; +}; + +int surface_hid_device_add(struct surface_hid_device *shid); +void surface_hid_device_destroy(struct surface_hid_device *shid); + +extern const struct dev_pm_ops surface_hid_pm_ops; + +#endif /* SURFACE_HID_CORE_H */ diff --git a/drivers/hid/surface-hid/surface_kbd.c b/drivers/hid/surface-hid/surface_kbd.c new file mode 100644 index 000000000000..0635341bc517 --- /dev/null +++ b/drivers/hid/surface-hid/surface_kbd.c @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Surface System Aggregator Module (SSAM) HID transport driver for the legacy + * keyboard interface (KBD/TC=0x08 subsystem). Provides support for the + * integrated HID keyboard on Surface Laptops 1 and 2. + * + * Copyright (C) 2019-2021 Maximilian Luz <luzmaximilian@gmail.com> + */ + +#include <asm/unaligned.h> +#include <linux/hid.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/types.h> + +#include <linux/surface_aggregator/controller.h> + +#include "surface_hid_core.h" + + +/* -- SAM interface (KBD). -------------------------------------------------- */ + +#define KBD_FEATURE_REPORT_SIZE 7 /* 6 + report ID */ + +enum surface_kbd_cid { + SURFACE_KBD_CID_GET_DESCRIPTOR = 0x00, + SURFACE_KBD_CID_SET_CAPSLOCK_LED = 0x01, + SURFACE_KBD_CID_EVT_INPUT_GENERIC = 0x03, + SURFACE_KBD_CID_EVT_INPUT_HOTKEYS = 0x04, + SURFACE_KBD_CID_GET_FEATURE_REPORT = 0x0b, +}; + +static int ssam_kbd_get_descriptor(struct surface_hid_device *shid, u8 entry, u8 *buf, size_t len) +{ + struct ssam_request rqst; + struct ssam_response rsp; + int status; + + rqst.target_category = shid->uid.category; + rqst.target_id = shid->uid.target; + rqst.command_id = SURFACE_KBD_CID_GET_DESCRIPTOR; + rqst.instance_id = shid->uid.instance; + rqst.flags = SSAM_REQUEST_HAS_RESPONSE; + rqst.length = sizeof(entry); + rqst.payload = &entry; + + rsp.capacity = len; + rsp.length = 0; + rsp.pointer = buf; + + status = ssam_retry(ssam_request_sync_onstack, shid->ctrl, &rqst, &rsp, sizeof(entry)); + if (status) + return status; + + if (rsp.length != len) { + dev_err(shid->dev, "invalid descriptor length: got %zu, expected, %zu\n", + rsp.length, len); + return -EPROTO; + } + + return 0; +} + +static int ssam_kbd_set_caps_led(struct surface_hid_device *shid, bool value) +{ + struct ssam_request rqst; + u8 value_u8 = value; + + rqst.target_category = shid->uid.category; + rqst.target_id = shid->uid.target; + rqst.command_id = SURFACE_KBD_CID_SET_CAPSLOCK_LED; + rqst.instance_id = shid->uid.instance; + rqst.flags = 0; + rqst.length = sizeof(value_u8); + rqst.payload = &value_u8; + + return ssam_retry(ssam_request_sync_onstack, shid->ctrl, &rqst, NULL, sizeof(value_u8)); +} + +static int ssam_kbd_get_feature_report(struct surface_hid_device *shid, u8 *buf, size_t len) +{ + struct ssam_request rqst; + struct ssam_response rsp; + u8 payload = 0; + int status; + + rqst.target_category = shid->uid.category; + rqst.target_id = shid->uid.target; + rqst.command_id = SURFACE_KBD_CID_GET_FEATURE_REPORT; + rqst.instance_id = shid->uid.instance; + rqst.flags = SSAM_REQUEST_HAS_RESPONSE; + rqst.length = sizeof(payload); + rqst.payload = &payload; + + rsp.capacity = len; + rsp.length = 0; + rsp.pointer = buf; + + status = ssam_retry(ssam_request_sync_onstack, shid->ctrl, &rqst, &rsp, sizeof(payload)); + if (status) + return status; + + if (rsp.length != len) { + dev_err(shid->dev, "invalid feature report length: got %zu, expected, %zu\n", + rsp.length, len); + return -EPROTO; + } + + return 0; +} + +static bool ssam_kbd_is_input_event(const struct ssam_event *event) +{ + if (event->command_id == SURFACE_KBD_CID_EVT_INPUT_GENERIC) + return true; + + if (event->command_id == SURFACE_KBD_CID_EVT_INPUT_HOTKEYS) + return true; + + return false; +} + +static u32 ssam_kbd_event_fn(struct ssam_event_notifier *nf, const struct ssam_event *event) +{ + struct surface_hid_device *shid = container_of(nf, struct surface_hid_device, notif); + + /* + * Check against device UID manually, as registry and device target + * category doesn't line up. + */ + + if (shid->uid.category != event->target_category) + return 0; + + if (shid->uid.target != event->target_id) + return 0; + + if (shid->uid.instance != event->instance_id) + return 0; + + if (!ssam_kbd_is_input_event(event)) + return 0; + + hid_input_report(shid->hid, HID_INPUT_REPORT, (u8 *)&event->data[0], event->length, 0); + return SSAM_NOTIF_HANDLED; +} + + +/* -- Transport driver (KBD). ----------------------------------------------- */ + +static int skbd_get_caps_led_value(struct hid_device *hid, u8 rprt_id, u8 *buf, size_t len) +{ + struct hid_field *field; + unsigned int offset, size; + int i; + + /* Get LED field. */ + field = hidinput_get_led_field(hid); + if (!field) + return -ENOENT; + + /* Check if we got the correct report. */ + if (len != hid_report_len(field->report)) + return -ENOENT; + + if (rprt_id != field->report->id) + return -ENOENT; + + /* Get caps lock LED index. */ + for (i = 0; i < field->report_count; i++) + if ((field->usage[i].hid & 0xffff) == 0x02) + break; + + if (i == field->report_count) + return -ENOENT; + + /* Extract value. */ + size = field->report_size; + offset = field->report_offset + i * size; + return !!hid_field_extract(hid, buf + 1, size, offset); +} + +static int skbd_output_report(struct surface_hid_device *shid, u8 rprt_id, u8 *buf, size_t len) +{ + int caps_led; + int status; + + caps_led = skbd_get_caps_led_value(shid->hid, rprt_id, buf, len); + if (caps_led < 0) + return -EIO; /* Only caps LED output reports are supported. */ + + status = ssam_kbd_set_caps_led(shid, caps_led); + if (status < 0) + return status; + + return len; +} + +static int skbd_get_feature_report(struct surface_hid_device *shid, u8 rprt_id, u8 *buf, size_t len) +{ + u8 report[KBD_FEATURE_REPORT_SIZE]; + int status; + + /* + * The keyboard only has a single hard-coded read-only feature report + * of size KBD_FEATURE_REPORT_SIZE. Try to load it and compare its + * report ID against the requested one. + */ + + if (len < ARRAY_SIZE(report)) + return -ENOSPC; + + status = ssam_kbd_get_feature_report(shid, report, ARRAY_SIZE(report)); + if (status < 0) + return status; + + if (rprt_id != report[0]) + return -ENOENT; + + memcpy(buf, report, ARRAY_SIZE(report)); + return len; +} + +static int skbd_set_feature_report(struct surface_hid_device *shid, u8 rprt_id, u8 *buf, size_t len) +{ + /* Not supported. See skbd_get_feature_report() for details. */ + return -EIO; +} + + +/* -- Driver setup. --------------------------------------------------------- */ + +static int surface_kbd_probe(struct platform_device *pdev) +{ + struct ssam_controller *ctrl; + struct surface_hid_device *shid; + + /* Add device link to EC. */ + ctrl = ssam_client_bind(&pdev->dev); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl) == -ENODEV ? -EPROBE_DEFER : PTR_ERR(ctrl); + + shid = devm_kzalloc(&pdev->dev, sizeof(*shid), GFP_KERNEL); + if (!shid) + return -ENOMEM; + + shid->dev = &pdev->dev; + shid->ctrl = ctrl; + + shid->uid.domain = SSAM_DOMAIN_SERIALHUB; + shid->uid.category = SSAM_SSH_TC_KBD; + shid->uid.target = 2; + shid->uid.instance = 0; + shid->uid.function = 0; + + shid->notif.base.priority = 1; + shid->notif.base.fn = ssam_kbd_event_fn; + shid->notif.event.reg = SSAM_EVENT_REGISTRY_SAM; + shid->notif.event.id.target_category = shid->uid.category; + shid->notif.event.id.instance = shid->uid.instance; + shid->notif.event.mask = SSAM_EVENT_MASK_NONE; + shid->notif.event.flags = 0; + + shid->ops.get_descriptor = ssam_kbd_get_descriptor; + shid->ops.output_report = skbd_output_report; + shid->ops.get_feature_report = skbd_get_feature_report; + shid->ops.set_feature_report = skbd_set_feature_report; + + platform_set_drvdata(pdev, shid); + return surface_hid_device_add(shid); +} + +static int surface_kbd_remove(struct platform_device *pdev) +{ + surface_hid_device_destroy(platform_get_drvdata(pdev)); + return 0; +} + +static const struct acpi_device_id surface_kbd_match[] = { + { "MSHW0096" }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, surface_kbd_match); + +static struct platform_driver surface_kbd_driver = { + .probe = surface_kbd_probe, + .remove = surface_kbd_remove, + .driver = { + .name = "surface_keyboard", + .acpi_match_table = surface_kbd_match, + .pm = &surface_hid_pm_ops, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; +module_platform_driver(surface_kbd_driver); + +MODULE_AUTHOR("Maximilian Luz <luzmaximilian@gmail.com>"); +MODULE_DESCRIPTION("HID legacy transport driver for Surface System Aggregator Module"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c index 8c471823a5af..2f776d78e3c1 100644 --- a/drivers/hv/hv_balloon.c +++ b/drivers/hv/hv_balloon.c @@ -726,7 +726,7 @@ static void hv_mem_hot_add(unsigned long start, unsigned long size, nid = memory_add_physaddr_to_nid(PFN_PHYS(start_pfn)); ret = add_memory(nid, PFN_PHYS((start_pfn)), - (HA_CHUNK << PAGE_SHIFT), MEMHP_MERGE_RESOURCE); + (HA_CHUNK << PAGE_SHIFT), MHP_MERGE_RESOURCE); if (ret) { pr_err("hot_add memory failed error is %d\n", ret); diff --git a/drivers/hwspinlock/omap_hwspinlock.c b/drivers/hwspinlock/omap_hwspinlock.c index 3b05560456ea..33eeff94fc2a 100644 --- a/drivers/hwspinlock/omap_hwspinlock.c +++ b/drivers/hwspinlock/omap_hwspinlock.c @@ -2,11 +2,12 @@ /* * OMAP hardware spinlock driver * - * Copyright (C) 2010-2015 Texas Instruments Incorporated - http://www.ti.com + * Copyright (C) 2010-2021 Texas Instruments Incorporated - https://www.ti.com * * Contact: Simon Que <sque@ti.com> * Hari Kanigeri <h-kanigeri2@ti.com> * Ohad Ben-Cohen <ohad@wizery.com> + * Suman Anna <s-anna@ti.com> */ #include <linux/kernel.h> @@ -164,6 +165,7 @@ static int omap_hwspinlock_remove(struct platform_device *pdev) static const struct of_device_id omap_hwspinlock_of_match[] = { { .compatible = "ti,omap4-hwspinlock", }, + { .compatible = "ti,am64-hwspinlock", }, { .compatible = "ti,am654-hwspinlock", }, { /* end */ }, }; diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c index 8e19e8cdcce5..e0740c6dbd54 100644 --- a/drivers/hwtracing/coresight/coresight-catu.c +++ b/drivers/hwtracing/coresight/coresight-catu.c @@ -401,8 +401,9 @@ static const struct attribute_group *catu_groups[] = { static inline int catu_wait_for_ready(struct catu_drvdata *drvdata) { - return coresight_timeout(drvdata->base, - CATU_STATUS, CATU_STATUS_READY, 1); + struct csdev_access *csa = &drvdata->csdev->access; + + return coresight_timeout(csa, CATU_STATUS, CATU_STATUS_READY, 1); } static int catu_enable_hw(struct catu_drvdata *drvdata, void *data) @@ -411,6 +412,7 @@ static int catu_enable_hw(struct catu_drvdata *drvdata, void *data) u32 control, mode; struct etr_buf *etr_buf = data; struct device *dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; if (catu_wait_for_ready(drvdata)) dev_warn(dev, "Timeout while waiting for READY\n"); @@ -421,7 +423,7 @@ static int catu_enable_hw(struct catu_drvdata *drvdata, void *data) return -EBUSY; } - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (rc) return rc; @@ -465,9 +467,10 @@ static int catu_disable_hw(struct catu_drvdata *drvdata) { int rc = 0; struct device *dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; catu_write_control(drvdata, 0); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); if (catu_wait_for_ready(drvdata)) { dev_info(dev, "Timeout while waiting for READY\n"); rc = -EAGAIN; @@ -551,6 +554,7 @@ static int catu_probe(struct amba_device *adev, const struct amba_id *id) dev->platform_data = pdata; drvdata->base = base; + catu_desc.access = CSDEV_ACCESS_IOMEM(base); catu_desc.pdata = pdata; catu_desc.dev = dev; catu_desc.groups = catu_groups; diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 4ba801dffcb7..0062c8935653 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -145,30 +145,32 @@ static int coresight_find_link_outport(struct coresight_device *csdev, return -ENODEV; } -static inline u32 coresight_read_claim_tags(void __iomem *base) +static inline u32 coresight_read_claim_tags(struct coresight_device *csdev) { - return readl_relaxed(base + CORESIGHT_CLAIMCLR); + return csdev_access_relaxed_read32(&csdev->access, CORESIGHT_CLAIMCLR); } -static inline bool coresight_is_claimed_self_hosted(void __iomem *base) +static inline bool coresight_is_claimed_self_hosted(struct coresight_device *csdev) { - return coresight_read_claim_tags(base) == CORESIGHT_CLAIM_SELF_HOSTED; + return coresight_read_claim_tags(csdev) == CORESIGHT_CLAIM_SELF_HOSTED; } -static inline bool coresight_is_claimed_any(void __iomem *base) +static inline bool coresight_is_claimed_any(struct coresight_device *csdev) { - return coresight_read_claim_tags(base) != 0; + return coresight_read_claim_tags(csdev) != 0; } -static inline void coresight_set_claim_tags(void __iomem *base) +static inline void coresight_set_claim_tags(struct coresight_device *csdev) { - writel_relaxed(CORESIGHT_CLAIM_SELF_HOSTED, base + CORESIGHT_CLAIMSET); + csdev_access_relaxed_write32(&csdev->access, CORESIGHT_CLAIM_SELF_HOSTED, + CORESIGHT_CLAIMSET); isb(); } -static inline void coresight_clear_claim_tags(void __iomem *base) +static inline void coresight_clear_claim_tags(struct coresight_device *csdev) { - writel_relaxed(CORESIGHT_CLAIM_SELF_HOSTED, base + CORESIGHT_CLAIMCLR); + csdev_access_relaxed_write32(&csdev->access, CORESIGHT_CLAIM_SELF_HOSTED, + CORESIGHT_CLAIMCLR); isb(); } @@ -182,27 +184,33 @@ static inline void coresight_clear_claim_tags(void __iomem *base) * Called with CS_UNLOCKed for the component. * Returns : 0 on success */ -int coresight_claim_device_unlocked(void __iomem *base) +int coresight_claim_device_unlocked(struct coresight_device *csdev) { - if (coresight_is_claimed_any(base)) + if (WARN_ON(!csdev)) + return -EINVAL; + + if (coresight_is_claimed_any(csdev)) return -EBUSY; - coresight_set_claim_tags(base); - if (coresight_is_claimed_self_hosted(base)) + coresight_set_claim_tags(csdev); + if (coresight_is_claimed_self_hosted(csdev)) return 0; /* There was a race setting the tags, clean up and fail */ - coresight_clear_claim_tags(base); + coresight_clear_claim_tags(csdev); return -EBUSY; } EXPORT_SYMBOL_GPL(coresight_claim_device_unlocked); -int coresight_claim_device(void __iomem *base) +int coresight_claim_device(struct coresight_device *csdev) { int rc; - CS_UNLOCK(base); - rc = coresight_claim_device_unlocked(base); - CS_LOCK(base); + if (WARN_ON(!csdev)) + return -EINVAL; + + CS_UNLOCK(csdev->access.base); + rc = coresight_claim_device_unlocked(csdev); + CS_LOCK(csdev->access.base); return rc; } @@ -212,11 +220,14 @@ EXPORT_SYMBOL_GPL(coresight_claim_device); * coresight_disclaim_device_unlocked : Clear the claim tags for the device. * Called with CS_UNLOCKed for the component. */ -void coresight_disclaim_device_unlocked(void __iomem *base) +void coresight_disclaim_device_unlocked(struct coresight_device *csdev) { - if (coresight_is_claimed_self_hosted(base)) - coresight_clear_claim_tags(base); + if (WARN_ON(!csdev)) + return; + + if (coresight_is_claimed_self_hosted(csdev)) + coresight_clear_claim_tags(csdev); else /* * The external agent may have not honoured our claim @@ -227,11 +238,14 @@ void coresight_disclaim_device_unlocked(void __iomem *base) } EXPORT_SYMBOL_GPL(coresight_disclaim_device_unlocked); -void coresight_disclaim_device(void __iomem *base) +void coresight_disclaim_device(struct coresight_device *csdev) { - CS_UNLOCK(base); - coresight_disclaim_device_unlocked(base); - CS_LOCK(base); + if (WARN_ON(!csdev)) + return; + + CS_UNLOCK(csdev->access.base); + coresight_disclaim_device_unlocked(csdev); + CS_LOCK(csdev->access.base); } EXPORT_SYMBOL_GPL(coresight_disclaim_device); @@ -1418,23 +1432,24 @@ static void coresight_remove_conns(struct coresight_device *csdev) } /** - * coresight_timeout - loop until a bit has changed to a specific state. - * @addr: base address of the area of interest. - * @offset: address of a register, starting from @addr. + * coresight_timeout - loop until a bit has changed to a specific register + * state. + * @csa: coresight device access for the device + * @offset: Offset of the register from the base of the device. * @position: the position of the bit of interest. * @value: the value the bit should have. * * Return: 0 as soon as the bit has taken the desired state or -EAGAIN if * TIMEOUT_US has elapsed, which ever happens first. */ - -int coresight_timeout(void __iomem *addr, u32 offset, int position, int value) +int coresight_timeout(struct csdev_access *csa, u32 offset, + int position, int value) { int i; u32 val; for (i = TIMEOUT_US; i > 0; i--) { - val = __raw_readl(addr + offset); + val = csdev_access_read32(csa, offset); /* waiting on the bit to go from 0 to 1 */ if (value) { if (val & BIT(position)) @@ -1458,6 +1473,48 @@ int coresight_timeout(void __iomem *addr, u32 offset, int position, int value) } EXPORT_SYMBOL_GPL(coresight_timeout); +u32 coresight_relaxed_read32(struct coresight_device *csdev, u32 offset) +{ + return csdev_access_relaxed_read32(&csdev->access, offset); +} + +u32 coresight_read32(struct coresight_device *csdev, u32 offset) +{ + return csdev_access_read32(&csdev->access, offset); +} + +void coresight_relaxed_write32(struct coresight_device *csdev, + u32 val, u32 offset) +{ + csdev_access_relaxed_write32(&csdev->access, val, offset); +} + +void coresight_write32(struct coresight_device *csdev, u32 val, u32 offset) +{ + csdev_access_write32(&csdev->access, val, offset); +} + +u64 coresight_relaxed_read64(struct coresight_device *csdev, u32 offset) +{ + return csdev_access_relaxed_read64(&csdev->access, offset); +} + +u64 coresight_read64(struct coresight_device *csdev, u32 offset) +{ + return csdev_access_read64(&csdev->access, offset); +} + +void coresight_relaxed_write64(struct coresight_device *csdev, + u64 val, u32 offset) +{ + csdev_access_relaxed_write64(&csdev->access, val, offset); +} + +void coresight_write64(struct coresight_device *csdev, u64 val, u32 offset) +{ + csdev_access_write64(&csdev->access, val, offset); +} + /* * coresight_release_platform_data: Release references to the devices connected * to the output port of this device. @@ -1522,6 +1579,7 @@ struct coresight_device *coresight_register(struct coresight_desc *desc) csdev->type = desc->type; csdev->subtype = desc->subtype; csdev->ops = desc->ops; + csdev->access = desc->access; csdev->orphan = false; csdev->dev.type = &coresight_dev_type[desc->type]; diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index 30e48809ba00..e2a3620cbf48 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -102,7 +102,7 @@ static int cti_enable_hw(struct cti_drvdata *drvdata) goto cti_state_unchanged; /* claim the device */ - rc = coresight_claim_device(drvdata->base); + rc = coresight_claim_device(drvdata->csdev); if (rc) goto cti_err_not_enabled; @@ -136,7 +136,7 @@ static void cti_cpuhp_enable_hw(struct cti_drvdata *drvdata) goto cti_hp_not_enabled; /* try to claim the device */ - if (coresight_claim_device(drvdata->base)) + if (coresight_claim_device(drvdata->csdev)) goto cti_hp_not_enabled; cti_write_all_hw_regs(drvdata); @@ -154,6 +154,7 @@ static int cti_disable_hw(struct cti_drvdata *drvdata) { struct cti_config *config = &drvdata->config; struct device *dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; spin_lock(&drvdata->spinlock); @@ -171,7 +172,7 @@ static int cti_disable_hw(struct cti_drvdata *drvdata) writel_relaxed(0, drvdata->base + CTICONTROL); config->hw_enabled = false; - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); spin_unlock(&drvdata->spinlock); pm_runtime_put(dev); @@ -655,6 +656,7 @@ static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, void *v) { struct cti_drvdata *drvdata; + struct coresight_device *csdev; unsigned int cpu = smp_processor_id(); int notify_res = NOTIFY_OK; @@ -662,6 +664,7 @@ static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, return NOTIFY_OK; drvdata = cti_cpu_drvdata[cpu]; + csdev = drvdata->csdev; if (WARN_ON_ONCE(drvdata->ctidev.cpu != cpu)) return NOTIFY_BAD; @@ -673,13 +676,13 @@ static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, /* CTI regs all static - we have a copy & nothing to save */ drvdata->config.hw_powered = false; if (drvdata->config.hw_enabled) - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(csdev); break; case CPU_PM_ENTER_FAILED: drvdata->config.hw_powered = true; if (drvdata->config.hw_enabled) { - if (coresight_claim_device(drvdata->base)) + if (coresight_claim_device(csdev)) drvdata->config.hw_enabled = false; } break; @@ -692,7 +695,7 @@ static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, /* check enable reference count to enable HW */ if (atomic_read(&drvdata->config.enable_req_count)) { /* check we can claim the device as we re-power */ - if (coresight_claim_device(drvdata->base)) + if (coresight_claim_device(csdev)) goto cti_notify_exit; drvdata->config.hw_enabled = true; @@ -736,7 +739,7 @@ static int cti_dying_cpu(unsigned int cpu) spin_lock(&drvdata->spinlock); drvdata->config.hw_powered = false; if (drvdata->config.hw_enabled) - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(drvdata->csdev); spin_unlock(&drvdata->spinlock); return 0; } @@ -868,6 +871,7 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + cti_desc.access = CSDEV_ACCESS_IOMEM(base); dev_set_drvdata(dev, drvdata); diff --git a/drivers/hwtracing/coresight/coresight-cti-platform.c b/drivers/hwtracing/coresight/coresight-cti-platform.c index 98f830c6ed50..ccef04f27f12 100644 --- a/drivers/hwtracing/coresight/coresight-cti-platform.c +++ b/drivers/hwtracing/coresight/coresight-cti-platform.c @@ -343,7 +343,6 @@ static int cti_plat_create_connection(struct device *dev, { struct cti_trig_con *tc = NULL; int cpuid = -1, err = 0; - struct fwnode_handle *cs_fwnode = NULL; struct coresight_device *csdev = NULL; const char *assoc_name = "unknown"; char cpu_name_str[16]; @@ -397,8 +396,9 @@ static int cti_plat_create_connection(struct device *dev, assoc_name = cpu_name_str; } else { /* associated device ? */ - cs_fwnode = fwnode_find_reference(fwnode, - CTI_DT_CSDEV_ASSOC, 0); + struct fwnode_handle *cs_fwnode = fwnode_find_reference(fwnode, + CTI_DT_CSDEV_ASSOC, + 0); if (!IS_ERR(cs_fwnode)) { assoc_name = cti_plat_get_csdev_or_node_name(cs_fwnode, &csdev); diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index 51c801c05e5c..f775cbee12b8 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -132,7 +132,7 @@ static void __etb_enable_hw(struct etb_drvdata *drvdata) static int etb_enable_hw(struct etb_drvdata *drvdata) { - int rc = coresight_claim_device(drvdata->base); + int rc = coresight_claim_device(drvdata->csdev); if (rc) return rc; @@ -252,6 +252,7 @@ static void __etb_disable_hw(struct etb_drvdata *drvdata) { u32 ffcr; struct device *dev = &drvdata->csdev->dev; + struct csdev_access *csa = &drvdata->csdev->access; CS_UNLOCK(drvdata->base); @@ -263,7 +264,7 @@ static void __etb_disable_hw(struct etb_drvdata *drvdata) ffcr |= ETB_FFCR_FON_MAN; writel_relaxed(ffcr, drvdata->base + ETB_FFCR); - if (coresight_timeout(drvdata->base, ETB_FFCR, ETB_FFCR_BIT, 0)) { + if (coresight_timeout(csa, ETB_FFCR, ETB_FFCR_BIT, 0)) { dev_err(dev, "timeout while waiting for completion of Manual Flush\n"); } @@ -271,7 +272,7 @@ static void __etb_disable_hw(struct etb_drvdata *drvdata) /* disable trace capture */ writel_relaxed(0x0, drvdata->base + ETB_CTL_REG); - if (coresight_timeout(drvdata->base, ETB_FFSR, ETB_FFSR_BIT, 1)) { + if (coresight_timeout(csa, ETB_FFSR, ETB_FFSR_BIT, 1)) { dev_err(dev, "timeout while waiting for Formatter to Stop\n"); } @@ -344,7 +345,7 @@ static void etb_disable_hw(struct etb_drvdata *drvdata) { __etb_disable_hw(drvdata); etb_dump_hw(drvdata); - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(drvdata->csdev); } static int etb_disable(struct coresight_device *csdev) @@ -757,6 +758,7 @@ static int etb_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); spin_lock_init(&drvdata->spinlock); diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index bdc34ca449f7..0f603b4094f2 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -27,17 +27,45 @@ static bool etm_perf_up; static DEFINE_PER_CPU(struct perf_output_handle, ctx_handle); static DEFINE_PER_CPU(struct coresight_device *, csdev_src); -/* ETMv3.5/PTM's ETMCR is 'config' */ +/* + * The PMU formats were orignally for ETMv3.5/PTM's ETMCR 'config'; + * now take them as general formats and apply on all ETMs. + */ PMU_FORMAT_ATTR(cycacc, "config:" __stringify(ETM_OPT_CYCACC)); -PMU_FORMAT_ATTR(contextid, "config:" __stringify(ETM_OPT_CTXTID)); +/* contextid1 enables tracing CONTEXTIDR_EL1 for ETMv4 */ +PMU_FORMAT_ATTR(contextid1, "config:" __stringify(ETM_OPT_CTXTID)); +/* contextid2 enables tracing CONTEXTIDR_EL2 for ETMv4 */ +PMU_FORMAT_ATTR(contextid2, "config:" __stringify(ETM_OPT_CTXTID2)); PMU_FORMAT_ATTR(timestamp, "config:" __stringify(ETM_OPT_TS)); PMU_FORMAT_ATTR(retstack, "config:" __stringify(ETM_OPT_RETSTK)); /* Sink ID - same for all ETMs */ PMU_FORMAT_ATTR(sinkid, "config2:0-31"); +/* + * contextid always traces the "PID". The PID is in CONTEXTIDR_EL1 + * when the kernel is running at EL1; when the kernel is at EL2, + * the PID is in CONTEXTIDR_EL2. + */ +static ssize_t format_attr_contextid_show(struct device *dev, + struct device_attribute *attr, + char *page) +{ + int pid_fmt = ETM_OPT_CTXTID; + +#if defined(CONFIG_CORESIGHT_SOURCE_ETM4X) + pid_fmt = is_kernel_in_hyp_mode() ? ETM_OPT_CTXTID2 : ETM_OPT_CTXTID; +#endif + return sprintf(page, "config:%d\n", pid_fmt); +} + +struct device_attribute format_attr_contextid = + __ATTR(contextid, 0444, format_attr_contextid_show, NULL); + static struct attribute *etm_config_formats_attr[] = { &format_attr_cycacc.attr, &format_attr_contextid.attr, + &format_attr_contextid1.attr, + &format_attr_contextid2.attr, &format_attr_timestamp.attr, &format_attr_retstack.attr, &format_attr_sinkid.attr, diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index 683a69e88efd..cf64ce73a741 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -358,10 +358,11 @@ static int etm_enable_hw(struct etm_drvdata *drvdata) int i, rc; u32 etmcr; struct etm_config *config = &drvdata->config; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (rc) goto done; @@ -566,6 +567,7 @@ static void etm_disable_hw(void *info) int i; struct etm_drvdata *drvdata = info; struct etm_config *config = &drvdata->config; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); etm_set_prog(drvdata); @@ -577,7 +579,7 @@ static void etm_disable_hw(void *info) config->cntr_val[i] = etm_readl(drvdata, ETMCNTVRn(i)); etm_set_pwrdwn(drvdata); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); @@ -602,7 +604,7 @@ static void etm_disable_perf(struct coresight_device *csdev) * power down the tracer. */ etm_set_pwrdwn(drvdata); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } @@ -839,6 +841,7 @@ static int etm_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); spin_lock_init(&drvdata->spinlock); diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 82787cba537d..15016f757828 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -27,6 +27,7 @@ #include <linux/seq_file.h> #include <linux/uaccess.h> #include <linux/perf_event.h> +#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/property.h> @@ -59,32 +60,99 @@ static u64 etm4_get_access_type(struct etmv4_config *config); static enum cpuhp_state hp_online; -static void etm4_os_unlock(struct etmv4_drvdata *drvdata) +struct etm4_init_arg { + unsigned int pid; + struct etmv4_drvdata *drvdata; + struct csdev_access *csa; +}; + +/* + * Check if TRCSSPCICRn(i) is implemented for a given instance. + * + * TRCSSPCICRn is implemented only if : + * TRCSSPCICR<n> is present only if all of the following are true: + * TRCIDR4.NUMSSCC > n. + * TRCIDR4.NUMPC > 0b0000 . + * TRCSSCSR<n>.PC == 0b1 + */ +static inline bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n) +{ + return (n < drvdata->nr_ss_cmp) && + drvdata->nr_pe && + (drvdata->config.ss_status[n] & TRCSSCSRn_PC); +} + +u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit) +{ + u64 res = 0; + + switch (offset) { + ETM4x_READ_SYSREG_CASES(res) + default : + pr_warn_ratelimited("etm4x: trying to read unsupported register @%x\n", + offset); + } + + if (!_relaxed) + __iormb(res); /* Imitate the !relaxed I/O helpers */ + + return res; +} + +void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit) +{ + if (!_relaxed) + __iowmb(); /* Imitate the !relaxed I/O helpers */ + if (!_64bit) + val &= GENMASK(31, 0); + + switch (offset) { + ETM4x_WRITE_SYSREG_CASES(val) + default : + pr_warn_ratelimited("etm4x: trying to write to unsupported register @%x\n", + offset); + } +} + +static void etm4_os_unlock_csa(struct etmv4_drvdata *drvdata, struct csdev_access *csa) { /* Writing 0 to TRCOSLAR unlocks the trace registers */ - writel_relaxed(0x0, drvdata->base + TRCOSLAR); + etm4x_relaxed_write32(csa, 0x0, TRCOSLAR); drvdata->os_unlock = true; isb(); } +static void etm4_os_unlock(struct etmv4_drvdata *drvdata) +{ + if (!WARN_ON(!drvdata->csdev)) + etm4_os_unlock_csa(drvdata, &drvdata->csdev->access); + +} + static void etm4_os_lock(struct etmv4_drvdata *drvdata) { + if (WARN_ON(!drvdata->csdev)) + return; + /* Writing 0x1 to TRCOSLAR locks the trace registers */ - writel_relaxed(0x1, drvdata->base + TRCOSLAR); + etm4x_relaxed_write32(&drvdata->csdev->access, 0x1, TRCOSLAR); drvdata->os_unlock = false; isb(); } -static bool etm4_arch_supported(u8 arch) +static void etm4_cs_lock(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) { - /* Mask out the minor version number */ - switch (arch & 0xf0) { - case ETM_ARCH_V4: - break; - default: - return false; - } - return true; + /* Software Lock is only accessible via memory mapped interface */ + if (csa->io_mem) + CS_LOCK(csa->base); +} + +static void etm4_cs_unlock(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + if (csa->io_mem) + CS_UNLOCK(csa->base); } static int etm4_cpu_id(struct coresight_device *csdev) @@ -201,57 +269,64 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) { int i, rc; struct etmv4_config *config = &drvdata->config; - struct device *etm_dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; + struct device *etm_dev = &csdev->dev; + struct csdev_access *csa = &csdev->access; + - CS_UNLOCK(drvdata->base); + etm4_cs_unlock(drvdata, csa); etm4_enable_arch_specific(drvdata); etm4_os_unlock(drvdata); - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (rc) goto done; /* Disable the trace unit before programming trace registers */ - writel_relaxed(0, drvdata->base + TRCPRGCTLR); + etm4x_relaxed_write32(csa, 0, TRCPRGCTLR); + + /* + * If we use system instructions, we need to synchronize the + * write to the TRCPRGCTLR, before accessing the TRCSTATR. + * See ARM IHI0064F, section + * "4.3.7 Synchronization of register updates" + */ + if (!csa->io_mem) + isb(); /* wait for TRCSTATR.IDLE to go up */ - if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) dev_err(etm_dev, "timeout while waiting for Idle Trace Status\n"); if (drvdata->nr_pe) - writel_relaxed(config->pe_sel, drvdata->base + TRCPROCSELR); - writel_relaxed(config->cfg, drvdata->base + TRCCONFIGR); + etm4x_relaxed_write32(csa, config->pe_sel, TRCPROCSELR); + etm4x_relaxed_write32(csa, config->cfg, TRCCONFIGR); /* nothing specific implemented */ - writel_relaxed(0x0, drvdata->base + TRCAUXCTLR); - writel_relaxed(config->eventctrl0, drvdata->base + TRCEVENTCTL0R); - writel_relaxed(config->eventctrl1, drvdata->base + TRCEVENTCTL1R); - writel_relaxed(config->stall_ctrl, drvdata->base + TRCSTALLCTLR); - writel_relaxed(config->ts_ctrl, drvdata->base + TRCTSCTLR); - writel_relaxed(config->syncfreq, drvdata->base + TRCSYNCPR); - writel_relaxed(config->ccctlr, drvdata->base + TRCCCCTLR); - writel_relaxed(config->bb_ctrl, drvdata->base + TRCBBCTLR); - writel_relaxed(drvdata->trcid, drvdata->base + TRCTRACEIDR); - writel_relaxed(config->vinst_ctrl, drvdata->base + TRCVICTLR); - writel_relaxed(config->viiectlr, drvdata->base + TRCVIIECTLR); - writel_relaxed(config->vissctlr, - drvdata->base + TRCVISSCTLR); + etm4x_relaxed_write32(csa, 0x0, TRCAUXCTLR); + etm4x_relaxed_write32(csa, config->eventctrl0, TRCEVENTCTL0R); + etm4x_relaxed_write32(csa, config->eventctrl1, TRCEVENTCTL1R); + if (drvdata->stallctl) + etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR); + etm4x_relaxed_write32(csa, config->ts_ctrl, TRCTSCTLR); + etm4x_relaxed_write32(csa, config->syncfreq, TRCSYNCPR); + etm4x_relaxed_write32(csa, config->ccctlr, TRCCCCTLR); + etm4x_relaxed_write32(csa, config->bb_ctrl, TRCBBCTLR); + etm4x_relaxed_write32(csa, drvdata->trcid, TRCTRACEIDR); + etm4x_relaxed_write32(csa, config->vinst_ctrl, TRCVICTLR); + etm4x_relaxed_write32(csa, config->viiectlr, TRCVIIECTLR); + etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR); if (drvdata->nr_pe_cmp) - writel_relaxed(config->vipcssctlr, - drvdata->base + TRCVIPCSSCTLR); + etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR); for (i = 0; i < drvdata->nrseqstate - 1; i++) - writel_relaxed(config->seq_ctrl[i], - drvdata->base + TRCSEQEVRn(i)); - writel_relaxed(config->seq_rst, drvdata->base + TRCSEQRSTEVR); - writel_relaxed(config->seq_state, drvdata->base + TRCSEQSTR); - writel_relaxed(config->ext_inp, drvdata->base + TRCEXTINSELR); + etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i)); + etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR); + etm4x_relaxed_write32(csa, config->seq_state, TRCSEQSTR); + etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR); for (i = 0; i < drvdata->nr_cntr; i++) { - writel_relaxed(config->cntrldvr[i], - drvdata->base + TRCCNTRLDVRn(i)); - writel_relaxed(config->cntr_ctrl[i], - drvdata->base + TRCCNTCTLRn(i)); - writel_relaxed(config->cntr_val[i], - drvdata->base + TRCCNTVRn(i)); + etm4x_relaxed_write32(csa, config->cntrldvr[i], TRCCNTRLDVRn(i)); + etm4x_relaxed_write32(csa, config->cntr_ctrl[i], TRCCNTCTLRn(i)); + etm4x_relaxed_write32(csa, config->cntr_val[i], TRCCNTVRn(i)); } /* @@ -259,54 +334,52 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) * such start at 2. */ for (i = 2; i < drvdata->nr_resource * 2; i++) - writel_relaxed(config->res_ctrl[i], - drvdata->base + TRCRSCTLRn(i)); + etm4x_relaxed_write32(csa, config->res_ctrl[i], TRCRSCTLRn(i)); for (i = 0; i < drvdata->nr_ss_cmp; i++) { /* always clear status bit on restart if using single-shot */ if (config->ss_ctrl[i] || config->ss_pe_cmp[i]) config->ss_status[i] &= ~BIT(31); - writel_relaxed(config->ss_ctrl[i], - drvdata->base + TRCSSCCRn(i)); - writel_relaxed(config->ss_status[i], - drvdata->base + TRCSSCSRn(i)); - writel_relaxed(config->ss_pe_cmp[i], - drvdata->base + TRCSSPCICRn(i)); + etm4x_relaxed_write32(csa, config->ss_ctrl[i], TRCSSCCRn(i)); + etm4x_relaxed_write32(csa, config->ss_status[i], TRCSSCSRn(i)); + if (etm4x_sspcicrn_present(drvdata, i)) + etm4x_relaxed_write32(csa, config->ss_pe_cmp[i], TRCSSPCICRn(i)); } for (i = 0; i < drvdata->nr_addr_cmp; i++) { - writeq_relaxed(config->addr_val[i], - drvdata->base + TRCACVRn(i)); - writeq_relaxed(config->addr_acc[i], - drvdata->base + TRCACATRn(i)); + etm4x_relaxed_write64(csa, config->addr_val[i], TRCACVRn(i)); + etm4x_relaxed_write64(csa, config->addr_acc[i], TRCACATRn(i)); } for (i = 0; i < drvdata->numcidc; i++) - writeq_relaxed(config->ctxid_pid[i], - drvdata->base + TRCCIDCVRn(i)); - writel_relaxed(config->ctxid_mask0, drvdata->base + TRCCIDCCTLR0); + etm4x_relaxed_write64(csa, config->ctxid_pid[i], TRCCIDCVRn(i)); + etm4x_relaxed_write32(csa, config->ctxid_mask0, TRCCIDCCTLR0); if (drvdata->numcidc > 4) - writel_relaxed(config->ctxid_mask1, drvdata->base + TRCCIDCCTLR1); + etm4x_relaxed_write32(csa, config->ctxid_mask1, TRCCIDCCTLR1); for (i = 0; i < drvdata->numvmidc; i++) - writeq_relaxed(config->vmid_val[i], - drvdata->base + TRCVMIDCVRn(i)); - writel_relaxed(config->vmid_mask0, drvdata->base + TRCVMIDCCTLR0); + etm4x_relaxed_write64(csa, config->vmid_val[i], TRCVMIDCVRn(i)); + etm4x_relaxed_write32(csa, config->vmid_mask0, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - writel_relaxed(config->vmid_mask1, drvdata->base + TRCVMIDCCTLR1); + etm4x_relaxed_write32(csa, config->vmid_mask1, TRCVMIDCCTLR1); if (!drvdata->skip_power_up) { + u32 trcpdcr = etm4x_relaxed_read32(csa, TRCPDCR); + /* * Request to keep the trace unit powered and also * emulation of powerdown */ - writel_relaxed(readl_relaxed(drvdata->base + TRCPDCR) | - TRCPDCR_PU, drvdata->base + TRCPDCR); + etm4x_relaxed_write32(csa, trcpdcr | TRCPDCR_PU, TRCPDCR); } /* Enable the trace unit */ - writel_relaxed(1, drvdata->base + TRCPRGCTLR); + etm4x_relaxed_write32(csa, 1, TRCPRGCTLR); + + /* Synchronize the register updates for sysreg access */ + if (!csa->io_mem) + isb(); /* wait for TRCSTATR.IDLE to go back down to '0' */ - if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 0)) + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 0)) dev_err(etm_dev, "timeout while waiting for Idle Trace Status\n"); @@ -318,7 +391,7 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) isb(); done: - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n", drvdata->cpu, rc); @@ -477,6 +550,19 @@ static int etm4_parse_event_config(struct etmv4_drvdata *drvdata, /* bit[6], Context ID tracing bit */ config->cfg |= BIT(ETM4_CFG_BIT_CTXTID); + /* + * If set bit ETM_OPT_CTXTID2 in perf config, this asks to trace VMID + * for recording CONTEXTIDR_EL2. Do not enable VMID tracing if the + * kernel is not running in EL2. + */ + if (attr->config & BIT(ETM_OPT_CTXTID2)) { + if (!is_kernel_in_hyp_mode()) { + ret = -EINVAL; + goto out; + } + config->cfg |= BIT(ETM4_CFG_BIT_VMID) | BIT(ETM4_CFG_BIT_VMID_OPT); + } + /* return stack - enable if selected and supported */ if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack) /* bit[12], Return stack enable bit */ @@ -570,20 +656,22 @@ static void etm4_disable_hw(void *info) u32 control; struct etmv4_drvdata *drvdata = info; struct etmv4_config *config = &drvdata->config; - struct device *etm_dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; + struct device *etm_dev = &csdev->dev; + struct csdev_access *csa = &csdev->access; int i; - CS_UNLOCK(drvdata->base); + etm4_cs_unlock(drvdata, csa); etm4_disable_arch_specific(drvdata); if (!drvdata->skip_power_up) { /* power can be removed from the trace unit now */ - control = readl_relaxed(drvdata->base + TRCPDCR); + control = etm4x_relaxed_read32(csa, TRCPDCR); control &= ~TRCPDCR_PU; - writel_relaxed(control, drvdata->base + TRCPDCR); + etm4x_relaxed_write32(csa, control, TRCPDCR); } - control = readl_relaxed(drvdata->base + TRCPRGCTLR); + control = etm4x_relaxed_read32(csa, TRCPRGCTLR); /* EN, bit[0] Trace unit enable bit */ control &= ~0x1; @@ -595,29 +683,27 @@ static void etm4_disable_hw(void *info) */ dsb(sy); isb(); - writel_relaxed(control, drvdata->base + TRCPRGCTLR); + etm4x_relaxed_write32(csa, control, TRCPRGCTLR); /* wait for TRCSTATR.PMSTABLE to go to '1' */ - if (coresight_timeout(drvdata->base, TRCSTATR, - TRCSTATR_PMSTABLE_BIT, 1)) + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) dev_err(etm_dev, "timeout while waiting for PM stable Trace Status\n"); /* read the status of the single shot comparators */ for (i = 0; i < drvdata->nr_ss_cmp; i++) { config->ss_status[i] = - readl_relaxed(drvdata->base + TRCSSCSRn(i)); + etm4x_relaxed_read32(csa, TRCSSCSRn(i)); } /* read back the current counter values */ for (i = 0; i < drvdata->nr_cntr; i++) { config->cntr_val[i] = - readl_relaxed(drvdata->base + TRCCNTVRn(i)); + etm4x_relaxed_read32(csa, TRCCNTVRn(i)); } - coresight_disclaim_device_unlocked(drvdata->base); - - CS_LOCK(drvdata->base); + coresight_disclaim_device_unlocked(csdev); + etm4_cs_lock(drvdata, csa); dev_dbg(&drvdata->csdev->dev, "cpu: %d disable smp call done\n", drvdata->cpu); @@ -641,7 +727,7 @@ static int etm4_disable_perf(struct coresight_device *csdev, * scheduled again. Configuration of the start/stop logic happens in * function etm4_set_event_filters(). */ - control = readl_relaxed(drvdata->base + TRCVICTLR); + control = etm4x_relaxed_read32(&csdev->access, TRCVICTLR); /* TRCVICTLR::SSSTATUS, bit[9] */ filters->ssstatus = (control & BIT(9)); @@ -712,24 +798,136 @@ static const struct coresight_ops etm4_cs_ops = { .source_ops = &etm4_source_ops, }; +static inline bool cpu_supports_sysreg_trace(void) +{ + u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); + + return ((dfr0 >> ID_AA64DFR0_TRACEVER_SHIFT) & 0xfUL) > 0; +} + +static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + u32 devarch; + + if (!cpu_supports_sysreg_trace()) + return false; + + /* + * ETMs implementing sysreg access must implement TRCDEVARCH. + */ + devarch = read_etm4x_sysreg_const_offset(TRCDEVARCH); + if ((devarch & ETM_DEVARCH_ID_MASK) != ETM_DEVARCH_ETMv4x_ARCH) + return false; + *csa = (struct csdev_access) { + .io_mem = false, + .read = etm4x_sysreg_read, + .write = etm4x_sysreg_write, + }; + + drvdata->arch = etm_devarch_to_arch(devarch); + return true; +} + +static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + u32 devarch = readl_relaxed(drvdata->base + TRCDEVARCH); + u32 idr1 = readl_relaxed(drvdata->base + TRCIDR1); + + /* + * All ETMs must implement TRCDEVARCH to indicate that + * the component is an ETMv4. To support any broken + * implementations we fall back to TRCIDR1 check, which + * is not really reliable. + */ + if ((devarch & ETM_DEVARCH_ID_MASK) == ETM_DEVARCH_ETMv4x_ARCH) { + drvdata->arch = etm_devarch_to_arch(devarch); + } else { + pr_warn("CPU%d: ETM4x incompatible TRCDEVARCH: %x, falling back to TRCIDR1\n", + smp_processor_id(), devarch); + + if (ETM_TRCIDR1_ARCH_MAJOR(idr1) != ETM_TRCIDR1_ARCH_ETMv4) + return false; + drvdata->arch = etm_trcidr_to_arch(idr1); + } + + *csa = CSDEV_ACCESS_IOMEM(drvdata->base); + return true; +} + +static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + /* + * Always choose the memory mapped io, if there is + * a memory map to prevent sysreg access on broken + * systems. + */ + if (drvdata->base) + return etm4_init_iomem_access(drvdata, csa); + + if (etm4_init_sysreg_access(drvdata, csa)) + return true; + + return false; +} + +static void cpu_enable_tracing(void) +{ + u64 dfr0 = read_sysreg(id_aa64dfr0_el1); + u64 trfcr; + + if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_TRACE_FILT_SHIFT)) + return; + + /* + * If the CPU supports v8.4 SelfHosted Tracing, enable + * tracing at the kernel EL and EL0, forcing to use the + * virtual time as the timestamp. + */ + trfcr = (TRFCR_ELx_TS_VIRTUAL | + TRFCR_ELx_ExTRE | + TRFCR_ELx_E0TRE); + + /* If we are running at EL2, allow tracing the CONTEXTIDR_EL2. */ + if (is_kernel_in_hyp_mode()) + trfcr |= TRFCR_EL2_CX; + + write_sysreg_s(trfcr, SYS_TRFCR_EL1); +} + static void etm4_init_arch_data(void *info) { u32 etmidr0; - u32 etmidr1; u32 etmidr2; u32 etmidr3; u32 etmidr4; u32 etmidr5; - struct etmv4_drvdata *drvdata = info; + struct etm4_init_arg *init_arg = info; + struct etmv4_drvdata *drvdata; + struct csdev_access *csa; int i; + drvdata = init_arg->drvdata; + csa = init_arg->csa; + + /* + * If we are unable to detect the access mechanism, + * or unable to detect the trace unit type, fail + * early. + */ + if (!etm4_init_csdev_access(drvdata, csa)) + return; + /* Make sure all registers are accessible */ - etm4_os_unlock(drvdata); + etm4_os_unlock_csa(drvdata, csa); + etm4_cs_unlock(drvdata, csa); - CS_UNLOCK(drvdata->base); + etm4_check_arch_features(drvdata, init_arg->pid); /* find all capabilities of the tracing unit */ - etmidr0 = readl_relaxed(drvdata->base + TRCIDR0); + etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0); /* INSTP0, bits[2:1] P0 tracing support field */ if (BMVAL(etmidr0, 1, 1) && BMVAL(etmidr0, 2, 2)) @@ -768,17 +966,8 @@ static void etm4_init_arch_data(void *info) /* TSSIZE, bits[28:24] Global timestamp size field */ drvdata->ts_size = BMVAL(etmidr0, 24, 28); - /* base architecture of trace unit */ - etmidr1 = readl_relaxed(drvdata->base + TRCIDR1); - /* - * TRCARCHMIN, bits[7:4] architecture the minor version number - * TRCARCHMAJ, bits[11:8] architecture major versin number - */ - drvdata->arch = BMVAL(etmidr1, 4, 11); - drvdata->config.arch = drvdata->arch; - /* maximum size of resources */ - etmidr2 = readl_relaxed(drvdata->base + TRCIDR2); + etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2); /* CIDSIZE, bits[9:5] Indicates the Context ID size */ drvdata->ctxid_size = BMVAL(etmidr2, 5, 9); /* VMIDSIZE, bits[14:10] Indicates the VMID size */ @@ -786,11 +975,12 @@ static void etm4_init_arch_data(void *info) /* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */ drvdata->ccsize = BMVAL(etmidr2, 25, 28); - etmidr3 = readl_relaxed(drvdata->base + TRCIDR3); + etmidr3 = etm4x_relaxed_read32(csa, TRCIDR3); /* CCITMIN, bits[11:0] minimum threshold value that can be programmed */ drvdata->ccitmin = BMVAL(etmidr3, 0, 11); /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */ drvdata->s_ex_level = BMVAL(etmidr3, 16, 19); + drvdata->config.s_ex_level = drvdata->s_ex_level; /* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */ drvdata->ns_ex_level = BMVAL(etmidr3, 20, 23); @@ -836,7 +1026,7 @@ static void etm4_init_arch_data(void *info) drvdata->nooverflow = false; /* number of resources trace unit supports */ - etmidr4 = readl_relaxed(drvdata->base + TRCIDR4); + etmidr4 = etm4x_relaxed_read32(csa, TRCIDR4); /* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */ drvdata->nr_addr_cmp = BMVAL(etmidr4, 0, 3); /* NUMPC, bits[15:12] number of PE comparator inputs for tracing */ @@ -852,7 +1042,7 @@ static void etm4_init_arch_data(void *info) * Otherwise for values 0x1 and above the number is N + 1 as per v4.2. */ drvdata->nr_resource = BMVAL(etmidr4, 16, 19); - if ((drvdata->arch < ETM4X_ARCH_4V3) || (drvdata->nr_resource > 0)) + if ((drvdata->arch < ETM_ARCH_V4_3) || (drvdata->nr_resource > 0)) drvdata->nr_resource += 1; /* * NUMSSCC, bits[23:20] the number of single-shot @@ -862,14 +1052,14 @@ static void etm4_init_arch_data(void *info) drvdata->nr_ss_cmp = BMVAL(etmidr4, 20, 23); for (i = 0; i < drvdata->nr_ss_cmp; i++) { drvdata->config.ss_status[i] = - readl_relaxed(drvdata->base + TRCSSCSRn(i)); + etm4x_relaxed_read32(csa, TRCSSCSRn(i)); } /* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */ drvdata->numcidc = BMVAL(etmidr4, 24, 27); /* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */ drvdata->numvmidc = BMVAL(etmidr4, 28, 31); - etmidr5 = readl_relaxed(drvdata->base + TRCIDR5); + etmidr5 = etm4x_relaxed_read32(csa, TRCIDR5); /* NUMEXTIN, bits[8:0] number of external inputs implemented */ drvdata->nr_ext_inp = BMVAL(etmidr5, 0, 8); /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */ @@ -891,23 +1081,20 @@ static void etm4_init_arch_data(void *info) drvdata->nrseqstate = BMVAL(etmidr5, 25, 27); /* NUMCNTR, bits[30:28] number of counters available for tracing */ drvdata->nr_cntr = BMVAL(etmidr5, 28, 30); - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); + cpu_enable_tracing(); +} + +static inline u32 etm4_get_victlr_access_type(struct etmv4_config *config) +{ + return etm4_get_access_type(config) << TRCVICTLR_EXLEVEL_SHIFT; } /* Set ELx trace filter access in the TRCVICTLR register */ static void etm4_set_victlr_access(struct etmv4_config *config) { - u64 access_type; - - config->vinst_ctrl &= ~(ETM_EXLEVEL_S_VICTLR_MASK | ETM_EXLEVEL_NS_VICTLR_MASK); - - /* - * TRCVICTLR::EXLEVEL_NS:EXLEVELS: Set kernel / user filtering - * bits in vinst_ctrl, same bit pattern as TRCACATRn values returned by - * etm4_get_access_type() but with a relative shift in this register. - */ - access_type = etm4_get_access_type(config) << ETM_EXLEVEL_LSHIFT_TRCVICTLR; - config->vinst_ctrl |= (u32)access_type; + config->vinst_ctrl &= ~TRCVICTLR_EXLEVEL_MASK; + config->vinst_ctrl |= etm4_get_victlr_access_type(config); } static void etm4_set_default_config(struct etmv4_config *config) @@ -937,12 +1124,9 @@ static u64 etm4_get_ns_access_type(struct etmv4_config *config) u64 access_type = 0; /* - * EXLEVEL_NS, bits[15:12] - * The Exception levels are: - * Bit[12] Exception level 0 - Application - * Bit[13] Exception level 1 - OS - * Bit[14] Exception level 2 - Hypervisor - * Bit[15] Never implemented + * EXLEVEL_NS, for NonSecure Exception levels. + * The mask here is a generic value and must be + * shifted to the corresponding field for the registers */ if (!is_kernel_in_hyp_mode()) { /* Stay away from hypervisor mode for non-VHE */ @@ -959,27 +1143,26 @@ static u64 etm4_get_ns_access_type(struct etmv4_config *config) return access_type; } +/* + * Construct the exception level masks for a given config. + * This must be shifted to the corresponding register field + * for usage. + */ static u64 etm4_get_access_type(struct etmv4_config *config) { - u64 access_type = etm4_get_ns_access_type(config); - u64 s_hyp = (config->arch & 0x0f) >= 0x4 ? ETM_EXLEVEL_S_HYP : 0; - - /* - * EXLEVEL_S, bits[11:8], don't trace anything happening - * in secure state. - */ - access_type |= (ETM_EXLEVEL_S_APP | - ETM_EXLEVEL_S_OS | - s_hyp | - ETM_EXLEVEL_S_MON); + /* All Secure exception levels are excluded from the trace */ + return etm4_get_ns_access_type(config) | (u64)config->s_ex_level; +} - return access_type; +static u64 etm4_get_comparator_access_type(struct etmv4_config *config) +{ + return etm4_get_access_type(config) << TRCACATR_EXLEVEL_SHIFT; } static void etm4_set_comparator_filter(struct etmv4_config *config, u64 start, u64 stop, int comparator) { - u64 access_type = etm4_get_access_type(config); + u64 access_type = etm4_get_comparator_access_type(config); /* First half of default address comparator */ config->addr_val[comparator] = start; @@ -1014,7 +1197,7 @@ static void etm4_set_start_stop_filter(struct etmv4_config *config, enum etm_addr_type type) { int shift; - u64 access_type = etm4_get_access_type(config); + u64 access_type = etm4_get_comparator_access_type(config); /* Configure the comparator */ config->addr_val[comparator] = address; @@ -1255,7 +1438,15 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) { int i, ret = 0; struct etmv4_save_state *state; - struct device *etm_dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; + struct csdev_access *csa; + struct device *etm_dev; + + if (WARN_ON(!csdev)) + return -ENODEV; + + etm_dev = &csdev->dev; + csa = &csdev->access; /* * As recommended by 3.4.1 ("The procedure when powering down the PE") @@ -1264,14 +1455,12 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) dsb(sy); isb(); - CS_UNLOCK(drvdata->base); - + etm4_cs_unlock(drvdata, csa); /* Lock the OS lock to disable trace and external debugger access */ etm4_os_lock(drvdata); /* wait for TRCSTATR.PMSTABLE to go up */ - if (coresight_timeout(drvdata->base, TRCSTATR, - TRCSTATR_PMSTABLE_BIT, 1)) { + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) { dev_err(etm_dev, "timeout while waiting for PM Stable Status\n"); etm4_os_unlock(drvdata); @@ -1281,55 +1470,57 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) state = drvdata->save_state; - state->trcprgctlr = readl(drvdata->base + TRCPRGCTLR); + state->trcprgctlr = etm4x_read32(csa, TRCPRGCTLR); if (drvdata->nr_pe) - state->trcprocselr = readl(drvdata->base + TRCPROCSELR); - state->trcconfigr = readl(drvdata->base + TRCCONFIGR); - state->trcauxctlr = readl(drvdata->base + TRCAUXCTLR); - state->trceventctl0r = readl(drvdata->base + TRCEVENTCTL0R); - state->trceventctl1r = readl(drvdata->base + TRCEVENTCTL1R); - state->trcstallctlr = readl(drvdata->base + TRCSTALLCTLR); - state->trctsctlr = readl(drvdata->base + TRCTSCTLR); - state->trcsyncpr = readl(drvdata->base + TRCSYNCPR); - state->trcccctlr = readl(drvdata->base + TRCCCCTLR); - state->trcbbctlr = readl(drvdata->base + TRCBBCTLR); - state->trctraceidr = readl(drvdata->base + TRCTRACEIDR); - state->trcqctlr = readl(drvdata->base + TRCQCTLR); - - state->trcvictlr = readl(drvdata->base + TRCVICTLR); - state->trcviiectlr = readl(drvdata->base + TRCVIIECTLR); - state->trcvissctlr = readl(drvdata->base + TRCVISSCTLR); + state->trcprocselr = etm4x_read32(csa, TRCPROCSELR); + state->trcconfigr = etm4x_read32(csa, TRCCONFIGR); + state->trcauxctlr = etm4x_read32(csa, TRCAUXCTLR); + state->trceventctl0r = etm4x_read32(csa, TRCEVENTCTL0R); + state->trceventctl1r = etm4x_read32(csa, TRCEVENTCTL1R); + if (drvdata->stallctl) + state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR); + state->trctsctlr = etm4x_read32(csa, TRCTSCTLR); + state->trcsyncpr = etm4x_read32(csa, TRCSYNCPR); + state->trcccctlr = etm4x_read32(csa, TRCCCCTLR); + state->trcbbctlr = etm4x_read32(csa, TRCBBCTLR); + state->trctraceidr = etm4x_read32(csa, TRCTRACEIDR); + state->trcqctlr = etm4x_read32(csa, TRCQCTLR); + + state->trcvictlr = etm4x_read32(csa, TRCVICTLR); + state->trcviiectlr = etm4x_read32(csa, TRCVIIECTLR); + state->trcvissctlr = etm4x_read32(csa, TRCVISSCTLR); if (drvdata->nr_pe_cmp) - state->trcvipcssctlr = readl(drvdata->base + TRCVIPCSSCTLR); - state->trcvdctlr = readl(drvdata->base + TRCVDCTLR); - state->trcvdsacctlr = readl(drvdata->base + TRCVDSACCTLR); - state->trcvdarcctlr = readl(drvdata->base + TRCVDARCCTLR); + state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR); + state->trcvdctlr = etm4x_read32(csa, TRCVDCTLR); + state->trcvdsacctlr = etm4x_read32(csa, TRCVDSACCTLR); + state->trcvdarcctlr = etm4x_read32(csa, TRCVDARCCTLR); for (i = 0; i < drvdata->nrseqstate - 1; i++) - state->trcseqevr[i] = readl(drvdata->base + TRCSEQEVRn(i)); + state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i)); - state->trcseqrstevr = readl(drvdata->base + TRCSEQRSTEVR); - state->trcseqstr = readl(drvdata->base + TRCSEQSTR); - state->trcextinselr = readl(drvdata->base + TRCEXTINSELR); + state->trcseqrstevr = etm4x_read32(csa, TRCSEQRSTEVR); + state->trcseqstr = etm4x_read32(csa, TRCSEQSTR); + state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR); for (i = 0; i < drvdata->nr_cntr; i++) { - state->trccntrldvr[i] = readl(drvdata->base + TRCCNTRLDVRn(i)); - state->trccntctlr[i] = readl(drvdata->base + TRCCNTCTLRn(i)); - state->trccntvr[i] = readl(drvdata->base + TRCCNTVRn(i)); + state->trccntrldvr[i] = etm4x_read32(csa, TRCCNTRLDVRn(i)); + state->trccntctlr[i] = etm4x_read32(csa, TRCCNTCTLRn(i)); + state->trccntvr[i] = etm4x_read32(csa, TRCCNTVRn(i)); } for (i = 0; i < drvdata->nr_resource * 2; i++) - state->trcrsctlr[i] = readl(drvdata->base + TRCRSCTLRn(i)); + state->trcrsctlr[i] = etm4x_read32(csa, TRCRSCTLRn(i)); for (i = 0; i < drvdata->nr_ss_cmp; i++) { - state->trcssccr[i] = readl(drvdata->base + TRCSSCCRn(i)); - state->trcsscsr[i] = readl(drvdata->base + TRCSSCSRn(i)); - state->trcsspcicr[i] = readl(drvdata->base + TRCSSPCICRn(i)); + state->trcssccr[i] = etm4x_read32(csa, TRCSSCCRn(i)); + state->trcsscsr[i] = etm4x_read32(csa, TRCSSCSRn(i)); + if (etm4x_sspcicrn_present(drvdata, i)) + state->trcsspcicr[i] = etm4x_read32(csa, TRCSSPCICRn(i)); } for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { - state->trcacvr[i] = readq(drvdata->base + TRCACVRn(i)); - state->trcacatr[i] = readq(drvdata->base + TRCACATRn(i)); + state->trcacvr[i] = etm4x_read64(csa, TRCACVRn(i)); + state->trcacatr[i] = etm4x_read64(csa, TRCACATRn(i)); } /* @@ -1340,25 +1531,26 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) */ for (i = 0; i < drvdata->numcidc; i++) - state->trccidcvr[i] = readq(drvdata->base + TRCCIDCVRn(i)); + state->trccidcvr[i] = etm4x_read64(csa, TRCCIDCVRn(i)); for (i = 0; i < drvdata->numvmidc; i++) - state->trcvmidcvr[i] = readq(drvdata->base + TRCVMIDCVRn(i)); + state->trcvmidcvr[i] = etm4x_read64(csa, TRCVMIDCVRn(i)); - state->trccidcctlr0 = readl(drvdata->base + TRCCIDCCTLR0); + state->trccidcctlr0 = etm4x_read32(csa, TRCCIDCCTLR0); if (drvdata->numcidc > 4) - state->trccidcctlr1 = readl(drvdata->base + TRCCIDCCTLR1); + state->trccidcctlr1 = etm4x_read32(csa, TRCCIDCCTLR1); - state->trcvmidcctlr0 = readl(drvdata->base + TRCVMIDCCTLR0); + state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - state->trcvmidcctlr1 = readl(drvdata->base + TRCVMIDCCTLR1); + state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1); - state->trcclaimset = readl(drvdata->base + TRCCLAIMCLR); + state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR); - state->trcpdcr = readl(drvdata->base + TRCPDCR); + if (!drvdata->skip_power_up) + state->trcpdcr = etm4x_read32(csa, TRCPDCR); /* wait for TRCSTATR.IDLE to go up */ - if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) { + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) { dev_err(etm_dev, "timeout while waiting for Idle Trace Status\n"); etm4_os_unlock(drvdata); @@ -1373,11 +1565,11 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) * potentially save power on systems that respect the TRCPDCR_PU * despite requesting software to save/restore state. */ - writel_relaxed((state->trcpdcr & ~TRCPDCR_PU), - drvdata->base + TRCPDCR); - + if (!drvdata->skip_power_up) + etm4x_relaxed_write32(csa, (state->trcpdcr & ~TRCPDCR_PU), + TRCPDCR); out: - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); return ret; } @@ -1385,91 +1577,83 @@ static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) { int i; struct etmv4_save_state *state = drvdata->save_state; + struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base); + struct csdev_access *csa = &tmp_csa; - CS_UNLOCK(drvdata->base); - - writel_relaxed(state->trcclaimset, drvdata->base + TRCCLAIMSET); + etm4_cs_unlock(drvdata, csa); + etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); - writel_relaxed(state->trcprgctlr, drvdata->base + TRCPRGCTLR); + etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR); if (drvdata->nr_pe) - writel_relaxed(state->trcprocselr, drvdata->base + TRCPROCSELR); - writel_relaxed(state->trcconfigr, drvdata->base + TRCCONFIGR); - writel_relaxed(state->trcauxctlr, drvdata->base + TRCAUXCTLR); - writel_relaxed(state->trceventctl0r, drvdata->base + TRCEVENTCTL0R); - writel_relaxed(state->trceventctl1r, drvdata->base + TRCEVENTCTL1R); - writel_relaxed(state->trcstallctlr, drvdata->base + TRCSTALLCTLR); - writel_relaxed(state->trctsctlr, drvdata->base + TRCTSCTLR); - writel_relaxed(state->trcsyncpr, drvdata->base + TRCSYNCPR); - writel_relaxed(state->trcccctlr, drvdata->base + TRCCCCTLR); - writel_relaxed(state->trcbbctlr, drvdata->base + TRCBBCTLR); - writel_relaxed(state->trctraceidr, drvdata->base + TRCTRACEIDR); - writel_relaxed(state->trcqctlr, drvdata->base + TRCQCTLR); - - writel_relaxed(state->trcvictlr, drvdata->base + TRCVICTLR); - writel_relaxed(state->trcviiectlr, drvdata->base + TRCVIIECTLR); - writel_relaxed(state->trcvissctlr, drvdata->base + TRCVISSCTLR); + etm4x_relaxed_write32(csa, state->trcprocselr, TRCPROCSELR); + etm4x_relaxed_write32(csa, state->trcconfigr, TRCCONFIGR); + etm4x_relaxed_write32(csa, state->trcauxctlr, TRCAUXCTLR); + etm4x_relaxed_write32(csa, state->trceventctl0r, TRCEVENTCTL0R); + etm4x_relaxed_write32(csa, state->trceventctl1r, TRCEVENTCTL1R); + if (drvdata->stallctl) + etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR); + etm4x_relaxed_write32(csa, state->trctsctlr, TRCTSCTLR); + etm4x_relaxed_write32(csa, state->trcsyncpr, TRCSYNCPR); + etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR); + etm4x_relaxed_write32(csa, state->trcbbctlr, TRCBBCTLR); + etm4x_relaxed_write32(csa, state->trctraceidr, TRCTRACEIDR); + etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR); + + etm4x_relaxed_write32(csa, state->trcvictlr, TRCVICTLR); + etm4x_relaxed_write32(csa, state->trcviiectlr, TRCVIIECTLR); + etm4x_relaxed_write32(csa, state->trcvissctlr, TRCVISSCTLR); if (drvdata->nr_pe_cmp) - writel_relaxed(state->trcvipcssctlr, drvdata->base + TRCVIPCSSCTLR); - writel_relaxed(state->trcvdctlr, drvdata->base + TRCVDCTLR); - writel_relaxed(state->trcvdsacctlr, drvdata->base + TRCVDSACCTLR); - writel_relaxed(state->trcvdarcctlr, drvdata->base + TRCVDARCCTLR); + etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR); + etm4x_relaxed_write32(csa, state->trcvdctlr, TRCVDCTLR); + etm4x_relaxed_write32(csa, state->trcvdsacctlr, TRCVDSACCTLR); + etm4x_relaxed_write32(csa, state->trcvdarcctlr, TRCVDARCCTLR); for (i = 0; i < drvdata->nrseqstate - 1; i++) - writel_relaxed(state->trcseqevr[i], - drvdata->base + TRCSEQEVRn(i)); + etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i)); - writel_relaxed(state->trcseqrstevr, drvdata->base + TRCSEQRSTEVR); - writel_relaxed(state->trcseqstr, drvdata->base + TRCSEQSTR); - writel_relaxed(state->trcextinselr, drvdata->base + TRCEXTINSELR); + etm4x_relaxed_write32(csa, state->trcseqrstevr, TRCSEQRSTEVR); + etm4x_relaxed_write32(csa, state->trcseqstr, TRCSEQSTR); + etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR); for (i = 0; i < drvdata->nr_cntr; i++) { - writel_relaxed(state->trccntrldvr[i], - drvdata->base + TRCCNTRLDVRn(i)); - writel_relaxed(state->trccntctlr[i], - drvdata->base + TRCCNTCTLRn(i)); - writel_relaxed(state->trccntvr[i], - drvdata->base + TRCCNTVRn(i)); + etm4x_relaxed_write32(csa, state->trccntrldvr[i], TRCCNTRLDVRn(i)); + etm4x_relaxed_write32(csa, state->trccntctlr[i], TRCCNTCTLRn(i)); + etm4x_relaxed_write32(csa, state->trccntvr[i], TRCCNTVRn(i)); } for (i = 0; i < drvdata->nr_resource * 2; i++) - writel_relaxed(state->trcrsctlr[i], - drvdata->base + TRCRSCTLRn(i)); + etm4x_relaxed_write32(csa, state->trcrsctlr[i], TRCRSCTLRn(i)); for (i = 0; i < drvdata->nr_ss_cmp; i++) { - writel_relaxed(state->trcssccr[i], - drvdata->base + TRCSSCCRn(i)); - writel_relaxed(state->trcsscsr[i], - drvdata->base + TRCSSCSRn(i)); - writel_relaxed(state->trcsspcicr[i], - drvdata->base + TRCSSPCICRn(i)); + etm4x_relaxed_write32(csa, state->trcssccr[i], TRCSSCCRn(i)); + etm4x_relaxed_write32(csa, state->trcsscsr[i], TRCSSCSRn(i)); + if (etm4x_sspcicrn_present(drvdata, i)) + etm4x_relaxed_write32(csa, state->trcsspcicr[i], TRCSSPCICRn(i)); } for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { - writeq_relaxed(state->trcacvr[i], - drvdata->base + TRCACVRn(i)); - writeq_relaxed(state->trcacatr[i], - drvdata->base + TRCACATRn(i)); + etm4x_relaxed_write64(csa, state->trcacvr[i], TRCACVRn(i)); + etm4x_relaxed_write64(csa, state->trcacatr[i], TRCACATRn(i)); } for (i = 0; i < drvdata->numcidc; i++) - writeq_relaxed(state->trccidcvr[i], - drvdata->base + TRCCIDCVRn(i)); + etm4x_relaxed_write64(csa, state->trccidcvr[i], TRCCIDCVRn(i)); for (i = 0; i < drvdata->numvmidc; i++) - writeq_relaxed(state->trcvmidcvr[i], - drvdata->base + TRCVMIDCVRn(i)); + etm4x_relaxed_write64(csa, state->trcvmidcvr[i], TRCVMIDCVRn(i)); - writel_relaxed(state->trccidcctlr0, drvdata->base + TRCCIDCCTLR0); + etm4x_relaxed_write32(csa, state->trccidcctlr0, TRCCIDCCTLR0); if (drvdata->numcidc > 4) - writel_relaxed(state->trccidcctlr1, drvdata->base + TRCCIDCCTLR1); + etm4x_relaxed_write32(csa, state->trccidcctlr1, TRCCIDCCTLR1); - writel_relaxed(state->trcvmidcctlr0, drvdata->base + TRCVMIDCCTLR0); + etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - writel_relaxed(state->trcvmidcctlr1, drvdata->base + TRCVMIDCCTLR1); + etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1); - writel_relaxed(state->trcclaimset, drvdata->base + TRCCLAIMSET); + etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); - writel_relaxed(state->trcpdcr, drvdata->base + TRCPDCR); + if (!drvdata->skip_power_up) + etm4x_relaxed_write32(csa, state->trcpdcr, TRCPDCR); drvdata->state_needs_restore = false; @@ -1482,7 +1666,7 @@ static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) /* Unlock the OS lock to re-enable trace and external debug access */ etm4_os_unlock(drvdata); - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); } static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, @@ -1569,15 +1753,13 @@ static void etm4_pm_clear(void) } } -static int etm4_probe(struct amba_device *adev, const struct amba_id *id) +static int etm4_probe(struct device *dev, void __iomem *base, u32 etm_pid) { int ret; - void __iomem *base; - struct device *dev = &adev->dev; struct coresight_platform_data *pdata = NULL; struct etmv4_drvdata *drvdata; - struct resource *res = &adev->res; struct coresight_desc desc = { 0 }; + struct etm4_init_arg init_arg = { 0 }; drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) @@ -1596,14 +1778,6 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) return -ENOMEM; } - if (fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up")) - drvdata->skip_power_up = true; - - /* Validity for the resource is already checked by the AMBA core */ - base = devm_ioremap_resource(dev, res); - if (IS_ERR(base)) - return PTR_ERR(base); - drvdata->base = base; spin_lock_init(&drvdata->spinlock); @@ -1616,13 +1790,22 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) if (!desc.name) return -ENOMEM; + init_arg.drvdata = drvdata; + init_arg.csa = &desc.access; + init_arg.pid = etm_pid; + if (smp_call_function_single(drvdata->cpu, - etm4_init_arch_data, drvdata, 1)) + etm4_init_arch_data, &init_arg, 1)) dev_err(dev, "ETM arch init failed\n"); - if (etm4_arch_supported(drvdata->arch) == false) + if (!drvdata->arch) return -EINVAL; + /* TRCPDCR is not accessible with system instructions. */ + if (!desc.access.io_mem || + fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up")) + drvdata->skip_power_up = true; + etm4_init_trace_id(drvdata); etm4_set_default(&drvdata->config); @@ -1630,7 +1813,7 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) if (IS_ERR(pdata)) return PTR_ERR(pdata); - adev->dev.platform_data = pdata; + dev->platform_data = pdata; desc.type = CORESIGHT_DEV_TYPE_SOURCE; desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC; @@ -1650,25 +1833,61 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) etmdrvdata[drvdata->cpu] = drvdata; - pm_runtime_put(&adev->dev); dev_info(&drvdata->csdev->dev, "CPU%d: ETM v%d.%d initialized\n", - drvdata->cpu, drvdata->arch >> 4, drvdata->arch & 0xf); + drvdata->cpu, ETM_ARCH_MAJOR_VERSION(drvdata->arch), + ETM_ARCH_MINOR_VERSION(drvdata->arch)); if (boot_enable) { coresight_enable(drvdata->csdev); drvdata->boot_enable = true; } - etm4_check_arch_features(drvdata, id->id); - return 0; } +static int etm4_probe_amba(struct amba_device *adev, const struct amba_id *id) +{ + void __iomem *base; + struct device *dev = &adev->dev; + struct resource *res = &adev->res; + int ret; + + /* Validity for the resource is already checked by the AMBA core */ + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + ret = etm4_probe(dev, base, id->id); + if (!ret) + pm_runtime_put(&adev->dev); + + return ret; +} + +static int etm4_probe_platform_dev(struct platform_device *pdev) +{ + int ret; + + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + + /* + * System register based devices could match the + * HW by reading appropriate registers on the HW + * and thus we could skip the PID. + */ + ret = etm4_probe(&pdev->dev, NULL, 0); + + pm_runtime_put(&pdev->dev); + return ret; +} + static struct amba_cs_uci_id uci_id_etm4[] = { { /* ETMv4 UCI data */ - .devarch = 0x47704a13, - .devarch_mask = 0xfff0ffff, + .devarch = ETM_DEVARCH_ETMv4x_ARCH, + .devarch_mask = ETM_DEVARCH_ID_MASK, .devtype = 0x00000013, } }; @@ -1680,15 +1899,12 @@ static void clear_etmdrvdata(void *info) etmdrvdata[cpu] = NULL; } -static void etm4_remove(struct amba_device *adev) +static int __exit etm4_remove_dev(struct etmv4_drvdata *drvdata) { - struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev); - etm_perf_symlink(drvdata->csdev, false); - /* - * Taking hotplug lock here to avoid racing between etm4_remove and - * CPU hotplug call backs. + * Taking hotplug lock here to avoid racing between etm4_remove_dev() + * and CPU hotplug call backs. */ cpus_read_lock(); /* @@ -1703,6 +1919,27 @@ static void etm4_remove(struct amba_device *adev) cpus_read_unlock(); coresight_unregister(drvdata->csdev); + + return 0; +} + +static void __exit etm4_remove_amba(struct amba_device *adev) +{ + struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev); + + if (drvdata) + etm4_remove_dev(drvdata); +} + +static int __exit etm4_remove_platform_dev(struct platform_device *pdev) +{ + int ret = 0; + struct etmv4_drvdata *drvdata = dev_get_drvdata(&pdev->dev); + + if (drvdata) + ret = etm4_remove_dev(drvdata); + pm_runtime_disable(&pdev->dev); + return ret; } static const struct amba_id etm4_ids[] = { @@ -1711,6 +1948,8 @@ static const struct amba_id etm4_ids[] = { CS_AMBA_ID(0x000bb95a), /* Cortex-A72 */ CS_AMBA_ID(0x000bb959), /* Cortex-A73 */ CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */ + CS_AMBA_UCI_ID(0x000bbd05, uci_id_etm4),/* Cortex-A55 */ + CS_AMBA_UCI_ID(0x000bbd0a, uci_id_etm4),/* Cortex-A75 */ CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),/* Neoverse N1 */ CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */ CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */ @@ -1726,17 +1965,32 @@ static const struct amba_id etm4_ids[] = { MODULE_DEVICE_TABLE(amba, etm4_ids); -static struct amba_driver etm4x_driver = { +static struct amba_driver etm4x_amba_driver = { .drv = { .name = "coresight-etm4x", .owner = THIS_MODULE, .suppress_bind_attrs = true, }, - .probe = etm4_probe, - .remove = etm4_remove, + .probe = etm4_probe_amba, + .remove = etm4_remove_amba, .id_table = etm4_ids, }; +static const struct of_device_id etm4_sysreg_match[] = { + { .compatible = "arm,coresight-etm4x-sysreg" }, + {} +}; + +static struct platform_driver etm4_platform_driver = { + .probe = etm4_probe_platform_dev, + .remove = etm4_remove_platform_dev, + .driver = { + .name = "coresight-etm4x", + .of_match_table = etm4_sysreg_match, + .suppress_bind_attrs = true, + }, +}; + static int __init etm4x_init(void) { int ret; @@ -1747,18 +2001,28 @@ static int __init etm4x_init(void) if (ret) return ret; - ret = amba_driver_register(&etm4x_driver); + ret = amba_driver_register(&etm4x_amba_driver); if (ret) { - pr_err("Error registering etm4x driver\n"); - etm4_pm_clear(); + pr_err("Error registering etm4x AMBA driver\n"); + goto clear_pm; } + ret = platform_driver_register(&etm4_platform_driver); + if (!ret) + return 0; + + pr_err("Error registering etm4x platform driver\n"); + amba_driver_unregister(&etm4x_amba_driver); + +clear_pm: + etm4_pm_clear(); return ret; } static void __exit etm4x_exit(void) { - amba_driver_unregister(&etm4x_driver); + amba_driver_unregister(&etm4x_amba_driver); + platform_driver_unregister(&etm4_platform_driver); etm4_pm_clear(); } diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index 989ce7b8ade7..0995a10790f4 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -389,7 +389,7 @@ static ssize_t mode_store(struct device *dev, config->eventctrl1 &= ~BIT(12); /* bit[8], Instruction stall bit */ - if (config->mode & ETM_MODE_ISTALL_EN) + if ((config->mode & ETM_MODE_ISTALL_EN) && (drvdata->stallctl == true)) config->stall_ctrl |= BIT(8); else config->stall_ctrl &= ~BIT(8); @@ -743,7 +743,7 @@ static ssize_t s_exlevel_vinst_show(struct device *dev, struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent); struct etmv4_config *config = &drvdata->config; - val = (config->vinst_ctrl & ETM_EXLEVEL_S_VICTLR_MASK) >> 16; + val = (config->vinst_ctrl & TRCVICTLR_EXLEVEL_S_MASK) >> TRCVICTLR_EXLEVEL_S_SHIFT; return scnprintf(buf, PAGE_SIZE, "%#lx\n", val); } @@ -760,10 +760,10 @@ static ssize_t s_exlevel_vinst_store(struct device *dev, spin_lock(&drvdata->spinlock); /* clear all EXLEVEL_S bits */ - config->vinst_ctrl &= ~(ETM_EXLEVEL_S_VICTLR_MASK); + config->vinst_ctrl &= ~(TRCVICTLR_EXLEVEL_S_MASK); /* enable instruction tracing for corresponding exception level */ val &= drvdata->s_ex_level; - config->vinst_ctrl |= (val << 16); + config->vinst_ctrl |= (val << TRCVICTLR_EXLEVEL_S_SHIFT); spin_unlock(&drvdata->spinlock); return size; } @@ -778,7 +778,7 @@ static ssize_t ns_exlevel_vinst_show(struct device *dev, struct etmv4_config *config = &drvdata->config; /* EXLEVEL_NS, bits[23:20] */ - val = (config->vinst_ctrl & ETM_EXLEVEL_NS_VICTLR_MASK) >> 20; + val = (config->vinst_ctrl & TRCVICTLR_EXLEVEL_NS_MASK) >> TRCVICTLR_EXLEVEL_NS_SHIFT; return scnprintf(buf, PAGE_SIZE, "%#lx\n", val); } @@ -795,10 +795,10 @@ static ssize_t ns_exlevel_vinst_store(struct device *dev, spin_lock(&drvdata->spinlock); /* clear EXLEVEL_NS bits */ - config->vinst_ctrl &= ~(ETM_EXLEVEL_NS_VICTLR_MASK); + config->vinst_ctrl &= ~(TRCVICTLR_EXLEVEL_NS_MASK); /* enable instruction tracing for corresponding exception level */ val &= drvdata->ns_ex_level; - config->vinst_ctrl |= (val << 20); + config->vinst_ctrl |= (val << TRCVICTLR_EXLEVEL_NS_SHIFT); spin_unlock(&drvdata->spinlock); return size; } @@ -2319,7 +2319,8 @@ static struct attribute *coresight_etmv4_attrs[] = { }; struct etmv4_reg { - void __iomem *addr; + struct coresight_device *csdev; + u32 offset; u32 data; }; @@ -2327,15 +2328,16 @@ static void do_smp_cross_read(void *data) { struct etmv4_reg *reg = data; - reg->data = readl_relaxed(reg->addr); + reg->data = etm4x_relaxed_read32(®->csdev->access, reg->offset); } -static u32 etmv4_cross_read(const struct device *dev, u32 offset) +static u32 etmv4_cross_read(const struct etmv4_drvdata *drvdata, u32 offset) { - struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); struct etmv4_reg reg; - reg.addr = drvdata->base + offset; + reg.offset = offset; + reg.csdev = drvdata->csdev; + /* * smp cross call ensures the CPU will be powered up before * accessing the ETMv4 trace core registers @@ -2344,72 +2346,120 @@ static u32 etmv4_cross_read(const struct device *dev, u32 offset) return reg.data; } -#define coresight_etm4x_reg(name, offset) \ - coresight_simple_reg32(struct etmv4_drvdata, name, offset) +static inline u32 coresight_etm4x_attr_to_offset(struct device_attribute *attr) +{ + struct dev_ext_attribute *eattr; + + eattr = container_of(attr, struct dev_ext_attribute, attr); + return (u32)(unsigned long)eattr->var; +} + +static ssize_t coresight_etm4x_reg_show(struct device *dev, + struct device_attribute *d_attr, + char *buf) +{ + u32 val, offset; + struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent); -#define coresight_etm4x_cross_read(name, offset) \ - coresight_simple_func(struct etmv4_drvdata, etmv4_cross_read, \ - name, offset) + offset = coresight_etm4x_attr_to_offset(d_attr); -coresight_etm4x_reg(trcpdcr, TRCPDCR); -coresight_etm4x_reg(trcpdsr, TRCPDSR); -coresight_etm4x_reg(trclsr, TRCLSR); -coresight_etm4x_reg(trcauthstatus, TRCAUTHSTATUS); -coresight_etm4x_reg(trcdevid, TRCDEVID); -coresight_etm4x_reg(trcdevtype, TRCDEVTYPE); -coresight_etm4x_reg(trcpidr0, TRCPIDR0); -coresight_etm4x_reg(trcpidr1, TRCPIDR1); -coresight_etm4x_reg(trcpidr2, TRCPIDR2); -coresight_etm4x_reg(trcpidr3, TRCPIDR3); -coresight_etm4x_cross_read(trcoslsr, TRCOSLSR); -coresight_etm4x_cross_read(trcconfig, TRCCONFIGR); -coresight_etm4x_cross_read(trctraceid, TRCTRACEIDR); + pm_runtime_get_sync(dev->parent); + val = etmv4_cross_read(drvdata, offset); + pm_runtime_put_sync(dev->parent); + + return scnprintf(buf, PAGE_SIZE, "0x%x\n", val); +} + +static inline bool +etm4x_register_implemented(struct etmv4_drvdata *drvdata, u32 offset) +{ + switch (offset) { + ETM4x_SYSREG_LIST_CASES + /* + * Registers accessible via system instructions are always + * implemented. + */ + return true; + ETM4x_MMAP_LIST_CASES + /* + * Registers accessible only via memory-mapped registers + * must not be accessed via system instructions. + * We cannot access the drvdata->csdev here, as this + * function is called during the device creation, via + * coresight_register() and the csdev is not initialized + * until that is done. So rely on the drvdata->base to + * detect if we have a memory mapped access. + */ + return !!drvdata->base; + } + + return false; +} + +/* + * Hide the ETM4x registers that may not be available on the + * hardware. + * There are certain management registers unavailable via system + * instructions. Make those sysfs attributes hidden on such + * systems. + */ +static umode_t +coresight_etm4x_attr_reg_implemented(struct kobject *kobj, + struct attribute *attr, int unused) +{ + struct device *dev = kobj_to_dev(kobj); + struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent); + struct device_attribute *d_attr; + u32 offset; + + d_attr = container_of(attr, struct device_attribute, attr); + offset = coresight_etm4x_attr_to_offset(d_attr); + + if (etm4x_register_implemented(drvdata, offset)) + return attr->mode; + return 0; +} + +#define coresight_etm4x_reg(name, offset) \ + &((struct dev_ext_attribute[]) { \ + { \ + __ATTR(name, 0444, coresight_etm4x_reg_show, NULL), \ + (void *)(unsigned long)offset \ + } \ + })[0].attr.attr static struct attribute *coresight_etmv4_mgmt_attrs[] = { - &dev_attr_trcoslsr.attr, - &dev_attr_trcpdcr.attr, - &dev_attr_trcpdsr.attr, - &dev_attr_trclsr.attr, - &dev_attr_trcconfig.attr, - &dev_attr_trctraceid.attr, - &dev_attr_trcauthstatus.attr, - &dev_attr_trcdevid.attr, - &dev_attr_trcdevtype.attr, - &dev_attr_trcpidr0.attr, - &dev_attr_trcpidr1.attr, - &dev_attr_trcpidr2.attr, - &dev_attr_trcpidr3.attr, + coresight_etm4x_reg(trcpdcr, TRCPDCR), + coresight_etm4x_reg(trcpdsr, TRCPDSR), + coresight_etm4x_reg(trclsr, TRCLSR), + coresight_etm4x_reg(trcauthstatus, TRCAUTHSTATUS), + coresight_etm4x_reg(trcdevid, TRCDEVID), + coresight_etm4x_reg(trcdevtype, TRCDEVTYPE), + coresight_etm4x_reg(trcpidr0, TRCPIDR0), + coresight_etm4x_reg(trcpidr1, TRCPIDR1), + coresight_etm4x_reg(trcpidr2, TRCPIDR2), + coresight_etm4x_reg(trcpidr3, TRCPIDR3), + coresight_etm4x_reg(trcoslsr, TRCOSLSR), + coresight_etm4x_reg(trcconfig, TRCCONFIGR), + coresight_etm4x_reg(trctraceid, TRCTRACEIDR), + coresight_etm4x_reg(trcdevarch, TRCDEVARCH), NULL, }; -coresight_etm4x_cross_read(trcidr0, TRCIDR0); -coresight_etm4x_cross_read(trcidr1, TRCIDR1); -coresight_etm4x_cross_read(trcidr2, TRCIDR2); -coresight_etm4x_cross_read(trcidr3, TRCIDR3); -coresight_etm4x_cross_read(trcidr4, TRCIDR4); -coresight_etm4x_cross_read(trcidr5, TRCIDR5); -/* trcidr[6,7] are reserved */ -coresight_etm4x_cross_read(trcidr8, TRCIDR8); -coresight_etm4x_cross_read(trcidr9, TRCIDR9); -coresight_etm4x_cross_read(trcidr10, TRCIDR10); -coresight_etm4x_cross_read(trcidr11, TRCIDR11); -coresight_etm4x_cross_read(trcidr12, TRCIDR12); -coresight_etm4x_cross_read(trcidr13, TRCIDR13); - static struct attribute *coresight_etmv4_trcidr_attrs[] = { - &dev_attr_trcidr0.attr, - &dev_attr_trcidr1.attr, - &dev_attr_trcidr2.attr, - &dev_attr_trcidr3.attr, - &dev_attr_trcidr4.attr, - &dev_attr_trcidr5.attr, + coresight_etm4x_reg(trcidr0, TRCIDR0), + coresight_etm4x_reg(trcidr1, TRCIDR1), + coresight_etm4x_reg(trcidr2, TRCIDR2), + coresight_etm4x_reg(trcidr3, TRCIDR3), + coresight_etm4x_reg(trcidr4, TRCIDR4), + coresight_etm4x_reg(trcidr5, TRCIDR5), /* trcidr[6,7] are reserved */ - &dev_attr_trcidr8.attr, - &dev_attr_trcidr9.attr, - &dev_attr_trcidr10.attr, - &dev_attr_trcidr11.attr, - &dev_attr_trcidr12.attr, - &dev_attr_trcidr13.attr, + coresight_etm4x_reg(trcidr8, TRCIDR8), + coresight_etm4x_reg(trcidr9, TRCIDR9), + coresight_etm4x_reg(trcidr10, TRCIDR10), + coresight_etm4x_reg(trcidr11, TRCIDR11), + coresight_etm4x_reg(trcidr12, TRCIDR12), + coresight_etm4x_reg(trcidr13, TRCIDR13), NULL, }; @@ -2418,6 +2468,7 @@ static const struct attribute_group coresight_etmv4_group = { }; static const struct attribute_group coresight_etmv4_mgmt_group = { + .is_visible = coresight_etm4x_attr_reg_implemented, .attrs = coresight_etmv4_mgmt_attrs, .name = "mgmt", }; diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 3dd3e0633328..0af60571aa23 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -45,13 +45,13 @@ #define TRCVDSACCTLR 0x0A4 #define TRCVDARCCTLR 0x0A8 /* Derived resources registers */ -#define TRCSEQEVRn(n) (0x100 + (n * 4)) +#define TRCSEQEVRn(n) (0x100 + (n * 4)) /* n = 0-2 */ #define TRCSEQRSTEVR 0x118 #define TRCSEQSTR 0x11C #define TRCEXTINSELR 0x120 -#define TRCCNTRLDVRn(n) (0x140 + (n * 4)) -#define TRCCNTCTLRn(n) (0x150 + (n * 4)) -#define TRCCNTVRn(n) (0x160 + (n * 4)) +#define TRCCNTRLDVRn(n) (0x140 + (n * 4)) /* n = 0-3 */ +#define TRCCNTCTLRn(n) (0x150 + (n * 4)) /* n = 0-3 */ +#define TRCCNTVRn(n) (0x160 + (n * 4)) /* n = 0-3 */ /* ID registers */ #define TRCIDR8 0x180 #define TRCIDR9 0x184 @@ -60,7 +60,7 @@ #define TRCIDR12 0x190 #define TRCIDR13 0x194 #define TRCIMSPEC0 0x1C0 -#define TRCIMSPECn(n) (0x1C0 + (n * 4)) +#define TRCIMSPECn(n) (0x1C0 + (n * 4)) /* n = 1-7 */ #define TRCIDR0 0x1E0 #define TRCIDR1 0x1E4 #define TRCIDR2 0x1E8 @@ -69,9 +69,12 @@ #define TRCIDR5 0x1F4 #define TRCIDR6 0x1F8 #define TRCIDR7 0x1FC -/* Resource selection registers */ +/* + * Resource selection registers, n = 2-31. + * First pair (regs 0, 1) is always present and is reserved. + */ #define TRCRSCTLRn(n) (0x200 + (n * 4)) -/* Single-shot comparator registers */ +/* Single-shot comparator registers, n = 0-7 */ #define TRCSSCCRn(n) (0x280 + (n * 4)) #define TRCSSCSRn(n) (0x2A0 + (n * 4)) #define TRCSSPCICRn(n) (0x2C0 + (n * 4)) @@ -81,11 +84,13 @@ #define TRCPDCR 0x310 #define TRCPDSR 0x314 /* Trace registers (0x318-0xEFC) */ -/* Comparator registers */ +/* Address Comparator registers n = 0-15 */ #define TRCACVRn(n) (0x400 + (n * 8)) #define TRCACATRn(n) (0x480 + (n * 8)) +/* Data Value Comparator Value registers, n = 0-7 */ #define TRCDVCVRn(n) (0x500 + (n * 16)) #define TRCDVCMRn(n) (0x580 + (n * 16)) +/* ContextID/Virtual ContextID comparators, n = 0-7 */ #define TRCCIDCVRn(n) (0x600 + (n * 8)) #define TRCVMIDCVRn(n) (0x640 + (n * 8)) #define TRCCIDCCTLR0 0x680 @@ -121,6 +126,332 @@ #define TRCCIDR2 0xFF8 #define TRCCIDR3 0xFFC +/* + * System instructions to access ETM registers. + * See ETMv4.4 spec ARM IHI0064F section 4.3.6 System instructions + */ +#define ETM4x_OFFSET_TO_REG(x) ((x) >> 2) + +#define ETM4x_CRn(n) (((n) >> 7) & 0x7) +#define ETM4x_Op2(n) (((n) >> 4) & 0x7) +#define ETM4x_CRm(n) ((n) & 0xf) + +#include <asm/sysreg.h> +#define ETM4x_REG_NUM_TO_SYSREG(n) \ + sys_reg(2, 1, ETM4x_CRn(n), ETM4x_CRm(n), ETM4x_Op2(n)) + +#define READ_ETM4x_REG(reg) \ + read_sysreg_s(ETM4x_REG_NUM_TO_SYSREG((reg))) +#define WRITE_ETM4x_REG(val, reg) \ + write_sysreg_s(val, ETM4x_REG_NUM_TO_SYSREG((reg))) + +#define read_etm4x_sysreg_const_offset(offset) \ + READ_ETM4x_REG(ETM4x_OFFSET_TO_REG(offset)) + +#define write_etm4x_sysreg_const_offset(val, offset) \ + WRITE_ETM4x_REG(val, ETM4x_OFFSET_TO_REG(offset)) + +#define CASE_READ(res, x) \ + case (x): { (res) = read_etm4x_sysreg_const_offset((x)); break; } + +#define CASE_WRITE(val, x) \ + case (x): { write_etm4x_sysreg_const_offset((val), (x)); break; } + +#define CASE_NOP(__unused, x) \ + case (x): /* fall through */ + +/* List of registers accessible via System instructions */ +#define ETM_SYSREG_LIST(op, val) \ + CASE_##op((val), TRCPRGCTLR) \ + CASE_##op((val), TRCPROCSELR) \ + CASE_##op((val), TRCSTATR) \ + CASE_##op((val), TRCCONFIGR) \ + CASE_##op((val), TRCAUXCTLR) \ + CASE_##op((val), TRCEVENTCTL0R) \ + CASE_##op((val), TRCEVENTCTL1R) \ + CASE_##op((val), TRCSTALLCTLR) \ + CASE_##op((val), TRCTSCTLR) \ + CASE_##op((val), TRCSYNCPR) \ + CASE_##op((val), TRCCCCTLR) \ + CASE_##op((val), TRCBBCTLR) \ + CASE_##op((val), TRCTRACEIDR) \ + CASE_##op((val), TRCQCTLR) \ + CASE_##op((val), TRCVICTLR) \ + CASE_##op((val), TRCVIIECTLR) \ + CASE_##op((val), TRCVISSCTLR) \ + CASE_##op((val), TRCVIPCSSCTLR) \ + CASE_##op((val), TRCVDCTLR) \ + CASE_##op((val), TRCVDSACCTLR) \ + CASE_##op((val), TRCVDARCCTLR) \ + CASE_##op((val), TRCSEQEVRn(0)) \ + CASE_##op((val), TRCSEQEVRn(1)) \ + CASE_##op((val), TRCSEQEVRn(2)) \ + CASE_##op((val), TRCSEQRSTEVR) \ + CASE_##op((val), TRCSEQSTR) \ + CASE_##op((val), TRCEXTINSELR) \ + CASE_##op((val), TRCCNTRLDVRn(0)) \ + CASE_##op((val), TRCCNTRLDVRn(1)) \ + CASE_##op((val), TRCCNTRLDVRn(2)) \ + CASE_##op((val), TRCCNTRLDVRn(3)) \ + CASE_##op((val), TRCCNTCTLRn(0)) \ + CASE_##op((val), TRCCNTCTLRn(1)) \ + CASE_##op((val), TRCCNTCTLRn(2)) \ + CASE_##op((val), TRCCNTCTLRn(3)) \ + CASE_##op((val), TRCCNTVRn(0)) \ + CASE_##op((val), TRCCNTVRn(1)) \ + CASE_##op((val), TRCCNTVRn(2)) \ + CASE_##op((val), TRCCNTVRn(3)) \ + CASE_##op((val), TRCIDR8) \ + CASE_##op((val), TRCIDR9) \ + CASE_##op((val), TRCIDR10) \ + CASE_##op((val), TRCIDR11) \ + CASE_##op((val), TRCIDR12) \ + CASE_##op((val), TRCIDR13) \ + CASE_##op((val), TRCIMSPECn(0)) \ + CASE_##op((val), TRCIMSPECn(1)) \ + CASE_##op((val), TRCIMSPECn(2)) \ + CASE_##op((val), TRCIMSPECn(3)) \ + CASE_##op((val), TRCIMSPECn(4)) \ + CASE_##op((val), TRCIMSPECn(5)) \ + CASE_##op((val), TRCIMSPECn(6)) \ + CASE_##op((val), TRCIMSPECn(7)) \ + CASE_##op((val), TRCIDR0) \ + CASE_##op((val), TRCIDR1) \ + CASE_##op((val), TRCIDR2) \ + CASE_##op((val), TRCIDR3) \ + CASE_##op((val), TRCIDR4) \ + CASE_##op((val), TRCIDR5) \ + CASE_##op((val), TRCIDR6) \ + CASE_##op((val), TRCIDR7) \ + CASE_##op((val), TRCRSCTLRn(2)) \ + CASE_##op((val), TRCRSCTLRn(3)) \ + CASE_##op((val), TRCRSCTLRn(4)) \ + CASE_##op((val), TRCRSCTLRn(5)) \ + CASE_##op((val), TRCRSCTLRn(6)) \ + CASE_##op((val), TRCRSCTLRn(7)) \ + CASE_##op((val), TRCRSCTLRn(8)) \ + CASE_##op((val), TRCRSCTLRn(9)) \ + CASE_##op((val), TRCRSCTLRn(10)) \ + CASE_##op((val), TRCRSCTLRn(11)) \ + CASE_##op((val), TRCRSCTLRn(12)) \ + CASE_##op((val), TRCRSCTLRn(13)) \ + CASE_##op((val), TRCRSCTLRn(14)) \ + CASE_##op((val), TRCRSCTLRn(15)) \ + CASE_##op((val), TRCRSCTLRn(16)) \ + CASE_##op((val), TRCRSCTLRn(17)) \ + CASE_##op((val), TRCRSCTLRn(18)) \ + CASE_##op((val), TRCRSCTLRn(19)) \ + CASE_##op((val), TRCRSCTLRn(20)) \ + CASE_##op((val), TRCRSCTLRn(21)) \ + CASE_##op((val), TRCRSCTLRn(22)) \ + CASE_##op((val), TRCRSCTLRn(23)) \ + CASE_##op((val), TRCRSCTLRn(24)) \ + CASE_##op((val), TRCRSCTLRn(25)) \ + CASE_##op((val), TRCRSCTLRn(26)) \ + CASE_##op((val), TRCRSCTLRn(27)) \ + CASE_##op((val), TRCRSCTLRn(28)) \ + CASE_##op((val), TRCRSCTLRn(29)) \ + CASE_##op((val), TRCRSCTLRn(30)) \ + CASE_##op((val), TRCRSCTLRn(31)) \ + CASE_##op((val), TRCSSCCRn(0)) \ + CASE_##op((val), TRCSSCCRn(1)) \ + CASE_##op((val), TRCSSCCRn(2)) \ + CASE_##op((val), TRCSSCCRn(3)) \ + CASE_##op((val), TRCSSCCRn(4)) \ + CASE_##op((val), TRCSSCCRn(5)) \ + CASE_##op((val), TRCSSCCRn(6)) \ + CASE_##op((val), TRCSSCCRn(7)) \ + CASE_##op((val), TRCSSCSRn(0)) \ + CASE_##op((val), TRCSSCSRn(1)) \ + CASE_##op((val), TRCSSCSRn(2)) \ + CASE_##op((val), TRCSSCSRn(3)) \ + CASE_##op((val), TRCSSCSRn(4)) \ + CASE_##op((val), TRCSSCSRn(5)) \ + CASE_##op((val), TRCSSCSRn(6)) \ + CASE_##op((val), TRCSSCSRn(7)) \ + CASE_##op((val), TRCSSPCICRn(0)) \ + CASE_##op((val), TRCSSPCICRn(1)) \ + CASE_##op((val), TRCSSPCICRn(2)) \ + CASE_##op((val), TRCSSPCICRn(3)) \ + CASE_##op((val), TRCSSPCICRn(4)) \ + CASE_##op((val), TRCSSPCICRn(5)) \ + CASE_##op((val), TRCSSPCICRn(6)) \ + CASE_##op((val), TRCSSPCICRn(7)) \ + CASE_##op((val), TRCOSLAR) \ + CASE_##op((val), TRCOSLSR) \ + CASE_##op((val), TRCACVRn(0)) \ + CASE_##op((val), TRCACVRn(1)) \ + CASE_##op((val), TRCACVRn(2)) \ + CASE_##op((val), TRCACVRn(3)) \ + CASE_##op((val), TRCACVRn(4)) \ + CASE_##op((val), TRCACVRn(5)) \ + CASE_##op((val), TRCACVRn(6)) \ + CASE_##op((val), TRCACVRn(7)) \ + CASE_##op((val), TRCACVRn(8)) \ + CASE_##op((val), TRCACVRn(9)) \ + CASE_##op((val), TRCACVRn(10)) \ + CASE_##op((val), TRCACVRn(11)) \ + CASE_##op((val), TRCACVRn(12)) \ + CASE_##op((val), TRCACVRn(13)) \ + CASE_##op((val), TRCACVRn(14)) \ + CASE_##op((val), TRCACVRn(15)) \ + CASE_##op((val), TRCACATRn(0)) \ + CASE_##op((val), TRCACATRn(1)) \ + CASE_##op((val), TRCACATRn(2)) \ + CASE_##op((val), TRCACATRn(3)) \ + CASE_##op((val), TRCACATRn(4)) \ + CASE_##op((val), TRCACATRn(5)) \ + CASE_##op((val), TRCACATRn(6)) \ + CASE_##op((val), TRCACATRn(7)) \ + CASE_##op((val), TRCACATRn(8)) \ + CASE_##op((val), TRCACATRn(9)) \ + CASE_##op((val), TRCACATRn(10)) \ + CASE_##op((val), TRCACATRn(11)) \ + CASE_##op((val), TRCACATRn(12)) \ + CASE_##op((val), TRCACATRn(13)) \ + CASE_##op((val), TRCACATRn(14)) \ + CASE_##op((val), TRCACATRn(15)) \ + CASE_##op((val), TRCDVCVRn(0)) \ + CASE_##op((val), TRCDVCVRn(1)) \ + CASE_##op((val), TRCDVCVRn(2)) \ + CASE_##op((val), TRCDVCVRn(3)) \ + CASE_##op((val), TRCDVCVRn(4)) \ + CASE_##op((val), TRCDVCVRn(5)) \ + CASE_##op((val), TRCDVCVRn(6)) \ + CASE_##op((val), TRCDVCVRn(7)) \ + CASE_##op((val), TRCDVCMRn(0)) \ + CASE_##op((val), TRCDVCMRn(1)) \ + CASE_##op((val), TRCDVCMRn(2)) \ + CASE_##op((val), TRCDVCMRn(3)) \ + CASE_##op((val), TRCDVCMRn(4)) \ + CASE_##op((val), TRCDVCMRn(5)) \ + CASE_##op((val), TRCDVCMRn(6)) \ + CASE_##op((val), TRCDVCMRn(7)) \ + CASE_##op((val), TRCCIDCVRn(0)) \ + CASE_##op((val), TRCCIDCVRn(1)) \ + CASE_##op((val), TRCCIDCVRn(2)) \ + CASE_##op((val), TRCCIDCVRn(3)) \ + CASE_##op((val), TRCCIDCVRn(4)) \ + CASE_##op((val), TRCCIDCVRn(5)) \ + CASE_##op((val), TRCCIDCVRn(6)) \ + CASE_##op((val), TRCCIDCVRn(7)) \ + CASE_##op((val), TRCVMIDCVRn(0)) \ + CASE_##op((val), TRCVMIDCVRn(1)) \ + CASE_##op((val), TRCVMIDCVRn(2)) \ + CASE_##op((val), TRCVMIDCVRn(3)) \ + CASE_##op((val), TRCVMIDCVRn(4)) \ + CASE_##op((val), TRCVMIDCVRn(5)) \ + CASE_##op((val), TRCVMIDCVRn(6)) \ + CASE_##op((val), TRCVMIDCVRn(7)) \ + CASE_##op((val), TRCCIDCCTLR0) \ + CASE_##op((val), TRCCIDCCTLR1) \ + CASE_##op((val), TRCVMIDCCTLR0) \ + CASE_##op((val), TRCVMIDCCTLR1) \ + CASE_##op((val), TRCCLAIMSET) \ + CASE_##op((val), TRCCLAIMCLR) \ + CASE_##op((val), TRCAUTHSTATUS) \ + CASE_##op((val), TRCDEVARCH) \ + CASE_##op((val), TRCDEVID) + +/* List of registers only accessible via memory-mapped interface */ +#define ETM_MMAP_LIST(op, val) \ + CASE_##op((val), TRCDEVTYPE) \ + CASE_##op((val), TRCPDCR) \ + CASE_##op((val), TRCPDSR) \ + CASE_##op((val), TRCDEVAFF0) \ + CASE_##op((val), TRCDEVAFF1) \ + CASE_##op((val), TRCLAR) \ + CASE_##op((val), TRCLSR) \ + CASE_##op((val), TRCITCTRL) \ + CASE_##op((val), TRCPIDR4) \ + CASE_##op((val), TRCPIDR0) \ + CASE_##op((val), TRCPIDR1) \ + CASE_##op((val), TRCPIDR2) \ + CASE_##op((val), TRCPIDR3) + +#define ETM4x_READ_SYSREG_CASES(res) ETM_SYSREG_LIST(READ, (res)) +#define ETM4x_WRITE_SYSREG_CASES(val) ETM_SYSREG_LIST(WRITE, (val)) + +#define ETM4x_SYSREG_LIST_CASES ETM_SYSREG_LIST(NOP, __unused) +#define ETM4x_MMAP_LIST_CASES ETM_MMAP_LIST(NOP, __unused) + +#define read_etm4x_sysreg_offset(offset, _64bit) \ + ({ \ + u64 __val; \ + \ + if (__builtin_constant_p((offset))) \ + __val = read_etm4x_sysreg_const_offset((offset)); \ + else \ + __val = etm4x_sysreg_read((offset), true, (_64bit)); \ + __val; \ + }) + +#define write_etm4x_sysreg_offset(val, offset, _64bit) \ + do { \ + if (__builtin_constant_p((offset))) \ + write_etm4x_sysreg_const_offset((val), \ + (offset)); \ + else \ + etm4x_sysreg_write((val), (offset), true, \ + (_64bit)); \ + } while (0) + + +#define etm4x_relaxed_read32(csa, offset) \ + ((u32)((csa)->io_mem ? \ + readl_relaxed((csa)->base + (offset)) : \ + read_etm4x_sysreg_offset((offset), false))) + +#define etm4x_relaxed_read64(csa, offset) \ + ((u64)((csa)->io_mem ? \ + readq_relaxed((csa)->base + (offset)) : \ + read_etm4x_sysreg_offset((offset), true))) + +#define etm4x_read32(csa, offset) \ + ({ \ + u32 __val = etm4x_relaxed_read32((csa), (offset)); \ + __iormb(__val); \ + __val; \ + }) + +#define etm4x_read64(csa, offset) \ + ({ \ + u64 __val = etm4x_relaxed_read64((csa), (offset)); \ + __iormb(__val); \ + __val; \ + }) + +#define etm4x_relaxed_write32(csa, val, offset) \ + do { \ + if ((csa)->io_mem) \ + writel_relaxed((val), (csa)->base + (offset)); \ + else \ + write_etm4x_sysreg_offset((val), (offset), \ + false); \ + } while (0) + +#define etm4x_relaxed_write64(csa, val, offset) \ + do { \ + if ((csa)->io_mem) \ + writeq_relaxed((val), (csa)->base + (offset)); \ + else \ + write_etm4x_sysreg_offset((val), (offset), \ + true); \ + } while (0) + +#define etm4x_write32(csa, val, offset) \ + do { \ + __iowmb(); \ + etm4x_relaxed_write32((csa), (val), (offset)); \ + } while (0) + +#define etm4x_write64(csa, val, offset) \ + do { \ + __iowmb(); \ + etm4x_relaxed_write64((csa), (val), (offset)); \ + } while (0) + + /* ETMv4 resources */ #define ETM_MAX_NR_PE 8 #define ETMv4_MAX_CNTR 4 @@ -137,7 +468,6 @@ #define ETM_MAX_RES_SEL 32 #define ETM_MAX_SS_CMP 8 -#define ETM_ARCH_V4 0x40 #define ETMv4_SYNC_MASK 0x1F #define ETM_CYC_THRESHOLD_MASK 0xFFF #define ETM_CYC_THRESHOLD_DEFAULT 0x100 @@ -175,34 +505,150 @@ ETM_MODE_EXCL_KERN | \ ETM_MODE_EXCL_USER) +/* + * TRCDEVARCH Bit field definitions + * Bits[31:21] - ARCHITECT = Always Arm Ltd. + * * Bits[31:28] = 0x4 + * * Bits[27:21] = 0b0111011 + * Bit[20] - PRESENT, Indicates the presence of this register. + * + * Bit[19:16] - REVISION, Revision of the architecture. + * + * Bit[15:0] - ARCHID, Identifies this component as an ETM + * * Bits[15:12] - architecture version of ETM + * * = 4 for ETMv4 + * * Bits[11:0] = 0xA13, architecture part number for ETM. + */ +#define ETM_DEVARCH_ARCHITECT_MASK GENMASK(31, 21) +#define ETM_DEVARCH_ARCHITECT_ARM ((0x4 << 28) | (0b0111011 << 21)) +#define ETM_DEVARCH_PRESENT BIT(20) +#define ETM_DEVARCH_REVISION_SHIFT 16 +#define ETM_DEVARCH_REVISION_MASK GENMASK(19, 16) +#define ETM_DEVARCH_REVISION(x) \ + (((x) & ETM_DEVARCH_REVISION_MASK) >> ETM_DEVARCH_REVISION_SHIFT) +#define ETM_DEVARCH_ARCHID_MASK GENMASK(15, 0) +#define ETM_DEVARCH_ARCHID_ARCH_VER_SHIFT 12 +#define ETM_DEVARCH_ARCHID_ARCH_VER_MASK GENMASK(15, 12) +#define ETM_DEVARCH_ARCHID_ARCH_VER(x) \ + (((x) & ETM_DEVARCH_ARCHID_ARCH_VER_MASK) >> ETM_DEVARCH_ARCHID_ARCH_VER_SHIFT) + +#define ETM_DEVARCH_MAKE_ARCHID_ARCH_VER(ver) \ + (((ver) << ETM_DEVARCH_ARCHID_ARCH_VER_SHIFT) & ETM_DEVARCH_ARCHID_ARCH_VER_MASK) + +#define ETM_DEVARCH_ARCHID_ARCH_PART(x) ((x) & 0xfffUL) + +#define ETM_DEVARCH_MAKE_ARCHID(major) \ + ((ETM_DEVARCH_MAKE_ARCHID_ARCH_VER(major)) | ETM_DEVARCH_ARCHID_ARCH_PART(0xA13)) + +#define ETM_DEVARCH_ARCHID_ETMv4x ETM_DEVARCH_MAKE_ARCHID(0x4) + +#define ETM_DEVARCH_ID_MASK \ + (ETM_DEVARCH_ARCHITECT_MASK | ETM_DEVARCH_ARCHID_MASK | ETM_DEVARCH_PRESENT) +#define ETM_DEVARCH_ETMv4x_ARCH \ + (ETM_DEVARCH_ARCHITECT_ARM | ETM_DEVARCH_ARCHID_ETMv4x | ETM_DEVARCH_PRESENT) + #define TRCSTATR_IDLE_BIT 0 #define TRCSTATR_PMSTABLE_BIT 1 #define ETM_DEFAULT_ADDR_COMP 0 +#define TRCSSCSRn_PC BIT(3) + /* PowerDown Control Register bits */ #define TRCPDCR_PU BIT(3) -/* secure state access levels - TRCACATRn */ -#define ETM_EXLEVEL_S_APP BIT(8) -#define ETM_EXLEVEL_S_OS BIT(9) -#define ETM_EXLEVEL_S_HYP BIT(10) -#define ETM_EXLEVEL_S_MON BIT(11) -/* non-secure state access levels - TRCACATRn */ -#define ETM_EXLEVEL_NS_APP BIT(12) -#define ETM_EXLEVEL_NS_OS BIT(13) -#define ETM_EXLEVEL_NS_HYP BIT(14) -#define ETM_EXLEVEL_NS_NA BIT(15) +#define TRCACATR_EXLEVEL_SHIFT 8 + +/* + * Exception level mask for Secure and Non-Secure ELs. + * ETM defines the bits for EL control (e.g, TRVICTLR, TRCACTRn). + * The Secure and Non-Secure ELs are always to gether. + * Non-secure EL3 is never implemented. + * We use the following generic mask as they appear in different + * registers and this can be shifted for the appropriate + * fields. + */ +#define ETM_EXLEVEL_S_APP BIT(0) /* Secure EL0 */ +#define ETM_EXLEVEL_S_OS BIT(1) /* Secure EL1 */ +#define ETM_EXLEVEL_S_HYP BIT(2) /* Secure EL2 */ +#define ETM_EXLEVEL_S_MON BIT(3) /* Secure EL3/Monitor */ +#define ETM_EXLEVEL_NS_APP BIT(4) /* NonSecure EL0 */ +#define ETM_EXLEVEL_NS_OS BIT(5) /* NonSecure EL1 */ +#define ETM_EXLEVEL_NS_HYP BIT(6) /* NonSecure EL2 */ + +#define ETM_EXLEVEL_MASK (GENMASK(6, 0)) +#define ETM_EXLEVEL_S_MASK (GENMASK(3, 0)) +#define ETM_EXLEVEL_NS_MASK (GENMASK(6, 4)) -/* access level control in TRCVICTLR - same bits as TRCACATRn but shifted */ -#define ETM_EXLEVEL_LSHIFT_TRCVICTLR 8 +/* access level controls in TRCACATRn */ +#define TRCACATR_EXLEVEL_SHIFT 8 + +/* access level control in TRCVICTLR */ +#define TRCVICTLR_EXLEVEL_SHIFT 16 +#define TRCVICTLR_EXLEVEL_S_SHIFT 16 +#define TRCVICTLR_EXLEVEL_NS_SHIFT 20 /* secure / non secure masks - TRCVICTLR, IDR3 */ -#define ETM_EXLEVEL_S_VICTLR_MASK GENMASK(19, 16) -/* NS MON (EL3) mode never implemented */ -#define ETM_EXLEVEL_NS_VICTLR_MASK GENMASK(22, 20) +#define TRCVICTLR_EXLEVEL_MASK (ETM_EXLEVEL_MASK << TRCVICTLR_EXLEVEL_SHIFT) +#define TRCVICTLR_EXLEVEL_S_MASK (ETM_EXLEVEL_S_MASK << TRCVICTLR_EXLEVEL_SHIFT) +#define TRCVICTLR_EXLEVEL_NS_MASK (ETM_EXLEVEL_NS_MASK << TRCVICTLR_EXLEVEL_SHIFT) + +#define ETM_TRCIDR1_ARCH_MAJOR_SHIFT 8 +#define ETM_TRCIDR1_ARCH_MAJOR_MASK (0xfU << ETM_TRCIDR1_ARCH_MAJOR_SHIFT) +#define ETM_TRCIDR1_ARCH_MAJOR(x) \ + (((x) & ETM_TRCIDR1_ARCH_MAJOR_MASK) >> ETM_TRCIDR1_ARCH_MAJOR_SHIFT) +#define ETM_TRCIDR1_ARCH_MINOR_SHIFT 4 +#define ETM_TRCIDR1_ARCH_MINOR_MASK (0xfU << ETM_TRCIDR1_ARCH_MINOR_SHIFT) +#define ETM_TRCIDR1_ARCH_MINOR(x) \ + (((x) & ETM_TRCIDR1_ARCH_MINOR_MASK) >> ETM_TRCIDR1_ARCH_MINOR_SHIFT) +#define ETM_TRCIDR1_ARCH_SHIFT ETM_TRCIDR1_ARCH_MINOR_SHIFT +#define ETM_TRCIDR1_ARCH_MASK \ + (ETM_TRCIDR1_ARCH_MAJOR_MASK | ETM_TRCIDR1_ARCH_MINOR_MASK) +#define ETM_TRCIDR1_ARCH_ETMv4 0x4 + +/* + * Driver representation of the ETM architecture. + * The version of an ETM component can be detected from + * + * TRCDEVARCH - CoreSight architected register + * - Bits[15:12] - Major version + * - Bits[19:16] - Minor version + * TRCIDR1 - ETM architected register + * - Bits[11:8] - Major version + * - Bits[7:4] - Minor version + * We must rely on TRCDEVARCH for the version information, + * however we don't want to break the support for potential + * old implementations which might not implement it. Thus + * we fall back to TRCIDR1 if TRCDEVARCH is not implemented + * for memory mapped components. + * Now to make certain decisions easier based on the version + * we use an internal representation of the version in the + * driver, as follows : + * + * ETM_ARCH_VERSION[7:0], where : + * Bits[7:4] - Major version + * Bits[3:0] - Minro version + */ +#define ETM_ARCH_VERSION(major, minor) \ + ((((major) & 0xfU) << 4) | (((minor) & 0xfU))) +#define ETM_ARCH_MAJOR_VERSION(arch) (((arch) >> 4) & 0xfU) +#define ETM_ARCH_MINOR_VERSION(arch) ((arch) & 0xfU) + +#define ETM_ARCH_V4 ETM_ARCH_VERSION(4, 0) /* Interpretation of resource numbers change at ETM v4.3 architecture */ -#define ETM4X_ARCH_4V3 0x43 +#define ETM_ARCH_V4_3 ETM_ARCH_VERSION(4, 3) + +static inline u8 etm_devarch_to_arch(u32 devarch) +{ + return ETM_ARCH_VERSION(ETM_DEVARCH_ARCHID_ARCH_VER(devarch), + ETM_DEVARCH_REVISION(devarch)); +} + +static inline u8 etm_trcidr_to_arch(u32 trcidr1) +{ + return ETM_ARCH_VERSION(ETM_TRCIDR1_ARCH_MAJOR(trcidr1), + ETM_TRCIDR1_ARCH_MINOR(trcidr1)); +} enum etm_impdef_type { ETM4_IMPDEF_HISI_CORE_COMMIT, @@ -256,7 +702,7 @@ enum etm_impdef_type { * @vmid_mask0: VM ID comparator mask for comparator 0-3. * @vmid_mask1: VM ID comparator mask for comparator 4-7. * @ext_inp: External input selection. - * @arch: ETM architecture version (for arch dependent config). + * @s_ex_level: Secure ELs where tracing is supported. */ struct etmv4_config { u32 mode; @@ -300,7 +746,7 @@ struct etmv4_config { u32 vmid_mask0; u32 vmid_mask1; u32 ext_inp; - u8 arch; + u8 s_ex_level; }; /** @@ -369,7 +815,7 @@ struct etmv4_save_state { * @spinlock: Only one at a time pls. * @mode: This tracer's mode, i.e sysFS, Perf or disabled. * @cpu: The cpu this component is affined to. - * @arch: ETM version number. + * @arch: ETM architecture version. * @nr_pe: The number of processing entity available for tracing. * @nr_pe_cmp: The number of processing entity comparator inputs that are * available for tracing. @@ -491,4 +937,7 @@ enum etm_addr_ctxtype { extern const struct attribute_group *coresight_etmv4_groups[]; void etm4_config_trace_mode(struct etmv4_config *config); + +u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit); +void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit); #endif diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c index 01f8f9285168..b363dd6bc510 100644 --- a/drivers/hwtracing/coresight/coresight-funnel.c +++ b/drivers/hwtracing/coresight/coresight-funnel.c @@ -52,13 +52,14 @@ static int dynamic_funnel_enable_hw(struct funnel_drvdata *drvdata, int port) { u32 functl; int rc = 0; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); functl = readl_relaxed(drvdata->base + FUNNEL_FUNCTL); /* Claim the device only when we enable the first slave */ if (!(functl & FUNNEL_ENSx_MASK)) { - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (rc) goto done; } @@ -101,6 +102,7 @@ static void dynamic_funnel_disable_hw(struct funnel_drvdata *drvdata, int inport) { u32 functl; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); @@ -110,7 +112,7 @@ static void dynamic_funnel_disable_hw(struct funnel_drvdata *drvdata, /* Disclaim the device if none of the slaves are now active */ if (!(functl & FUNNEL_ENSx_MASK)) - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } @@ -242,6 +244,7 @@ static int funnel_probe(struct device *dev, struct resource *res) } drvdata->base = base; desc.groups = coresight_funnel_groups; + desc.access = CSDEV_ACCESS_IOMEM(base); } dev_set_drvdata(dev, drvdata); diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c index 34fc2f6f3ea9..b86acbc74cf0 100644 --- a/drivers/hwtracing/coresight/coresight-replicator.c +++ b/drivers/hwtracing/coresight/coresight-replicator.c @@ -45,12 +45,14 @@ struct replicator_drvdata { static void dynamic_replicator_reset(struct replicator_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + CS_UNLOCK(drvdata->base); - if (!coresight_claim_device_unlocked(drvdata->base)) { + if (!coresight_claim_device_unlocked(csdev)) { writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER0); writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER1); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); } CS_LOCK(drvdata->base); @@ -70,6 +72,7 @@ static int dynamic_replicator_enable(struct replicator_drvdata *drvdata, { int rc = 0; u32 id0val, id1val; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); @@ -84,7 +87,7 @@ static int dynamic_replicator_enable(struct replicator_drvdata *drvdata, id0val = id1val = 0xff; if (id0val == 0xff && id1val == 0xff) - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (!rc) { switch (outport) { @@ -140,6 +143,7 @@ static void dynamic_replicator_disable(struct replicator_drvdata *drvdata, int inport, int outport) { u32 reg; + struct coresight_device *csdev = drvdata->csdev; switch (outport) { case 0: @@ -160,7 +164,7 @@ static void dynamic_replicator_disable(struct replicator_drvdata *drvdata, if ((readl_relaxed(drvdata->base + REPLICATOR_IDFILTER0) == 0xff) && (readl_relaxed(drvdata->base + REPLICATOR_IDFILTER1) == 0xff)) - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } @@ -254,6 +258,7 @@ static int replicator_probe(struct device *dev, struct resource *res) } drvdata->base = base; desc.groups = replicator_groups; + desc.access = CSDEV_ACCESS_IOMEM(base); } if (fwnode_property_present(dev_fwnode(dev), diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index 423df0d30d9c..58062a5a8238 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -258,6 +258,7 @@ static void stm_disable(struct coresight_device *csdev, struct perf_event *event) { struct stm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct csdev_access *csa = &csdev->access; /* * For as long as the tracer isn't disabled another entity can't @@ -270,7 +271,7 @@ static void stm_disable(struct coresight_device *csdev, spin_unlock(&drvdata->spinlock); /* Wait until the engine has completely stopped */ - coresight_timeout(drvdata->base, STMTCSR, STMTCSR_BUSY_BIT, 0); + coresight_timeout(csa, STMTCSR, STMTCSR_BUSY_BIT, 0); pm_runtime_put(csdev->dev.parent); @@ -884,6 +885,7 @@ static int stm_probe(struct amba_device *adev, const struct amba_id *id) if (IS_ERR(base)) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); ret = stm_get_stimulus_area(dev, &ch_res); if (ret) diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c index e29b3914fc0f..74c6323d4d6a 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-core.c +++ b/drivers/hwtracing/coresight/coresight-tmc-core.c @@ -33,16 +33,20 @@ DEFINE_CORESIGHT_DEVLIST(etr_devs, "tmc_etr"); void tmc_wait_for_tmcready(struct tmc_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + struct csdev_access *csa = &csdev->access; + /* Ensure formatter, unformatter and hardware fifo are empty */ - if (coresight_timeout(drvdata->base, - TMC_STS, TMC_STS_TMCREADY_BIT, 1)) { - dev_err(&drvdata->csdev->dev, + if (coresight_timeout(csa, TMC_STS, TMC_STS_TMCREADY_BIT, 1)) { + dev_err(&csdev->dev, "timeout while waiting for TMC to be Ready\n"); } } void tmc_flush_and_stop(struct tmc_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + struct csdev_access *csa = &csdev->access; u32 ffcr; ffcr = readl_relaxed(drvdata->base + TMC_FFCR); @@ -51,9 +55,8 @@ void tmc_flush_and_stop(struct tmc_drvdata *drvdata) ffcr |= BIT(TMC_FFCR_FLUSHMAN_BIT); writel_relaxed(ffcr, drvdata->base + TMC_FFCR); /* Ensure flush completes */ - if (coresight_timeout(drvdata->base, - TMC_FFCR, TMC_FFCR_FLUSHMAN_BIT, 0)) { - dev_err(&drvdata->csdev->dev, + if (coresight_timeout(csa, TMC_FFCR, TMC_FFCR_FLUSHMAN_BIT, 0)) { + dev_err(&csdev->dev, "timeout while waiting for completion of Manual Flush\n"); } @@ -456,6 +459,7 @@ static int tmc_probe(struct amba_device *adev, const struct amba_id *id) } drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); spin_lock_init(&drvdata->spinlock); diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index 989d965f3d90..45b85edfc690 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -37,7 +37,7 @@ static void __tmc_etb_enable_hw(struct tmc_drvdata *drvdata) static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata) { - int rc = coresight_claim_device(drvdata->base); + int rc = coresight_claim_device(drvdata->csdev); if (rc) return rc; @@ -88,7 +88,7 @@ static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata) static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) { __tmc_etb_disable_hw(drvdata); - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(drvdata->csdev); } static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata) @@ -109,7 +109,7 @@ static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata) static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) { - int rc = coresight_claim_device(drvdata->base); + int rc = coresight_claim_device(drvdata->csdev); if (rc) return rc; @@ -120,11 +120,13 @@ static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + CS_UNLOCK(drvdata->base); tmc_flush_and_stop(drvdata); tmc_disable_hw(drvdata); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index bf5230e39c5b..acdb59e0e661 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -1040,7 +1040,7 @@ static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata, rc = tmc_etr_enable_catu(drvdata, etr_buf); if (rc) return rc; - rc = coresight_claim_device(drvdata->base); + rc = coresight_claim_device(drvdata->csdev); if (!rc) { drvdata->etr_buf = etr_buf; __tmc_etr_enable_hw(drvdata); @@ -1134,7 +1134,7 @@ void tmc_etr_disable_hw(struct tmc_drvdata *drvdata) __tmc_etr_disable_hw(drvdata); /* Disable CATU device if this ETR is connected to one */ tmc_etr_disable_catu(drvdata); - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(drvdata->csdev); /* Reset the ETR buf used by hardware */ drvdata->etr_buf = NULL; } diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c index f77c4b0ea4aa..34d37abd2c8d 100644 --- a/drivers/hwtracing/coresight/coresight-tpiu.c +++ b/drivers/hwtracing/coresight/coresight-tpiu.c @@ -60,49 +60,45 @@ struct tpiu_drvdata { struct coresight_device *csdev; }; -static void tpiu_enable_hw(struct tpiu_drvdata *drvdata) +static void tpiu_enable_hw(struct csdev_access *csa) { - CS_UNLOCK(drvdata->base); + CS_UNLOCK(csa->base); /* TODO: fill this up */ - CS_LOCK(drvdata->base); + CS_LOCK(csa->base); } static int tpiu_enable(struct coresight_device *csdev, u32 mode, void *__unused) { - struct tpiu_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - - tpiu_enable_hw(drvdata); + tpiu_enable_hw(&csdev->access); atomic_inc(csdev->refcnt); dev_dbg(&csdev->dev, "TPIU enabled\n"); return 0; } -static void tpiu_disable_hw(struct tpiu_drvdata *drvdata) +static void tpiu_disable_hw(struct csdev_access *csa) { - CS_UNLOCK(drvdata->base); + CS_UNLOCK(csa->base); /* Clear formatter and stop on flush */ - writel_relaxed(FFCR_STOP_FI, drvdata->base + TPIU_FFCR); + csdev_access_relaxed_write32(csa, FFCR_STOP_FI, TPIU_FFCR); /* Generate manual flush */ - writel_relaxed(FFCR_STOP_FI | FFCR_FON_MAN, drvdata->base + TPIU_FFCR); + csdev_access_relaxed_write32(csa, FFCR_STOP_FI | FFCR_FON_MAN, TPIU_FFCR); /* Wait for flush to complete */ - coresight_timeout(drvdata->base, TPIU_FFCR, FFCR_FON_MAN_BIT, 0); + coresight_timeout(csa, TPIU_FFCR, FFCR_FON_MAN_BIT, 0); /* Wait for formatter to stop */ - coresight_timeout(drvdata->base, TPIU_FFSR, FFSR_FT_STOPPED_BIT, 1); + coresight_timeout(csa, TPIU_FFSR, FFSR_FT_STOPPED_BIT, 1); - CS_LOCK(drvdata->base); + CS_LOCK(csa->base); } static int tpiu_disable(struct coresight_device *csdev) { - struct tpiu_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - if (atomic_dec_return(csdev->refcnt)) return -EBUSY; - tpiu_disable_hw(drvdata); + tpiu_disable_hw(&csdev->access); dev_dbg(&csdev->dev, "TPIU disabled\n"); return 0; @@ -149,9 +145,10 @@ static int tpiu_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); /* Disable tpiu to support older devices */ - tpiu_disable_hw(drvdata); + tpiu_disable_hw(&desc.access); pdata = coresight_get_platform_data(dev); if (IS_ERR(pdata)) diff --git a/drivers/i2c/busses/i2c-brcmstb.c b/drivers/i2c/busses/i2c-brcmstb.c index d4e0a0f6732a..ba766d24219e 100644 --- a/drivers/i2c/busses/i2c-brcmstb.c +++ b/drivers/i2c/busses/i2c-brcmstb.c @@ -316,7 +316,7 @@ static int brcmstb_send_i2c_cmd(struct brcmstb_i2c_dev *dev, goto cmd_out; } - if ((CMD_RD || CMD_WR) && + if ((cmd == CMD_RD || cmd == CMD_WR) && bsc_readl(dev, iic_enable) & BSC_IIC_EN_NOACK_MASK) { rc = -EREMOTEIO; dev_dbg(dev->device, "controller received NOACK intr for %s\n", diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h index 85307cfa7109..5392b82f68a4 100644 --- a/drivers/i2c/busses/i2c-designware-core.h +++ b/drivers/i2c/busses/i2c-designware-core.h @@ -38,6 +38,8 @@ #define DW_IC_CON_TX_EMPTY_CTRL BIT(8) #define DW_IC_CON_RX_FIFO_FULL_HLD_CTRL BIT(9) +#define DW_IC_DATA_CMD_DAT GENMASK(7, 0) + /* * Registers offset */ diff --git a/drivers/i2c/busses/i2c-designware-master.c b/drivers/i2c/busses/i2c-designware-master.c index d6425ad6e6a3..dd27b9dbe931 100644 --- a/drivers/i2c/busses/i2c-designware-master.c +++ b/drivers/i2c/busses/i2c-designware-master.c @@ -432,7 +432,7 @@ i2c_dw_read(struct dw_i2c_dev *dev) regmap_read(dev->map, DW_IC_DATA_CMD, &tmp); /* Ensure length byte is a valid value */ if (flags & I2C_M_RECV_LEN && - tmp <= I2C_SMBUS_BLOCK_MAX && tmp > 0) { + (tmp & DW_IC_DATA_CMD_DAT) <= I2C_SMBUS_BLOCK_MAX && tmp > 0) { len = i2c_dw_recv_len(dev, tmp); } *buf++ = tmp; diff --git a/drivers/i2c/busses/i2c-exynos5.c b/drivers/i2c/busses/i2c-exynos5.c index 20a9881a0d6c..5ac30d95650c 100644 --- a/drivers/i2c/busses/i2c-exynos5.c +++ b/drivers/i2c/busses/i2c-exynos5.c @@ -606,6 +606,7 @@ static void exynos5_i2c_message_start(struct exynos5_i2c *i2c, int stop) u32 i2c_ctl; u32 int_en = 0; u32 i2c_auto_conf = 0; + u32 i2c_addr = 0; u32 fifo_ctl; unsigned long flags; unsigned short trig_lvl; @@ -640,7 +641,12 @@ static void exynos5_i2c_message_start(struct exynos5_i2c *i2c, int stop) int_en |= HSI2C_INT_TX_ALMOSTEMPTY_EN; } - writel(HSI2C_SLV_ADDR_MAS(i2c->msg->addr), i2c->regs + HSI2C_ADDR); + i2c_addr = HSI2C_SLV_ADDR_MAS(i2c->msg->addr); + + if (i2c->op_clock >= I2C_MAX_FAST_MODE_PLUS_FREQ) + i2c_addr |= HSI2C_MASTER_ID(MASTER_ID(i2c->adap.nr)); + + writel(i2c_addr, i2c->regs + HSI2C_ADDR); writel(fifo_ctl, i2c->regs + HSI2C_FIFO_CTL); writel(i2c_ctl, i2c->regs + HSI2C_CTL); diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c index c3f584795911..214b4c913a13 100644 --- a/drivers/i2c/busses/i2c-qcom-geni.c +++ b/drivers/i2c/busses/i2c-qcom-geni.c @@ -375,32 +375,6 @@ static void geni_i2c_tx_msg_cleanup(struct geni_i2c_dev *gi2c, } } -static void geni_i2c_stop_xfer(struct geni_i2c_dev *gi2c) -{ - int ret; - u32 geni_status; - struct i2c_msg *cur; - - /* Resume device, as runtime suspend can happen anytime during transfer */ - ret = pm_runtime_get_sync(gi2c->se.dev); - if (ret < 0) { - dev_err(gi2c->se.dev, "Failed to resume device: %d\n", ret); - return; - } - - geni_status = readl_relaxed(gi2c->se.base + SE_GENI_STATUS); - if (geni_status & M_GENI_CMD_ACTIVE) { - cur = gi2c->cur; - geni_i2c_abort_xfer(gi2c); - if (cur->flags & I2C_M_RD) - geni_i2c_rx_msg_cleanup(gi2c, cur); - else - geni_i2c_tx_msg_cleanup(gi2c, cur); - } - - pm_runtime_put_sync_suspend(gi2c->se.dev); -} - static int geni_i2c_rx_one_msg(struct geni_i2c_dev *gi2c, struct i2c_msg *msg, u32 m_param) { @@ -676,13 +650,6 @@ static int geni_i2c_remove(struct platform_device *pdev) return 0; } -static void geni_i2c_shutdown(struct platform_device *pdev) -{ - struct geni_i2c_dev *gi2c = platform_get_drvdata(pdev); - - geni_i2c_stop_xfer(gi2c); -} - static int __maybe_unused geni_i2c_runtime_suspend(struct device *dev) { int ret; @@ -747,7 +714,6 @@ MODULE_DEVICE_TABLE(of, geni_i2c_dt_match); static struct platform_driver geni_i2c_driver = { .probe = geni_i2c_probe, .remove = geni_i2c_remove, - .shutdown = geni_i2c_shutdown, .driver = { .name = "geni_i2c", .pm = &geni_i2c_pm_ops, diff --git a/drivers/ide/falconide.c b/drivers/ide/falconide.c index 77af4c1a3f38..bb86d84558d9 100644 --- a/drivers/ide/falconide.c +++ b/drivers/ide/falconide.c @@ -164,6 +164,7 @@ static int __init falconide_init(struct platform_device *pdev) if (rc) goto err_free; + platform_set_drvdata(pdev, host); return 0; err_free: ide_host_free(host); @@ -174,7 +175,7 @@ err: static int falconide_remove(struct platform_device *pdev) { - struct ide_host *host = dev_get_drvdata(&pdev->dev); + struct ide_host *host = platform_get_drvdata(pdev); ide_host_remove(host); diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index be996dba040c..3d194bb60840 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -3651,6 +3651,7 @@ static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv, struct ib_cm_sidr_rep_param *param) { struct ib_mad_send_buf *msg; + unsigned long flags; int ret; lockdep_assert_held(&cm_id_priv->lock); @@ -3676,12 +3677,12 @@ static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv, return ret; } cm_id_priv->id.state = IB_CM_IDLE; - spin_lock_irq(&cm.lock); + spin_lock_irqsave(&cm.lock, flags); if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) { rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table); RB_CLEAR_NODE(&cm_id_priv->sidr_id_node); } - spin_unlock_irq(&cm.lock); + spin_unlock_irqrestore(&cm.lock, flags); return 0; } diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c index e47c5949013f..ff047eb024ab 100644 --- a/drivers/infiniband/core/uverbs_ioctl.c +++ b/drivers/infiniband/core/uverbs_ioctl.c @@ -91,7 +91,7 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm, } /** - * uverbs_alloc() - Quickly allocate memory for use with a bundle + * _uverbs_alloc() - Quickly allocate memory for use with a bundle * @bundle: The bundle * @size: Number of bytes to allocate * @flags: Allocator flags diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index ebc2a4355fa5..de3c2fc6f361 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -2073,8 +2073,10 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( num_alloc_xa_entries++; event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL); - if (!event_sub) + if (!event_sub) { + err = -ENOMEM; goto err; + } list_add_tail(&event_sub->event_list, &sub_list); uverbs_uobject_get(&ev_file->uobj); diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 374698186662..b103555b1f5d 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -1082,7 +1082,7 @@ end: return ret ? ret : npages; } -/** +/* * Parse a series of data segments for page fault handling. * * @dev: Pointer to mlx5 IB device diff --git a/drivers/infiniband/sw/rxe/Kconfig b/drivers/infiniband/sw/rxe/Kconfig index 452149066792..06b8dc5093f7 100644 --- a/drivers/infiniband/sw/rxe/Kconfig +++ b/drivers/infiniband/sw/rxe/Kconfig @@ -4,6 +4,7 @@ config RDMA_RXE depends on INET && PCI && INFINIBAND depends on INFINIBAND_VIRT_DMA select NET_UDP_TUNNEL + select CRYPTO select CRYPTO_CRC32 help This driver implements the InfiniBand RDMA transport over diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index a8ac791a1bb9..17a361b8dbb1 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -547,6 +547,7 @@ int rxe_completer(void *arg) struct sk_buff *skb = NULL; struct rxe_pkt_info *pkt = NULL; enum comp_state state; + int ret = 0; rxe_add_ref(qp); @@ -554,7 +555,8 @@ int rxe_completer(void *arg) qp->req.state == QP_STATE_RESET) { rxe_drain_resp_pkts(qp, qp->valid && qp->req.state == QP_STATE_ERROR); - goto exit; + ret = -EAGAIN; + goto done; } if (qp->comp.timeout) { @@ -564,8 +566,10 @@ int rxe_completer(void *arg) qp->comp.timeout_retry = 0; } - if (qp->req.need_retry) - goto exit; + if (qp->req.need_retry) { + ret = -EAGAIN; + goto done; + } state = COMPST_GET_ACK; @@ -636,8 +640,6 @@ int rxe_completer(void *arg) break; case COMPST_DONE: - if (pkt) - free_pkt(pkt); goto done; case COMPST_EXIT: @@ -660,7 +662,8 @@ int rxe_completer(void *arg) qp->qp_timeout_jiffies) mod_timer(&qp->retrans_timer, jiffies + qp->qp_timeout_jiffies); - goto exit; + ret = -EAGAIN; + goto done; case COMPST_ERROR_RETRY: /* we come here if the retry timer fired and we did @@ -672,18 +675,18 @@ int rxe_completer(void *arg) */ /* there is nothing to retry in this case */ - if (!wqe || (wqe->state == wqe_state_posted)) - goto exit; + if (!wqe || (wqe->state == wqe_state_posted)) { + pr_warn("Retry attempted without a valid wqe\n"); + ret = -EAGAIN; + goto done; + } /* if we've started a retry, don't start another * retry sequence, unless this is a timeout. */ if (qp->comp.started_retry && - !qp->comp.timeout_retry) { - if (pkt) - free_pkt(pkt); + !qp->comp.timeout_retry) goto done; - } if (qp->comp.retry_cnt > 0) { if (qp->comp.retry_cnt != 7) @@ -704,8 +707,6 @@ int rxe_completer(void *arg) qp->comp.started_retry = 1; rxe_run_task(&qp->req.task, 0); } - if (pkt) - free_pkt(pkt); goto done; } else { @@ -726,8 +727,8 @@ int rxe_completer(void *arg) mod_timer(&qp->rnr_nak_timer, jiffies + rnrnak_jiffies(aeth_syn(pkt) & ~AETH_TYPE_MASK)); - free_pkt(pkt); - goto exit; + ret = -EAGAIN; + goto done; } else { rxe_counter_inc(rxe, RXE_CNT_RNR_RETRY_EXCEEDED); @@ -740,25 +741,15 @@ int rxe_completer(void *arg) WARN_ON_ONCE(wqe->status == IB_WC_SUCCESS); do_complete(qp, wqe); rxe_qp_error(qp); - if (pkt) - free_pkt(pkt); - goto exit; + ret = -EAGAIN; + goto done; } } -exit: - /* we come here if we are done with processing and want the task to - * exit from the loop calling us - */ - WARN_ON_ONCE(skb); - rxe_drop_ref(qp); - return -EAGAIN; - done: - /* we come here if we have processed a packet we want the task to call - * us again to see if there is anything else to do - */ - WARN_ON_ONCE(skb); + if (pkt) + free_pkt(pkt); rxe_drop_ref(qp); - return 0; + + return ret; } diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 0701bd1ffd1a..01662727dca0 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -407,14 +407,22 @@ int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb) return 0; } +/* fix up a send packet to match the packets + * received from UDP before looping them back + */ void rxe_loopback(struct sk_buff *skb) { + struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); + if (skb->protocol == htons(ETH_P_IP)) skb_pull(skb, sizeof(struct iphdr)); else skb_pull(skb, sizeof(struct ipv6hdr)); - rxe_rcv(skb); + if (WARN_ON(!ib_device_try_get(&pkt->rxe->ib_dev))) + kfree_skb(skb); + else + rxe_rcv(skb); } struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c index 45d2f711bce2..7a49e27da23a 100644 --- a/drivers/infiniband/sw/rxe/rxe_recv.c +++ b/drivers/infiniband/sw/rxe/rxe_recv.c @@ -237,8 +237,6 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb) struct rxe_mc_elem *mce; struct rxe_qp *qp; union ib_gid dgid; - struct sk_buff *per_qp_skb; - struct rxe_pkt_info *per_qp_pkt; int err; if (skb->protocol == htons(ETH_P_IP)) @@ -250,10 +248,15 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb) /* lookup mcast group corresponding to mgid, takes a ref */ mcg = rxe_pool_get_key(&rxe->mc_grp_pool, &dgid); if (!mcg) - goto err1; /* mcast group not registered */ + goto drop; /* mcast group not registered */ spin_lock_bh(&mcg->mcg_lock); + /* this is unreliable datagram service so we let + * failures to deliver a multicast packet to a + * single QP happen and just move on and try + * the rest of them on the list + */ list_for_each_entry(mce, &mcg->qp_list, qp_list) { qp = mce->qp; @@ -266,39 +269,47 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb) if (err) continue; - /* for all but the last qp create a new clone of the - * skb and pass to the qp. If an error occurs in the - * checks for the last qp in the list we need to - * free the skb since it hasn't been passed on to - * rxe_rcv_pkt() which would free it later. + /* for all but the last QP create a new clone of the + * skb and pass to the QP. Pass the original skb to + * the last QP in the list. */ if (mce->qp_list.next != &mcg->qp_list) { - per_qp_skb = skb_clone(skb, GFP_ATOMIC); - if (WARN_ON(!ib_device_try_get(&rxe->ib_dev))) { - kfree_skb(per_qp_skb); + struct sk_buff *cskb; + struct rxe_pkt_info *cpkt; + + cskb = skb_clone(skb, GFP_ATOMIC); + if (unlikely(!cskb)) continue; + + if (WARN_ON(!ib_device_try_get(&rxe->ib_dev))) { + kfree_skb(cskb); + break; } + + cpkt = SKB_TO_PKT(cskb); + cpkt->qp = qp; + rxe_add_ref(qp); + rxe_rcv_pkt(cpkt, cskb); } else { - per_qp_skb = skb; - /* show we have consumed the skb */ - skb = NULL; + pkt->qp = qp; + rxe_add_ref(qp); + rxe_rcv_pkt(pkt, skb); + skb = NULL; /* mark consumed */ } - - if (unlikely(!per_qp_skb)) - continue; - - per_qp_pkt = SKB_TO_PKT(per_qp_skb); - per_qp_pkt->qp = qp; - rxe_add_ref(qp); - rxe_rcv_pkt(per_qp_pkt, per_qp_skb); } spin_unlock_bh(&mcg->mcg_lock); rxe_drop_ref(mcg); /* drop ref from rxe_pool_get_key. */ -err1: - /* free skb if not consumed */ + if (likely(!skb)) + return; + + /* This only occurs if one of the checks fails on the last + * QP in the list above + */ + +drop: kfree_skb(skb); ib_device_put(&rxe->ib_dev); } diff --git a/drivers/input/joydev.c b/drivers/input/joydev.c index a2b5fbba2d3b..430dc6975004 100644 --- a/drivers/input/joydev.c +++ b/drivers/input/joydev.c @@ -456,7 +456,7 @@ static int joydev_handle_JSIOCSAXMAP(struct joydev *joydev, if (IS_ERR(abspam)) return PTR_ERR(abspam); - for (i = 0; i < joydev->nabs; i++) { + for (i = 0; i < len && i < joydev->nabs; i++) { if (abspam[i] > ABS_MAX) { retval = -EINVAL; goto out; @@ -480,6 +480,9 @@ static int joydev_handle_JSIOCSBTNMAP(struct joydev *joydev, int i; int retval = 0; + if (len % sizeof(*keypam)) + return -EINVAL; + len = min(len, sizeof(joydev->keypam)); /* Validate the map. */ @@ -487,7 +490,7 @@ static int joydev_handle_JSIOCSBTNMAP(struct joydev *joydev, if (IS_ERR(keypam)) return PTR_ERR(keypam); - for (i = 0; i < joydev->nkey; i++) { + for (i = 0; i < (len / 2) && i < joydev->nkey; i++) { if (keypam[i] > KEY_MAX || keypam[i] < BTN_MISC) { retval = -EINVAL; goto out; diff --git a/drivers/input/joystick/Kconfig b/drivers/input/joystick/Kconfig index b080f0cfb068..5e38899058c1 100644 --- a/drivers/input/joystick/Kconfig +++ b/drivers/input/joystick/Kconfig @@ -382,4 +382,11 @@ config JOYSTICK_FSIA6B To compile this driver as a module, choose M here: the module will be called fsia6b. +config JOYSTICK_N64 + bool "N64 controller" + depends on MACH_NINTENDO64 + help + Say Y here if you want enable support for the four + built-in controller ports on the Nintendo 64 console. + endif diff --git a/drivers/input/joystick/Makefile b/drivers/input/joystick/Makefile index 58232b3057d3..31d720c9e493 100644 --- a/drivers/input/joystick/Makefile +++ b/drivers/input/joystick/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_JOYSTICK_INTERACT) += interact.o obj-$(CONFIG_JOYSTICK_JOYDUMP) += joydump.o obj-$(CONFIG_JOYSTICK_MAGELLAN) += magellan.o obj-$(CONFIG_JOYSTICK_MAPLE) += maplecontrol.o +obj-$(CONFIG_JOYSTICK_N64) += n64joy.o obj-$(CONFIG_JOYSTICK_PSXPAD_SPI) += psxpad-spi.o obj-$(CONFIG_JOYSTICK_PXRC) += pxrc.o obj-$(CONFIG_JOYSTICK_SIDEWINDER) += sidewinder.o @@ -37,4 +38,3 @@ obj-$(CONFIG_JOYSTICK_WARRIOR) += warrior.o obj-$(CONFIG_JOYSTICK_WALKERA0701) += walkera0701.o obj-$(CONFIG_JOYSTICK_XPAD) += xpad.o obj-$(CONFIG_JOYSTICK_ZHENHUA) += zhenhua.o - diff --git a/drivers/input/joystick/n64joy.c b/drivers/input/joystick/n64joy.c new file mode 100644 index 000000000000..8bcc529942bc --- /dev/null +++ b/drivers/input/joystick/n64joy.c @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for the four N64 controllers. + * + * Copyright (c) 2021 Lauri Kasanen + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/errno.h> +#include <linux/init.h> +#include <linux/input.h> +#include <linux/limits.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/timer.h> + +MODULE_AUTHOR("Lauri Kasanen <cand@gmx.com>"); +MODULE_DESCRIPTION("Driver for N64 controllers"); +MODULE_LICENSE("GPL"); + +#define PIF_RAM 0x1fc007c0 + +#define SI_DRAM_REG 0 +#define SI_READ_REG 1 +#define SI_WRITE_REG 4 +#define SI_STATUS_REG 6 + +#define SI_STATUS_DMA_BUSY BIT(0) +#define SI_STATUS_IO_BUSY BIT(1) + +#define N64_CONTROLLER_ID 0x0500 + +#define MAX_CONTROLLERS 4 + +static const char *n64joy_phys[MAX_CONTROLLERS] = { + "n64joy/port0", + "n64joy/port1", + "n64joy/port2", + "n64joy/port3", +}; + +struct n64joy_priv { + u64 si_buf[8] ____cacheline_aligned; + struct timer_list timer; + struct mutex n64joy_mutex; + struct input_dev *n64joy_dev[MAX_CONTROLLERS]; + u32 __iomem *reg_base; + u8 n64joy_opened; +}; + +struct joydata { + unsigned int: 16; /* unused */ + unsigned int err: 2; + unsigned int: 14; /* unused */ + + union { + u32 data; + + struct { + unsigned int a: 1; + unsigned int b: 1; + unsigned int z: 1; + unsigned int start: 1; + unsigned int up: 1; + unsigned int down: 1; + unsigned int left: 1; + unsigned int right: 1; + unsigned int: 2; /* unused */ + unsigned int l: 1; + unsigned int r: 1; + unsigned int c_up: 1; + unsigned int c_down: 1; + unsigned int c_left: 1; + unsigned int c_right: 1; + signed int x: 8; + signed int y: 8; + }; + }; +}; + +static void n64joy_write_reg(u32 __iomem *reg_base, const u8 reg, const u32 value) +{ + writel(value, reg_base + reg); +} + +static u32 n64joy_read_reg(u32 __iomem *reg_base, const u8 reg) +{ + return readl(reg_base + reg); +} + +static void n64joy_wait_si_dma(u32 __iomem *reg_base) +{ + while (n64joy_read_reg(reg_base, SI_STATUS_REG) & + (SI_STATUS_DMA_BUSY | SI_STATUS_IO_BUSY)) + cpu_relax(); +} + +static void n64joy_exec_pif(struct n64joy_priv *priv, const u64 in[8]) +{ + unsigned long flags; + + dma_cache_wback_inv((unsigned long) in, 8 * 8); + dma_cache_inv((unsigned long) priv->si_buf, 8 * 8); + + local_irq_save(flags); + + n64joy_wait_si_dma(priv->reg_base); + + barrier(); + n64joy_write_reg(priv->reg_base, SI_DRAM_REG, virt_to_phys(in)); + barrier(); + n64joy_write_reg(priv->reg_base, SI_WRITE_REG, PIF_RAM); + barrier(); + + n64joy_wait_si_dma(priv->reg_base); + + barrier(); + n64joy_write_reg(priv->reg_base, SI_DRAM_REG, virt_to_phys(priv->si_buf)); + barrier(); + n64joy_write_reg(priv->reg_base, SI_READ_REG, PIF_RAM); + barrier(); + + n64joy_wait_si_dma(priv->reg_base); + + local_irq_restore(flags); +} + +static const u64 polldata[] ____cacheline_aligned = { + 0xff010401ffffffff, + 0xff010401ffffffff, + 0xff010401ffffffff, + 0xff010401ffffffff, + 0xfe00000000000000, + 0, + 0, + 1 +}; + +static void n64joy_poll(struct timer_list *t) +{ + const struct joydata *data; + struct n64joy_priv *priv = container_of(t, struct n64joy_priv, timer); + struct input_dev *dev; + u32 i; + + n64joy_exec_pif(priv, polldata); + + data = (struct joydata *) priv->si_buf; + + for (i = 0; i < MAX_CONTROLLERS; i++) { + if (!priv->n64joy_dev[i]) + continue; + + dev = priv->n64joy_dev[i]; + + /* d-pad */ + input_report_key(dev, BTN_DPAD_UP, data[i].up); + input_report_key(dev, BTN_DPAD_DOWN, data[i].down); + input_report_key(dev, BTN_DPAD_LEFT, data[i].left); + input_report_key(dev, BTN_DPAD_RIGHT, data[i].right); + + /* c buttons */ + input_report_key(dev, BTN_FORWARD, data[i].c_up); + input_report_key(dev, BTN_BACK, data[i].c_down); + input_report_key(dev, BTN_LEFT, data[i].c_left); + input_report_key(dev, BTN_RIGHT, data[i].c_right); + + /* matching buttons */ + input_report_key(dev, BTN_START, data[i].start); + input_report_key(dev, BTN_Z, data[i].z); + + /* remaining ones: a, b, l, r */ + input_report_key(dev, BTN_0, data[i].a); + input_report_key(dev, BTN_1, data[i].b); + input_report_key(dev, BTN_2, data[i].l); + input_report_key(dev, BTN_3, data[i].r); + + input_report_abs(dev, ABS_X, data[i].x); + input_report_abs(dev, ABS_Y, data[i].y); + + input_sync(dev); + } + + mod_timer(&priv->timer, jiffies + msecs_to_jiffies(16)); +} + +static int n64joy_open(struct input_dev *dev) +{ + struct n64joy_priv *priv = input_get_drvdata(dev); + int err; + + err = mutex_lock_interruptible(&priv->n64joy_mutex); + if (err) + return err; + + if (!priv->n64joy_opened) { + /* + * We could use the vblank irq, but it's not important if + * the poll point slightly changes. + */ + timer_setup(&priv->timer, n64joy_poll, 0); + mod_timer(&priv->timer, jiffies + msecs_to_jiffies(16)); + } + + priv->n64joy_opened++; + + mutex_unlock(&priv->n64joy_mutex); + return err; +} + +static void n64joy_close(struct input_dev *dev) +{ + struct n64joy_priv *priv = input_get_drvdata(dev); + + mutex_lock(&priv->n64joy_mutex); + if (!--priv->n64joy_opened) + del_timer_sync(&priv->timer); + mutex_unlock(&priv->n64joy_mutex); +} + +static const u64 __initconst scandata[] ____cacheline_aligned = { + 0xff010300ffffffff, + 0xff010300ffffffff, + 0xff010300ffffffff, + 0xff010300ffffffff, + 0xfe00000000000000, + 0, + 0, + 1 +}; + +/* + * The target device is embedded and RAM-constrained. We save RAM + * by initializing in __init code that gets dropped late in boot. + * For the same reason there is no module or unloading support. + */ +static int __init n64joy_probe(struct platform_device *pdev) +{ + const struct joydata *data; + struct n64joy_priv *priv; + struct input_dev *dev; + int err = 0; + u32 i, j, found = 0; + + priv = kzalloc(sizeof(struct n64joy_priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + mutex_init(&priv->n64joy_mutex); + + priv->reg_base = devm_platform_ioremap_resource(pdev, 0); + if (!priv->reg_base) { + err = -EINVAL; + goto fail; + } + + /* The controllers are not hotpluggable, so we can scan in init */ + n64joy_exec_pif(priv, scandata); + + data = (struct joydata *) priv->si_buf; + + for (i = 0; i < MAX_CONTROLLERS; i++) { + if (!data[i].err && data[i].data >> 16 == N64_CONTROLLER_ID) { + found++; + + dev = priv->n64joy_dev[i] = input_allocate_device(); + if (!priv->n64joy_dev[i]) { + err = -ENOMEM; + goto fail; + } + + input_set_drvdata(dev, priv); + + dev->name = "N64 controller"; + dev->phys = n64joy_phys[i]; + dev->id.bustype = BUS_HOST; + dev->id.vendor = 0; + dev->id.product = data[i].data >> 16; + dev->id.version = 0; + dev->dev.parent = &pdev->dev; + + dev->open = n64joy_open; + dev->close = n64joy_close; + + /* d-pad */ + input_set_capability(dev, EV_KEY, BTN_DPAD_UP); + input_set_capability(dev, EV_KEY, BTN_DPAD_DOWN); + input_set_capability(dev, EV_KEY, BTN_DPAD_LEFT); + input_set_capability(dev, EV_KEY, BTN_DPAD_RIGHT); + /* c buttons */ + input_set_capability(dev, EV_KEY, BTN_LEFT); + input_set_capability(dev, EV_KEY, BTN_RIGHT); + input_set_capability(dev, EV_KEY, BTN_FORWARD); + input_set_capability(dev, EV_KEY, BTN_BACK); + /* matching buttons */ + input_set_capability(dev, EV_KEY, BTN_START); + input_set_capability(dev, EV_KEY, BTN_Z); + /* remaining ones: a, b, l, r */ + input_set_capability(dev, EV_KEY, BTN_0); + input_set_capability(dev, EV_KEY, BTN_1); + input_set_capability(dev, EV_KEY, BTN_2); + input_set_capability(dev, EV_KEY, BTN_3); + + for (j = 0; j < 2; j++) + input_set_abs_params(dev, ABS_X + j, + S8_MIN, S8_MAX, 0, 0); + + err = input_register_device(dev); + if (err) { + input_free_device(dev); + goto fail; + } + } + } + + pr_info("%u controller(s) connected\n", found); + + if (!found) + return -ENODEV; + + return 0; +fail: + for (i = 0; i < MAX_CONTROLLERS; i++) { + if (!priv->n64joy_dev[i]) + continue; + input_unregister_device(priv->n64joy_dev[i]); + } + return err; +} + +static struct platform_driver n64joy_driver = { + .driver = { + .name = "n64joy", + }, +}; + +static int __init n64joy_init(void) +{ + return platform_driver_probe(&n64joy_driver, n64joy_probe); +} + +module_init(n64joy_init); diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 8cc8ca4a9ac0..9f0d07dcbf06 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -305,6 +305,7 @@ static const struct xpad_device { { 0x1bad, 0xfd00, "Razer Onza TE", 0, XTYPE_XBOX360 }, { 0x1bad, 0xfd01, "Razer Onza", 0, XTYPE_XBOX360 }, { 0x20d6, 0x2001, "BDA Xbox Series X Wired Controller", 0, XTYPE_XBOXONE }, + { 0x20d6, 0x2009, "PowerA Enhanced Wired Controller for Xbox Series X|S", 0, XTYPE_XBOXONE }, { 0x20d6, 0x281f, "PowerA Wired Controller For Xbox 360", 0, XTYPE_XBOX360 }, { 0x2e24, 0x0652, "Hyperkin Duke X-Box One pad", 0, XTYPE_XBOXONE }, { 0x24c6, 0x5000, "Razer Atrox Arcade Stick", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 }, diff --git a/drivers/input/keyboard/Kconfig b/drivers/input/keyboard/Kconfig index 2b321c17054a..32d15809ae58 100644 --- a/drivers/input/keyboard/Kconfig +++ b/drivers/input/keyboard/Kconfig @@ -446,7 +446,7 @@ config KEYBOARD_MPR121 config KEYBOARD_SNVS_PWRKEY tristate "IMX SNVS Power Key Driver" - depends on ARCH_MXC || COMPILE_TEST + depends on ARCH_MXC || (COMPILE_TEST && HAS_IOMEM) depends on OF help This is the snvs powerkey driver for the Freescale i.MX application @@ -685,7 +685,7 @@ config KEYBOARD_OMAP config KEYBOARD_OMAP4 tristate "TI OMAP4+ keypad support" - depends on OF || ARCH_OMAP2PLUS + depends on (OF && HAS_IOMEM) || ARCH_OMAP2PLUS select INPUT_MATRIXKMAP help Say Y here if you want to use the OMAP4+ keypad. @@ -773,7 +773,7 @@ config KEYBOARD_CAP11XX config KEYBOARD_BCM tristate "Broadcom keypad driver" - depends on OF && HAVE_CLK + depends on OF && HAVE_CLK && HAS_IOMEM select INPUT_MATRIXKMAP default ARCH_BCM_CYGNUS help diff --git a/drivers/input/keyboard/applespi.c b/drivers/input/keyboard/applespi.c index d22223154177..eda1b23002b5 100644 --- a/drivers/input/keyboard/applespi.c +++ b/drivers/input/keyboard/applespi.c @@ -48,6 +48,7 @@ #include <linux/efi.h> #include <linux/input.h> #include <linux/input/mt.h> +#include <linux/ktime.h> #include <linux/leds.h> #include <linux/module.h> #include <linux/spinlock.h> @@ -409,7 +410,7 @@ struct applespi_data { unsigned int cmd_msg_cntr; /* lock to protect the above parameters and flags below */ spinlock_t cmd_msg_lock; - bool cmd_msg_queued; + ktime_t cmd_msg_queued; enum applespi_evt_type cmd_evt_type; struct led_classdev backlight_info; @@ -729,7 +730,7 @@ static void applespi_msg_complete(struct applespi_data *applespi, wake_up_all(&applespi->drain_complete); if (is_write_msg) { - applespi->cmd_msg_queued = false; + applespi->cmd_msg_queued = 0; applespi_send_cmd_msg(applespi); } @@ -748,6 +749,8 @@ static void applespi_async_write_complete(void *context) applespi->tx_status, APPLESPI_STATUS_SIZE); + udelay(SPI_RW_CHG_DELAY_US); + if (!applespi_check_write_status(applespi, applespi->wr_m.status)) { /* * If we got an error, we presumably won't get the expected @@ -771,8 +774,16 @@ static int applespi_send_cmd_msg(struct applespi_data *applespi) return 0; /* check whether send is in progress */ - if (applespi->cmd_msg_queued) - return 0; + if (applespi->cmd_msg_queued) { + if (ktime_ms_delta(ktime_get(), applespi->cmd_msg_queued) < 1000) + return 0; + + dev_warn(&applespi->spi->dev, "Command %d timed out\n", + applespi->cmd_evt_type); + + applespi->cmd_msg_queued = 0; + applespi->write_active = false; + } /* set up packet */ memset(packet, 0, APPLESPI_PACKET_SIZE); @@ -869,7 +880,7 @@ static int applespi_send_cmd_msg(struct applespi_data *applespi) return sts; } - applespi->cmd_msg_queued = true; + applespi->cmd_msg_queued = ktime_get_coarse(); applespi->write_active = true; return 0; @@ -1921,7 +1932,7 @@ static int __maybe_unused applespi_resume(struct device *dev) applespi->drain = false; applespi->have_cl_led_on = false; applespi->have_bl_level = 0; - applespi->cmd_msg_queued = false; + applespi->cmd_msg_queued = 0; applespi->read_active = false; applespi->write_active = false; diff --git a/drivers/input/keyboard/cros_ec_keyb.c b/drivers/input/keyboard/cros_ec_keyb.c index b379ed762878..38457d9641bd 100644 --- a/drivers/input/keyboard/cros_ec_keyb.c +++ b/drivers/input/keyboard/cros_ec_keyb.c @@ -27,6 +27,8 @@ #include <asm/unaligned.h> +#define MAX_NUM_TOP_ROW_KEYS 15 + /** * struct cros_ec_keyb - Structure representing EC keyboard device * @@ -42,6 +44,9 @@ * @idev: The input device for the matrix keys. * @bs_idev: The input device for non-matrix buttons and switches (or NULL). * @notifier: interrupt event notifier for transport devices + * @function_row_physmap: An array of the encoded rows/columns for the top + * row function keys, in an order from left to right + * @num_function_row_keys: The number of top row keys in a custom keyboard */ struct cros_ec_keyb { unsigned int rows; @@ -58,6 +63,9 @@ struct cros_ec_keyb { struct input_dev *idev; struct input_dev *bs_idev; struct notifier_block notifier; + + u16 function_row_physmap[MAX_NUM_TOP_ROW_KEYS]; + size_t num_function_row_keys; }; /** @@ -527,6 +535,11 @@ static int cros_ec_keyb_register_matrix(struct cros_ec_keyb *ckdev) struct input_dev *idev; const char *phys; int err; + struct property *prop; + const __be32 *p; + u16 *physmap; + u32 key_pos; + int row, col; err = matrix_keypad_parse_properties(dev, &ckdev->rows, &ckdev->cols); if (err) @@ -578,6 +591,21 @@ static int cros_ec_keyb_register_matrix(struct cros_ec_keyb *ckdev) ckdev->idev = idev; cros_ec_keyb_compute_valid_keys(ckdev); + physmap = ckdev->function_row_physmap; + of_property_for_each_u32(dev->of_node, "function-row-physmap", + prop, p, key_pos) { + if (ckdev->num_function_row_keys == MAX_NUM_TOP_ROW_KEYS) { + dev_warn(dev, "Only support up to %d top row keys\n", + MAX_NUM_TOP_ROW_KEYS); + break; + } + row = KEY_ROW(key_pos); + col = KEY_COL(key_pos); + *physmap = MATRIX_SCAN_CODE(row, col, ckdev->row_shift); + physmap++; + ckdev->num_function_row_keys++; + } + err = input_register_device(ckdev->idev); if (err) { dev_err(dev, "cannot register input device\n"); @@ -587,6 +615,51 @@ static int cros_ec_keyb_register_matrix(struct cros_ec_keyb *ckdev) return 0; } +static ssize_t function_row_physmap_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + ssize_t size = 0; + int i; + struct cros_ec_keyb *ckdev = dev_get_drvdata(dev); + u16 *physmap = ckdev->function_row_physmap; + + for (i = 0; i < ckdev->num_function_row_keys; i++) + size += scnprintf(buf + size, PAGE_SIZE - size, + "%s%02X", size ? " " : "", physmap[i]); + if (size) + size += scnprintf(buf + size, PAGE_SIZE - size, "\n"); + + return size; +} + +static DEVICE_ATTR_RO(function_row_physmap); + +static struct attribute *cros_ec_keyb_attrs[] = { + &dev_attr_function_row_physmap.attr, + NULL, +}; + +static umode_t cros_ec_keyb_attr_is_visible(struct kobject *kobj, + struct attribute *attr, + int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct cros_ec_keyb *ckdev = dev_get_drvdata(dev); + + if (attr == &dev_attr_function_row_physmap.attr && + !ckdev->num_function_row_keys) + return 0; + + return attr->mode; +} + +static const struct attribute_group cros_ec_keyb_attr_group = { + .is_visible = cros_ec_keyb_attr_is_visible, + .attrs = cros_ec_keyb_attrs, +}; + + static int cros_ec_keyb_probe(struct platform_device *pdev) { struct cros_ec_device *ec = dev_get_drvdata(pdev->dev.parent); @@ -617,6 +690,12 @@ static int cros_ec_keyb_probe(struct platform_device *pdev) return err; } + err = devm_device_add_group(dev, &cros_ec_keyb_attr_group); + if (err) { + dev_err(dev, "failed to create attributes. err=%d\n", err); + return err; + } + ckdev->notifier.notifier_call = cros_ec_keyb_work; err = blocking_notifier_chain_register(&ckdev->ec->event_notifier, &ckdev->notifier); diff --git a/drivers/input/keyboard/omap4-keypad.c b/drivers/input/keyboard/omap4-keypad.c index b17ac2a295b9..43375b38ee59 100644 --- a/drivers/input/keyboard/omap4-keypad.c +++ b/drivers/input/keyboard/omap4-keypad.c @@ -60,6 +60,8 @@ ((((dbms) * 1000) / ((1 << ((ptv) + 1)) * (1000000 / 32768))) - 1) #define OMAP4_VAL_DEBOUNCINGTIME_16MS \ OMAP4_KEYPAD_DEBOUNCINGTIME_MS(16, OMAP4_KEYPAD_PTV_DIV_128) +#define OMAP4_KEYPAD_AUTOIDLE_MS 50 /* Approximate measured time */ +#define OMAP4_KEYPAD_IDLE_CHECK_MS (OMAP4_KEYPAD_AUTOIDLE_MS / 2) enum { KBD_REVISION_OMAP4 = 0, @@ -71,6 +73,7 @@ struct omap4_keypad { void __iomem *base; unsigned int irq; + struct mutex lock; /* for key scan */ unsigned int rows; unsigned int cols; @@ -78,7 +81,7 @@ struct omap4_keypad { u32 irqreg_offset; unsigned int row_shift; bool no_autorepeat; - unsigned char key_state[8]; + u64 keys; unsigned short *keymap; }; @@ -107,6 +110,55 @@ static void kbd_write_irqreg(struct omap4_keypad *keypad_data, keypad_data->base + keypad_data->irqreg_offset + offset); } +static int omap4_keypad_report_keys(struct omap4_keypad *keypad_data, + u64 keys, bool down) +{ + struct input_dev *input_dev = keypad_data->input; + unsigned int col, row, code; + DECLARE_BITMAP(mask, 64); + unsigned long bit; + int events = 0; + + bitmap_from_u64(mask, keys); + + for_each_set_bit(bit, mask, keypad_data->rows * BITS_PER_BYTE) { + row = bit / BITS_PER_BYTE; + col = bit % BITS_PER_BYTE; + code = MATRIX_SCAN_CODE(row, col, keypad_data->row_shift); + + input_event(input_dev, EV_MSC, MSC_SCAN, code); + input_report_key(input_dev, keypad_data->keymap[code], down); + + events++; + } + + if (events) + input_sync(input_dev); + + return events; +} + +static void omap4_keypad_scan_keys(struct omap4_keypad *keypad_data, u64 keys) +{ + u64 changed; + + mutex_lock(&keypad_data->lock); + + changed = keys ^ keypad_data->keys; + + /* + * Report key up events separately and first. This matters in case we + * lost key-up interrupt and just now catching up. + */ + omap4_keypad_report_keys(keypad_data, changed & ~keys, false); + + /* Report key down events */ + omap4_keypad_report_keys(keypad_data, changed & keys, true); + + keypad_data->keys = keys; + + mutex_unlock(&keypad_data->lock); +} /* Interrupt handlers */ static irqreturn_t omap4_keypad_irq_handler(int irq, void *dev_id) @@ -122,48 +174,44 @@ static irqreturn_t omap4_keypad_irq_handler(int irq, void *dev_id) static irqreturn_t omap4_keypad_irq_thread_fn(int irq, void *dev_id) { struct omap4_keypad *keypad_data = dev_id; - struct input_dev *input_dev = keypad_data->input; - unsigned char key_state[ARRAY_SIZE(keypad_data->key_state)]; - unsigned int col, row, code, changed; - u32 *new_state = (u32 *) key_state; - - *new_state = kbd_readl(keypad_data, OMAP4_KBD_FULLCODE31_0); - *(new_state + 1) = kbd_readl(keypad_data, OMAP4_KBD_FULLCODE63_32); - - for (row = 0; row < keypad_data->rows; row++) { - changed = key_state[row] ^ keypad_data->key_state[row]; - if (!changed) - continue; - - for (col = 0; col < keypad_data->cols; col++) { - if (changed & (1 << col)) { - code = MATRIX_SCAN_CODE(row, col, - keypad_data->row_shift); - input_event(input_dev, EV_MSC, MSC_SCAN, code); - input_report_key(input_dev, - keypad_data->keymap[code], - key_state[row] & (1 << col)); - } - } + struct device *dev = keypad_data->input->dev.parent; + u32 low, high; + int error; + u64 keys; + + error = pm_runtime_get_sync(dev); + if (error < 0) { + pm_runtime_put_noidle(dev); + return IRQ_NONE; } - input_sync(input_dev); + low = kbd_readl(keypad_data, OMAP4_KBD_FULLCODE31_0); + high = kbd_readl(keypad_data, OMAP4_KBD_FULLCODE63_32); + keys = low | (u64)high << 32; - memcpy(keypad_data->key_state, key_state, - sizeof(keypad_data->key_state)); + omap4_keypad_scan_keys(keypad_data, keys); /* clear pending interrupts */ kbd_write_irqreg(keypad_data, OMAP4_KBD_IRQSTATUS, kbd_read_irqreg(keypad_data, OMAP4_KBD_IRQSTATUS)); + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + return IRQ_HANDLED; } static int omap4_keypad_open(struct input_dev *input) { struct omap4_keypad *keypad_data = input_get_drvdata(input); + struct device *dev = input->dev.parent; + int error; - pm_runtime_get_sync(input->dev.parent); + error = pm_runtime_get_sync(dev); + if (error < 0) { + pm_runtime_put_noidle(dev); + return error; + } disable_irq(keypad_data->irq); @@ -176,13 +224,15 @@ static int omap4_keypad_open(struct input_dev *input) kbd_write_irqreg(keypad_data, OMAP4_KBD_IRQSTATUS, kbd_read_irqreg(keypad_data, OMAP4_KBD_IRQSTATUS)); kbd_write_irqreg(keypad_data, OMAP4_KBD_IRQENABLE, - OMAP4_DEF_IRQENABLE_EVENTEN | - OMAP4_DEF_IRQENABLE_LONGKEY); + OMAP4_DEF_IRQENABLE_EVENTEN); kbd_writel(keypad_data, OMAP4_KBD_WAKEUPENABLE, - OMAP4_DEF_WUP_EVENT_ENA | OMAP4_DEF_WUP_LONG_KEY_ENA); + OMAP4_DEF_WUP_EVENT_ENA); enable_irq(keypad_data->irq); + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + return 0; } @@ -200,14 +250,20 @@ static void omap4_keypad_stop(struct omap4_keypad *keypad_data) static void omap4_keypad_close(struct input_dev *input) { - struct omap4_keypad *keypad_data; + struct omap4_keypad *keypad_data = input_get_drvdata(input); + struct device *dev = input->dev.parent; + int error; + + error = pm_runtime_get_sync(dev); + if (error < 0) + pm_runtime_put_noidle(dev); - keypad_data = input_get_drvdata(input); disable_irq(keypad_data->irq); omap4_keypad_stop(keypad_data); enable_irq(keypad_data->irq); - pm_runtime_put_sync(input->dev.parent); + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); } static int omap4_keypad_parse_dt(struct device *dev, @@ -252,8 +308,41 @@ static int omap4_keypad_check_revision(struct device *dev, return 0; } +/* + * Errata ID i689 "1.32 Keyboard Key Up Event Can Be Missed". + * Interrupt may not happen for key-up events. We must clear stuck + * key-up events after the keyboard hardware has auto-idled. + */ +static int __maybe_unused omap4_keypad_runtime_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct omap4_keypad *keypad_data = platform_get_drvdata(pdev); + u32 active; + + active = kbd_readl(keypad_data, OMAP4_KBD_STATEMACHINE); + if (active) { + pm_runtime_mark_last_busy(dev); + return -EBUSY; + } + + omap4_keypad_scan_keys(keypad_data, 0); + + return 0; +} + +static const struct dev_pm_ops omap4_keypad_pm_ops = { + SET_RUNTIME_PM_OPS(omap4_keypad_runtime_suspend, NULL, NULL) +}; + +static void omap4_disable_pm(void *d) +{ + pm_runtime_dont_use_autosuspend(d); + pm_runtime_disable(d); +} + static int omap4_keypad_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; struct omap4_keypad *keypad_data; struct input_dev *input_dev; struct resource *res; @@ -271,63 +360,62 @@ static int omap4_keypad_probe(struct platform_device *pdev) if (irq < 0) return irq; - keypad_data = kzalloc(sizeof(struct omap4_keypad), GFP_KERNEL); + keypad_data = devm_kzalloc(dev, sizeof(*keypad_data), GFP_KERNEL); if (!keypad_data) { - dev_err(&pdev->dev, "keypad_data memory allocation failed\n"); + dev_err(dev, "keypad_data memory allocation failed\n"); return -ENOMEM; } keypad_data->irq = irq; + mutex_init(&keypad_data->lock); + platform_set_drvdata(pdev, keypad_data); - error = omap4_keypad_parse_dt(&pdev->dev, keypad_data); + error = omap4_keypad_parse_dt(dev, keypad_data); if (error) - goto err_free_keypad; + return error; - res = request_mem_region(res->start, resource_size(res), pdev->name); - if (!res) { - dev_err(&pdev->dev, "can't request mem region\n"); - error = -EBUSY; - goto err_free_keypad; - } + keypad_data->base = devm_ioremap_resource(dev, res); + if (IS_ERR(keypad_data->base)) + return PTR_ERR(keypad_data->base); - keypad_data->base = ioremap(res->start, resource_size(res)); - if (!keypad_data->base) { - dev_err(&pdev->dev, "can't ioremap mem resource\n"); - error = -ENOMEM; - goto err_release_mem; - } + pm_runtime_use_autosuspend(dev); + pm_runtime_set_autosuspend_delay(dev, OMAP4_KEYPAD_IDLE_CHECK_MS); + pm_runtime_enable(dev); - pm_runtime_enable(&pdev->dev); + error = devm_add_action_or_reset(dev, omap4_disable_pm, dev); + if (error) { + dev_err(dev, "unable to register cleanup action\n"); + return error; + } /* * Enable clocks for the keypad module so that we can read * revision register. */ - error = pm_runtime_get_sync(&pdev->dev); + error = pm_runtime_get_sync(dev); if (error) { - dev_err(&pdev->dev, "pm_runtime_get_sync() failed\n"); - pm_runtime_put_noidle(&pdev->dev); - } else { - error = omap4_keypad_check_revision(&pdev->dev, - keypad_data); - if (!error) { - /* Ensure device does not raise interrupts */ - omap4_keypad_stop(keypad_data); - } - pm_runtime_put_sync(&pdev->dev); + dev_err(dev, "pm_runtime_get_sync() failed\n"); + pm_runtime_put_noidle(dev); + return error; + } + + error = omap4_keypad_check_revision(dev, keypad_data); + if (!error) { + /* Ensure device does not raise interrupts */ + omap4_keypad_stop(keypad_data); } + + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); if (error) - goto err_pm_disable; + return error; /* input device allocation */ - keypad_data->input = input_dev = input_allocate_device(); - if (!input_dev) { - error = -ENOMEM; - goto err_pm_disable; - } + keypad_data->input = input_dev = devm_input_allocate_device(dev); + if (!input_dev) + return -ENOMEM; input_dev->name = pdev->name; - input_dev->dev.parent = &pdev->dev; input_dev->id.bustype = BUS_HOST; input_dev->id.vendor = 0x0001; input_dev->id.product = 0x0001; @@ -344,84 +432,51 @@ static int omap4_keypad_probe(struct platform_device *pdev) keypad_data->row_shift = get_count_order(keypad_data->cols); max_keys = keypad_data->rows << keypad_data->row_shift; - keypad_data->keymap = kcalloc(max_keys, - sizeof(keypad_data->keymap[0]), - GFP_KERNEL); + keypad_data->keymap = devm_kcalloc(dev, + max_keys, + sizeof(keypad_data->keymap[0]), + GFP_KERNEL); if (!keypad_data->keymap) { - dev_err(&pdev->dev, "Not enough memory for keymap\n"); - error = -ENOMEM; - goto err_free_input; + dev_err(dev, "Not enough memory for keymap\n"); + return -ENOMEM; } error = matrix_keypad_build_keymap(NULL, NULL, keypad_data->rows, keypad_data->cols, keypad_data->keymap, input_dev); if (error) { - dev_err(&pdev->dev, "failed to build keymap\n"); - goto err_free_keymap; + dev_err(dev, "failed to build keymap\n"); + return error; } - error = request_threaded_irq(keypad_data->irq, omap4_keypad_irq_handler, - omap4_keypad_irq_thread_fn, IRQF_ONESHOT, - "omap4-keypad", keypad_data); + error = devm_request_threaded_irq(dev, keypad_data->irq, + omap4_keypad_irq_handler, + omap4_keypad_irq_thread_fn, + IRQF_ONESHOT, + "omap4-keypad", keypad_data); if (error) { - dev_err(&pdev->dev, "failed to register interrupt\n"); - goto err_free_keymap; + dev_err(dev, "failed to register interrupt\n"); + return error; } error = input_register_device(keypad_data->input); - if (error < 0) { - dev_err(&pdev->dev, "failed to register input device\n"); - goto err_free_irq; + if (error) { + dev_err(dev, "failed to register input device\n"); + return error; } - device_init_wakeup(&pdev->dev, true); - error = dev_pm_set_wake_irq(&pdev->dev, keypad_data->irq); + device_init_wakeup(dev, true); + error = dev_pm_set_wake_irq(dev, keypad_data->irq); if (error) - dev_warn(&pdev->dev, - "failed to set up wakeup irq: %d\n", error); - - platform_set_drvdata(pdev, keypad_data); + dev_warn(dev, "failed to set up wakeup irq: %d\n", error); return 0; - -err_free_irq: - free_irq(keypad_data->irq, keypad_data); -err_free_keymap: - kfree(keypad_data->keymap); -err_free_input: - input_free_device(input_dev); -err_pm_disable: - pm_runtime_disable(&pdev->dev); - iounmap(keypad_data->base); -err_release_mem: - release_mem_region(res->start, resource_size(res)); -err_free_keypad: - kfree(keypad_data); - return error; } static int omap4_keypad_remove(struct platform_device *pdev) { - struct omap4_keypad *keypad_data = platform_get_drvdata(pdev); - struct resource *res; - dev_pm_clear_wake_irq(&pdev->dev); - free_irq(keypad_data->irq, keypad_data); - - pm_runtime_disable(&pdev->dev); - - input_unregister_device(keypad_data->input); - - iounmap(keypad_data->base); - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - release_mem_region(res->start, resource_size(res)); - - kfree(keypad_data->keymap); - kfree(keypad_data); - return 0; } @@ -437,6 +492,7 @@ static struct platform_driver omap4_keypad_driver = { .driver = { .name = "omap4-keypad", .of_match_table = omap_keypad_dt_match, + .pm = &omap4_keypad_pm_ops, }, }; module_platform_driver(omap4_keypad_driver); diff --git a/drivers/input/misc/da7280.c b/drivers/input/misc/da7280.c index 37568b00873d..b08610d6e575 100644 --- a/drivers/input/misc/da7280.c +++ b/drivers/input/misc/da7280.c @@ -863,6 +863,7 @@ static void da7280_parse_properties(struct device *dev, gpi_str3[7] = '0' + i; haptics->gpi_ctl[i].polarity = 0; error = device_property_read_string(dev, gpi_str3, &str); + if (!error) haptics->gpi_ctl[i].polarity = da7280_haptic_of_gpi_pol_str(dev, str); } @@ -1299,11 +1300,13 @@ static int __maybe_unused da7280_resume(struct device *dev) return retval; } +#ifdef CONFIG_OF static const struct of_device_id da7280_of_match[] = { { .compatible = "dlg,da7280", }, { } }; MODULE_DEVICE_TABLE(of, da7280_of_match); +#endif static const struct i2c_device_id da7280_i2c_id[] = { { "da7280", }, diff --git a/drivers/input/mouse/alps.c b/drivers/input/mouse/alps.c index b067bfd2699c..4a6b33bbe7ea 100644 --- a/drivers/input/mouse/alps.c +++ b/drivers/input/mouse/alps.c @@ -986,7 +986,7 @@ static void alps_get_finger_coordinate_v7(struct input_mt_pos *mt, case V7_PACKET_ID_TWO: mt[1].x &= ~0x000F; mt[1].y |= 0x000F; - /* Detect false-postive touches where x & y report max value */ + /* Detect false-positive touches where x & y report max value */ if (mt[1].y == 0x7ff && mt[1].x == 0xff0) { mt[1].x = 0; /* y gets set to 0 at the end of this function */ diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c index 8fb7b4385ded..ffad142801b3 100644 --- a/drivers/input/mouse/synaptics.c +++ b/drivers/input/mouse/synaptics.c @@ -1106,8 +1106,11 @@ static void synaptics_process_packet(struct psmouse *psmouse) num_fingers = hw.w + 2; break; case 2: - if (SYN_MODEL_PEN(info->model_id)) - ; /* Nothing, treat a pen as a single finger */ + /* + * SYN_MODEL_PEN(info->model_id): even if + * the device supports pen, we treat it as + * a single finger. + */ break; case 4 ... 15: if (SYN_CAP_PALMDETECT(info->capabilities)) diff --git a/drivers/input/serio/Kconfig b/drivers/input/serio/Kconfig index 0754744b9ce5..f39b7b3f7942 100644 --- a/drivers/input/serio/Kconfig +++ b/drivers/input/serio/Kconfig @@ -255,7 +255,7 @@ config SERIO_ARC_PS2 config SERIO_APBPS2 tristate "GRLIB APBPS2 PS/2 keyboard/mouse controller" - depends on OF + depends on OF && HAS_IOMEM help Say Y here if you want support for GRLIB APBPS2 peripherals used to connect to PS/2 keyboard and/or mouse. diff --git a/drivers/input/serio/i8042-x86ia64io.h b/drivers/input/serio/i8042-x86ia64io.h index c74b020796a9..9119e12a5778 100644 --- a/drivers/input/serio/i8042-x86ia64io.h +++ b/drivers/input/serio/i8042-x86ia64io.h @@ -588,6 +588,10 @@ static const struct dmi_system_id i8042_dmi_noselftest_table[] = { DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), DMI_MATCH(DMI_CHASSIS_TYPE, "10"), /* Notebook */ }, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_CHASSIS_TYPE, "31"), /* Convertible Notebook */ + }, }, { } }; diff --git a/drivers/input/tablet/aiptek.c b/drivers/input/tablet/aiptek.c index e08b0ef078e8..fcb1b646436a 100644 --- a/drivers/input/tablet/aiptek.c +++ b/drivers/input/tablet/aiptek.c @@ -1036,9 +1036,9 @@ static ssize_t show_tabletSize(struct device *dev, struct device_attribute *attr { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%dx%d\n", - input_abs_get_max(aiptek->inputdev, ABS_X) + 1, - input_abs_get_max(aiptek->inputdev, ABS_Y) + 1); + return sysfs_emit(buf, "%dx%d\n", + input_abs_get_max(aiptek->inputdev, ABS_X) + 1, + input_abs_get_max(aiptek->inputdev, ABS_Y) + 1); } /* These structs define the sysfs files, param #1 is the name of the @@ -1064,9 +1064,8 @@ static ssize_t show_tabletPointerMode(struct device *dev, struct device_attribut { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%s\n", - map_val_to_str(pointer_mode_map, - aiptek->curSetting.pointerMode)); + return sysfs_emit(buf, "%s\n", map_val_to_str(pointer_mode_map, + aiptek->curSetting.pointerMode)); } static ssize_t @@ -1101,9 +1100,8 @@ static ssize_t show_tabletCoordinateMode(struct device *dev, struct device_attri { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%s\n", - map_val_to_str(coordinate_mode_map, - aiptek->curSetting.coordinateMode)); + return sysfs_emit(buf, "%s\n", map_val_to_str(coordinate_mode_map, + aiptek->curSetting.coordinateMode)); } static ssize_t @@ -1143,9 +1141,8 @@ static ssize_t show_tabletToolMode(struct device *dev, struct device_attribute * { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%s\n", - map_val_to_str(tool_mode_map, - aiptek->curSetting.toolMode)); + return sysfs_emit(buf, "%s\n", map_val_to_str(tool_mode_map, + aiptek->curSetting.toolMode)); } static ssize_t @@ -1174,10 +1171,9 @@ static ssize_t show_tabletXtilt(struct device *dev, struct device_attribute *att struct aiptek *aiptek = dev_get_drvdata(dev); if (aiptek->curSetting.xTilt == AIPTEK_TILT_DISABLE) { - return snprintf(buf, PAGE_SIZE, "disable\n"); + return sysfs_emit(buf, "disable\n"); } else { - return snprintf(buf, PAGE_SIZE, "%d\n", - aiptek->curSetting.xTilt); + return sysfs_emit(buf, "%d\n", aiptek->curSetting.xTilt); } } @@ -1216,10 +1212,9 @@ static ssize_t show_tabletYtilt(struct device *dev, struct device_attribute *att struct aiptek *aiptek = dev_get_drvdata(dev); if (aiptek->curSetting.yTilt == AIPTEK_TILT_DISABLE) { - return snprintf(buf, PAGE_SIZE, "disable\n"); + return sysfs_emit(buf, "disable\n"); } else { - return snprintf(buf, PAGE_SIZE, "%d\n", - aiptek->curSetting.yTilt); + return sysfs_emit(buf, "%d\n", aiptek->curSetting.yTilt); } } @@ -1257,7 +1252,7 @@ static ssize_t show_tabletJitterDelay(struct device *dev, struct device_attribut { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%d\n", aiptek->curSetting.jitterDelay); + return sysfs_emit(buf, "%d\n", aiptek->curSetting.jitterDelay); } static ssize_t @@ -1286,8 +1281,7 @@ static ssize_t show_tabletProgrammableDelay(struct device *dev, struct device_at { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%d\n", - aiptek->curSetting.programmableDelay); + return sysfs_emit(buf, "%d\n", aiptek->curSetting.programmableDelay); } static ssize_t @@ -1316,7 +1310,7 @@ static ssize_t show_tabletEventsReceived(struct device *dev, struct device_attri { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%ld\n", aiptek->eventCount); + return sysfs_emit(buf, "%ld\n", aiptek->eventCount); } static DEVICE_ATTR(event_count, S_IRUGO, show_tabletEventsReceived, NULL); @@ -1355,7 +1349,7 @@ static ssize_t show_tabletDiagnosticMessage(struct device *dev, struct device_at default: return 0; } - return snprintf(buf, PAGE_SIZE, retMsg); + return sysfs_emit(buf, retMsg); } static DEVICE_ATTR(diagnostic, S_IRUGO, show_tabletDiagnosticMessage, NULL); @@ -1375,9 +1369,8 @@ static ssize_t show_tabletStylusUpper(struct device *dev, struct device_attribut { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%s\n", - map_val_to_str(stylus_button_map, - aiptek->curSetting.stylusButtonUpper)); + return sysfs_emit(buf, "%s\n", map_val_to_str(stylus_button_map, + aiptek->curSetting.stylusButtonUpper)); } static ssize_t @@ -1406,9 +1399,8 @@ static ssize_t show_tabletStylusLower(struct device *dev, struct device_attribut { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%s\n", - map_val_to_str(stylus_button_map, - aiptek->curSetting.stylusButtonLower)); + return sysfs_emit(buf, "%s\n", map_val_to_str(stylus_button_map, + aiptek->curSetting.stylusButtonLower)); } static ssize_t @@ -1444,9 +1436,8 @@ static ssize_t show_tabletMouseLeft(struct device *dev, struct device_attribute { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%s\n", - map_val_to_str(mouse_button_map, - aiptek->curSetting.mouseButtonLeft)); + return sysfs_emit(buf, "%s\n", map_val_to_str(mouse_button_map, + aiptek->curSetting.mouseButtonLeft)); } static ssize_t @@ -1474,9 +1465,8 @@ static ssize_t show_tabletMouseMiddle(struct device *dev, struct device_attribut { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%s\n", - map_val_to_str(mouse_button_map, - aiptek->curSetting.mouseButtonMiddle)); + return sysfs_emit(buf, "%s\n", map_val_to_str(mouse_button_map, + aiptek->curSetting.mouseButtonMiddle)); } static ssize_t @@ -1504,9 +1494,8 @@ static ssize_t show_tabletMouseRight(struct device *dev, struct device_attribute { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%s\n", - map_val_to_str(mouse_button_map, - aiptek->curSetting.mouseButtonRight)); + return sysfs_emit(buf, "%s\n", map_val_to_str(mouse_button_map, + aiptek->curSetting.mouseButtonRight)); } static ssize_t @@ -1535,10 +1524,9 @@ static ssize_t show_tabletWheel(struct device *dev, struct device_attribute *att struct aiptek *aiptek = dev_get_drvdata(dev); if (aiptek->curSetting.wheel == AIPTEK_WHEEL_DISABLE) { - return snprintf(buf, PAGE_SIZE, "disable\n"); + return sysfs_emit(buf, "disable\n"); } else { - return snprintf(buf, PAGE_SIZE, "%d\n", - aiptek->curSetting.wheel); + return sysfs_emit(buf, "%d\n", aiptek->curSetting.wheel); } } @@ -1568,8 +1556,7 @@ static ssize_t show_tabletExecute(struct device *dev, struct device_attribute *a /* There is nothing useful to display, so a one-line manual * is in order... */ - return snprintf(buf, PAGE_SIZE, - "Write anything to this file to program your tablet.\n"); + return sysfs_emit(buf, "Write anything to this file to program your tablet.\n"); } static ssize_t @@ -1600,7 +1587,7 @@ static ssize_t show_tabletODMCode(struct device *dev, struct device_attribute *a { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "0x%04x\n", aiptek->features.odmCode); + return sysfs_emit(buf, "0x%04x\n", aiptek->features.odmCode); } static DEVICE_ATTR(odm_code, S_IRUGO, show_tabletODMCode, NULL); @@ -1613,7 +1600,7 @@ static ssize_t show_tabletModelCode(struct device *dev, struct device_attribute { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "0x%04x\n", aiptek->features.modelCode); + return sysfs_emit(buf, "0x%04x\n", aiptek->features.modelCode); } static DEVICE_ATTR(model_code, S_IRUGO, show_tabletModelCode, NULL); @@ -1626,8 +1613,7 @@ static ssize_t show_firmwareCode(struct device *dev, struct device_attribute *at { struct aiptek *aiptek = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%04x\n", - aiptek->features.firmwareCode); + return sysfs_emit(buf, "%04x\n", aiptek->features.firmwareCode); } static DEVICE_ATTR(firmware_code, S_IRUGO, show_firmwareCode, NULL); diff --git a/drivers/input/touchscreen/Kconfig b/drivers/input/touchscreen/Kconfig index cc18f54ea887..529614d364fe 100644 --- a/drivers/input/touchscreen/Kconfig +++ b/drivers/input/touchscreen/Kconfig @@ -608,7 +608,7 @@ config TOUCHSCREEN_MTOUCH config TOUCHSCREEN_IMX6UL_TSC tristate "Freescale i.MX6UL touchscreen controller" - depends on (OF && GPIOLIB) || COMPILE_TEST + depends on ((OF && GPIOLIB) || COMPILE_TEST) && HAS_IOMEM help Say Y here if you have a Freescale i.MX6UL, and want to use the internal touchscreen controller. diff --git a/drivers/input/touchscreen/ads7846.c b/drivers/input/touchscreen/ads7846.c index a703870ca7bd..f113a27aeb1e 100644 --- a/drivers/input/touchscreen/ads7846.c +++ b/drivers/input/touchscreen/ads7846.c @@ -64,24 +64,13 @@ struct ads7846_buf { u8 cmd; - /* - * This union is a temporary hack. The driver does an in-place - * endianness conversion. This will be cleaned up in the next - * patch. - */ - union { - __be16 data_be16; - u16 data; - }; + __be16 data; } __packed; - -struct ts_event { - bool ignore; - struct ads7846_buf x; - struct ads7846_buf y; - struct ads7846_buf z1; - struct ads7846_buf z2; +struct ads7846_buf_layout { + unsigned int offset; + unsigned int count; + unsigned int skip; }; /* @@ -90,12 +79,18 @@ struct ts_event { * systems where main memory is not DMA-coherent (most non-x86 boards). */ struct ads7846_packet { - struct ts_event tc; - struct ads7846_buf read_x_cmd; - struct ads7846_buf read_y_cmd; - struct ads7846_buf read_z1_cmd; - struct ads7846_buf read_z2_cmd; + unsigned int count; + unsigned int count_skip; + unsigned int cmds; + unsigned int last_cmd_idx; + struct ads7846_buf_layout l[5]; + struct ads7846_buf *rx; + struct ads7846_buf *tx; + struct ads7846_buf pwrdown_cmd; + + bool ignore; + u16 x, y, z1, z2; }; struct ads7846 { @@ -194,7 +189,6 @@ struct ads7846 { #define READ_Y(vref) (READ_12BIT_DFR(y, 1, vref)) #define READ_Z1(vref) (READ_12BIT_DFR(z1, 1, vref)) #define READ_Z2(vref) (READ_12BIT_DFR(z2, 1, vref)) - #define READ_X(vref) (READ_12BIT_DFR(x, 1, vref)) #define PWRDOWN (READ_12BIT_DFR(y, 0, 0)) /* LAST */ @@ -207,6 +201,21 @@ struct ads7846 { #define REF_ON (READ_12BIT_DFR(x, 1, 1)) #define REF_OFF (READ_12BIT_DFR(y, 0, 0)) +/* Order commands in the most optimal way to reduce Vref switching and + * settling time: + * Measure: X; Vref: X+, X-; IN: Y+ + * Measure: Y; Vref: Y+, Y-; IN: X+ + * Measure: Z1; Vref: Y+, X-; IN: X+ + * Measure: Z2; Vref: Y+, X-; IN: Y- + */ +enum ads7846_cmds { + ADS7846_X, + ADS7846_Y, + ADS7846_Z1, + ADS7846_Z2, + ADS7846_PWDOWN, +}; + static int get_pendown_state(struct ads7846 *ts) { if (ts->get_pendown_state) @@ -689,26 +698,109 @@ static int ads7846_no_filter(void *ads, int data_idx, int *val) return ADS7846_FILTER_OK; } -static int ads7846_get_value(struct ads7846 *ts, struct spi_message *m) +static int ads7846_get_value(struct ads7846_buf *buf) { int value; - struct spi_transfer *t = - list_entry(m->transfers.prev, struct spi_transfer, transfer_list); - struct ads7846_buf *buf = t->rx_buf; - value = be16_to_cpup(&buf->data_be16); + value = be16_to_cpup(&buf->data); /* enforce ADC output is 12 bits width */ return (value >> 3) & 0xfff; } -static void ads7846_update_value(struct spi_message *m, int val) +static void ads7846_set_cmd_val(struct ads7846 *ts, enum ads7846_cmds cmd_idx, + u16 val) +{ + struct ads7846_packet *packet = ts->packet; + + switch (cmd_idx) { + case ADS7846_Y: + packet->y = val; + break; + case ADS7846_X: + packet->x = val; + break; + case ADS7846_Z1: + packet->z1 = val; + break; + case ADS7846_Z2: + packet->z2 = val; + break; + default: + WARN_ON_ONCE(1); + } +} + +static u8 ads7846_get_cmd(enum ads7846_cmds cmd_idx, int vref) +{ + switch (cmd_idx) { + case ADS7846_Y: + return READ_Y(vref); + case ADS7846_X: + return READ_X(vref); + + /* 7846 specific commands */ + case ADS7846_Z1: + return READ_Z1(vref); + case ADS7846_Z2: + return READ_Z2(vref); + case ADS7846_PWDOWN: + return PWRDOWN; + default: + WARN_ON_ONCE(1); + } + + return 0; +} + +static bool ads7846_cmd_need_settle(enum ads7846_cmds cmd_idx) +{ + switch (cmd_idx) { + case ADS7846_X: + case ADS7846_Y: + case ADS7846_Z1: + case ADS7846_Z2: + return true; + case ADS7846_PWDOWN: + return false; + default: + WARN_ON_ONCE(1); + } + + return false; +} + +static int ads7846_filter(struct ads7846 *ts) { - struct spi_transfer *t = - list_entry(m->transfers.prev, struct spi_transfer, transfer_list); - struct ads7846_buf *buf = t->rx_buf; + struct ads7846_packet *packet = ts->packet; + int action; + int val; + unsigned int cmd_idx, b; - buf->data = val; + packet->ignore = false; + for (cmd_idx = packet->last_cmd_idx; cmd_idx < packet->cmds - 1; cmd_idx++) { + struct ads7846_buf_layout *l = &packet->l[cmd_idx]; + + packet->last_cmd_idx = cmd_idx; + + for (b = l->skip; b < l->count; b++) { + val = ads7846_get_value(&packet->rx[l->offset + b]); + + action = ts->filter(ts->filter_data, cmd_idx, &val); + if (action == ADS7846_FILTER_REPEAT) { + if (b == l->count - 1) + return -EAGAIN; + } else if (action == ADS7846_FILTER_OK) { + ads7846_set_cmd_val(ts, cmd_idx, val); + break; + } else { + packet->ignore = true; + return 0; + } + } + } + + return 0; } static void ads7846_read_state(struct ads7846 *ts) @@ -716,52 +808,26 @@ static void ads7846_read_state(struct ads7846 *ts) struct ads7846_packet *packet = ts->packet; struct spi_message *m; int msg_idx = 0; - int val; - int action; int error; - while (msg_idx < ts->msg_count) { + packet->last_cmd_idx = 0; + while (true) { ts->wait_for_sync(); m = &ts->msg[msg_idx]; error = spi_sync(ts->spi, m); if (error) { dev_err(&ts->spi->dev, "spi_sync --> %d\n", error); - packet->tc.ignore = true; + packet->ignore = true; return; } - /* - * Last message is power down request, no need to convert - * or filter the value. - */ - if (msg_idx < ts->msg_count - 1) { - - val = ads7846_get_value(ts, m); - - action = ts->filter(ts->filter_data, msg_idx, &val); - switch (action) { - case ADS7846_FILTER_REPEAT: - continue; - - case ADS7846_FILTER_IGNORE: - packet->tc.ignore = true; - msg_idx = ts->msg_count - 1; - continue; - - case ADS7846_FILTER_OK: - ads7846_update_value(m, val); - packet->tc.ignore = false; - msg_idx++; - break; + error = ads7846_filter(ts); + if (error) + continue; - default: - BUG(); - } - } else { - msg_idx++; - } + return; } } @@ -771,19 +837,14 @@ static void ads7846_report_state(struct ads7846 *ts) unsigned int Rt; u16 x, y, z1, z2; - /* - * ads7846_get_value() does in-place conversion (including byte swap) - * from on-the-wire format as part of debouncing to get stable - * readings. - */ - x = packet->tc.x.data; - y = packet->tc.y.data; + x = packet->x; + y = packet->y; if (ts->model == 7845) { z1 = 0; z2 = 0; } else { - z1 = packet->tc.z1.data; - z2 = packet->tc.z2.data; + z1 = packet->z1; + z2 = packet->z2; } /* range filtering */ @@ -816,9 +877,9 @@ static void ads7846_report_state(struct ads7846 *ts) * the maximum. Don't report it to user space, repeat at least * once more the measurement */ - if (packet->tc.ignore || Rt > ts->pressure_max) { + if (packet->ignore || Rt > ts->pressure_max) { dev_vdbg(&ts->spi->dev, "ignored %d pressure %d\n", - packet->tc.ignore, Rt); + packet->ignore, Rt); return; } @@ -979,13 +1040,59 @@ static int ads7846_setup_pendown(struct spi_device *spi, * Set up the transfers to read touchscreen state; this assumes we * use formula #2 for pressure, not #3. */ -static void ads7846_setup_spi_msg(struct ads7846 *ts, +static int ads7846_setup_spi_msg(struct ads7846 *ts, const struct ads7846_platform_data *pdata) { struct spi_message *m = &ts->msg[0]; struct spi_transfer *x = ts->xfer; struct ads7846_packet *packet = ts->packet; int vref = pdata->keep_vref_on; + unsigned int count, offset = 0; + unsigned int cmd_idx, b; + unsigned long time; + size_t size = 0; + + /* time per bit */ + time = NSEC_PER_SEC / ts->spi->max_speed_hz; + + count = pdata->settle_delay_usecs * NSEC_PER_USEC / time; + packet->count_skip = DIV_ROUND_UP(count, 24); + + if (ts->debounce_max && ts->debounce_rep) + /* ads7846_debounce_filter() is making ts->debounce_rep + 2 + * reads. So we need to get all samples for normal case. */ + packet->count = ts->debounce_rep + 2; + else + packet->count = 1; + + if (ts->model == 7846) + packet->cmds = 5; /* x, y, z1, z2, pwdown */ + else + packet->cmds = 3; /* x, y, pwdown */ + + for (cmd_idx = 0; cmd_idx < packet->cmds; cmd_idx++) { + struct ads7846_buf_layout *l = &packet->l[cmd_idx]; + unsigned int max_count; + + if (ads7846_cmd_need_settle(cmd_idx)) + max_count = packet->count + packet->count_skip; + else + max_count = packet->count; + + l->offset = offset; + offset += max_count; + l->count = max_count; + l->skip = packet->count_skip; + size += sizeof(*packet->tx) * max_count; + } + + packet->tx = devm_kzalloc(&ts->spi->dev, size, GFP_KERNEL); + if (!packet->tx) + return -ENOMEM; + + packet->rx = devm_kzalloc(&ts->spi->dev, size, GFP_KERNEL); + if (!packet->rx) + return -ENOMEM; if (ts->model == 7873) { /* @@ -1001,117 +1108,20 @@ static void ads7846_setup_spi_msg(struct ads7846 *ts, spi_message_init(m); m->context = ts; - packet->read_y_cmd.cmd = READ_Y(vref); - x->tx_buf = &packet->read_y_cmd; - x->rx_buf = &packet->tc.y; - x->len = 3; - spi_message_add_tail(x, m); + for (cmd_idx = 0; cmd_idx < packet->cmds; cmd_idx++) { + struct ads7846_buf_layout *l = &packet->l[cmd_idx]; + u8 cmd = ads7846_get_cmd(cmd_idx, vref); - /* - * The first sample after switching drivers can be low quality; - * optionally discard it, using a second one after the signals - * have had enough time to stabilize. - */ - if (pdata->settle_delay_usecs) { - x->delay.value = pdata->settle_delay_usecs; - x->delay.unit = SPI_DELAY_UNIT_USECS; - x++; - - x->tx_buf = &packet->read_y_cmd; - x->rx_buf = &packet->tc.y; - x->len = 3; - spi_message_add_tail(x, m); + for (b = 0; b < l->count; b++) + packet->tx[l->offset + b].cmd = cmd; } - ts->msg_count++; - m++; - spi_message_init(m); - m->context = ts; - - /* turn y- off, x+ on, then leave in lowpower */ - x++; - packet->read_x_cmd.cmd = READ_X(vref); - x->tx_buf = &packet->read_x_cmd; - x->rx_buf = &packet->tc.x; - x->len = 3; + x->tx_buf = packet->tx; + x->rx_buf = packet->rx; + x->len = size; spi_message_add_tail(x, m); - /* ... maybe discard first sample ... */ - if (pdata->settle_delay_usecs) { - x->delay.value = pdata->settle_delay_usecs; - x->delay.unit = SPI_DELAY_UNIT_USECS; - - x++; - x->tx_buf = &packet->read_x_cmd; - x->rx_buf = &packet->tc.x; - x->len = 3; - spi_message_add_tail(x, m); - } - - /* turn y+ off, x- on; we'll use formula #2 */ - if (ts->model == 7846) { - ts->msg_count++; - m++; - spi_message_init(m); - m->context = ts; - - x++; - packet->read_z1_cmd.cmd = READ_Z1(vref); - x->tx_buf = &packet->read_z1_cmd; - x->rx_buf = &packet->tc.z1; - x->len = 3; - spi_message_add_tail(x, m); - - /* ... maybe discard first sample ... */ - if (pdata->settle_delay_usecs) { - x->delay.value = pdata->settle_delay_usecs; - x->delay.unit = SPI_DELAY_UNIT_USECS; - - x++; - x->tx_buf = &packet->read_z1_cmd; - x->rx_buf = &packet->tc.z1; - x->len = 3; - spi_message_add_tail(x, m); - } - - ts->msg_count++; - m++; - spi_message_init(m); - m->context = ts; - - x++; - packet->read_z2_cmd.cmd = READ_Z2(vref); - x->tx_buf = &packet->read_z2_cmd; - x->rx_buf = &packet->tc.z2; - x->len = 3; - spi_message_add_tail(x, m); - - /* ... maybe discard first sample ... */ - if (pdata->settle_delay_usecs) { - x->delay.value = pdata->settle_delay_usecs; - x->delay.unit = SPI_DELAY_UNIT_USECS; - - x++; - x->tx_buf = &packet->read_z2_cmd; - x->rx_buf = &packet->tc.z2; - x->len = 3; - spi_message_add_tail(x, m); - } - } - - /* power down */ - ts->msg_count++; - m++; - spi_message_init(m); - m->context = ts; - - x++; - packet->pwrdown_cmd.cmd = PWRDOWN; - x->tx_buf = &packet->pwrdown_cmd; - x->len = 3; - - CS_CHANGE(*x); - spi_message_add_tail(x, m); + return 0; } #ifdef CONFIG_OF diff --git a/drivers/input/touchscreen/elants_i2c.c b/drivers/input/touchscreen/elants_i2c.c index d51cb910fba1..4c2b579f6c8b 100644 --- a/drivers/input/touchscreen/elants_i2c.c +++ b/drivers/input/touchscreen/elants_i2c.c @@ -56,6 +56,7 @@ #define QUEUE_HEADER_SINGLE 0x62 #define QUEUE_HEADER_NORMAL 0X63 #define QUEUE_HEADER_WAIT 0x64 +#define QUEUE_HEADER_NORMAL2 0x66 /* Command header definition */ #define CMD_HEADER_WRITE 0x54 @@ -69,6 +70,7 @@ #define CMD_HEADER_REK 0x66 /* FW position data */ +#define PACKET_SIZE_OLD 40 #define PACKET_SIZE 55 #define MAX_CONTACT_NUM 10 #define FW_POS_HEADER 0 @@ -90,6 +92,8 @@ /* FW read command, 0x53 0x?? 0x0, 0x01 */ #define E_ELAN_INFO_FW_VER 0x00 #define E_ELAN_INFO_BC_VER 0x10 +#define E_ELAN_INFO_X_RES 0x60 +#define E_ELAN_INFO_Y_RES 0x63 #define E_ELAN_INFO_REK 0xD0 #define E_ELAN_INFO_TEST_VER 0xE0 #define E_ELAN_INFO_FW_ID 0xF0 @@ -112,6 +116,11 @@ #define ELAN_POWERON_DELAY_USEC 500 #define ELAN_RESET_DELAY_MSEC 20 +enum elants_chip_id { + EKTH3500, + EKTF3624, +}; + enum elants_state { ELAN_STATE_NORMAL, ELAN_WAIT_QUEUE_HEADER, @@ -143,9 +152,12 @@ struct elants_data { unsigned int y_res; unsigned int x_max; unsigned int y_max; + unsigned int phy_x; + unsigned int phy_y; struct touchscreen_properties prop; enum elants_state state; + enum elants_chip_id chip_id; enum elants_iap_mode iap_mode; /* Guards against concurrent access to the device via sysfs */ @@ -433,7 +445,51 @@ static int elants_i2c_query_bc_version(struct elants_data *ts) return 0; } -static int elants_i2c_query_ts_info(struct elants_data *ts) +static int elants_i2c_query_ts_info_ektf(struct elants_data *ts) +{ + struct i2c_client *client = ts->client; + int error; + u8 resp[4]; + u16 phy_x, phy_y; + const u8 get_xres_cmd[] = { + CMD_HEADER_READ, E_ELAN_INFO_X_RES, 0x00, 0x00 + }; + const u8 get_yres_cmd[] = { + CMD_HEADER_READ, E_ELAN_INFO_Y_RES, 0x00, 0x00 + }; + + /* Get X/Y size in mm */ + error = elants_i2c_execute_command(client, get_xres_cmd, + sizeof(get_xres_cmd), + resp, sizeof(resp), 1, + "get X size"); + if (error) + return error; + + phy_x = resp[2] | ((resp[3] & 0xF0) << 4); + + error = elants_i2c_execute_command(client, get_yres_cmd, + sizeof(get_yres_cmd), + resp, sizeof(resp), 1, + "get Y size"); + if (error) + return error; + + phy_y = resp[2] | ((resp[3] & 0xF0) << 4); + + dev_dbg(&client->dev, "phy_x=%d, phy_y=%d\n", phy_x, phy_y); + + ts->phy_x = phy_x; + ts->phy_y = phy_y; + + /* eKTF doesn't report max size, set it to default values */ + ts->x_max = 2240 - 1; + ts->y_max = 1408 - 1; + + return 0; +} + +static int elants_i2c_query_ts_info_ekth(struct elants_data *ts) { struct i2c_client *client = ts->client; int error; @@ -508,6 +564,8 @@ static int elants_i2c_query_ts_info(struct elants_data *ts) ts->x_res = DIV_ROUND_CLOSEST(ts->x_max, phy_x); ts->y_max = ELAN_TS_RESOLUTION(cols, osr); ts->y_res = DIV_ROUND_CLOSEST(ts->y_max, phy_y); + ts->phy_x = phy_x; + ts->phy_y = phy_y; } return 0; @@ -587,8 +645,19 @@ static int elants_i2c_initialize(struct elants_data *ts) error = elants_i2c_query_fw_version(ts); if (!error) error = elants_i2c_query_test_version(ts); - if (!error) - error = elants_i2c_query_ts_info(ts); + + switch (ts->chip_id) { + case EKTH3500: + if (!error) + error = elants_i2c_query_ts_info_ekth(ts); + break; + case EKTF3624: + if (!error) + error = elants_i2c_query_ts_info_ektf(ts); + break; + default: + BUG(); + } if (error) ts->iap_mode = ELAN_IAP_RECOVERY; @@ -853,7 +922,8 @@ out: * Event reporting. */ -static void elants_i2c_mt_event(struct elants_data *ts, u8 *buf) +static void elants_i2c_mt_event(struct elants_data *ts, u8 *buf, + size_t packet_size) { struct input_dev *input = ts->input; unsigned int n_fingers; @@ -880,8 +950,24 @@ static void elants_i2c_mt_event(struct elants_data *ts, u8 *buf) pos = &buf[FW_POS_XY + i * 3]; x = (((u16)pos[0] & 0xf0) << 4) | pos[1]; y = (((u16)pos[0] & 0x0f) << 8) | pos[2]; - p = buf[FW_POS_PRESSURE + i]; - w = buf[FW_POS_WIDTH + i]; + + /* + * eKTF3624 may have use "old" touch-report format, + * depending on a device and TS firmware version. + * For example, ASUS Transformer devices use the "old" + * format, while ASUS Nexus 7 uses the "new" formant. + */ + if (packet_size == PACKET_SIZE_OLD && + ts->chip_id == EKTF3624) { + w = buf[FW_POS_WIDTH + i / 2]; + w >>= 4 * (~i & 1); + w |= w << 4; + w |= !w; + p = w; + } else { + p = buf[FW_POS_PRESSURE + i]; + w = buf[FW_POS_WIDTH + i]; + } dev_dbg(&ts->client->dev, "i=%d x=%d y=%d p=%d w=%d\n", i, x, y, p, w); @@ -913,7 +999,8 @@ static u8 elants_i2c_calculate_checksum(u8 *buf) return checksum; } -static void elants_i2c_event(struct elants_data *ts, u8 *buf) +static void elants_i2c_event(struct elants_data *ts, u8 *buf, + size_t packet_size) { u8 checksum = elants_i2c_calculate_checksum(buf); @@ -927,7 +1014,7 @@ static void elants_i2c_event(struct elants_data *ts, u8 *buf) "%s: unknown packet type: %02x\n", __func__, buf[FW_POS_HEADER]); else - elants_i2c_mt_event(ts, buf); + elants_i2c_mt_event(ts, buf, packet_size); } static irqreturn_t elants_i2c_irq(int irq, void *_dev) @@ -970,7 +1057,6 @@ static irqreturn_t elants_i2c_irq(int irq, void *_dev) switch (ts->buf[FW_HDR_TYPE]) { case CMD_HEADER_HELLO: case CMD_HEADER_RESP: - case CMD_HEADER_REK: break; case QUEUE_HEADER_WAIT: @@ -985,9 +1071,24 @@ static irqreturn_t elants_i2c_irq(int irq, void *_dev) break; case QUEUE_HEADER_SINGLE: - elants_i2c_event(ts, &ts->buf[HEADER_SIZE]); + elants_i2c_event(ts, &ts->buf[HEADER_SIZE], + ts->buf[FW_HDR_LENGTH]); break; + case QUEUE_HEADER_NORMAL2: /* CMD_HEADER_REK */ + /* + * Depending on firmware version, eKTF3624 touchscreens + * may utilize one of these opcodes for the touch events: + * 0x63 (NORMAL) and 0x66 (NORMAL2). The 0x63 is used by + * older firmware version and differs from 0x66 such that + * touch pressure value needs to be adjusted. The 0x66 + * opcode of newer firmware is equal to 0x63 of eKTH3500. + */ + if (ts->chip_id != EKTF3624) + break; + + fallthrough; + case QUEUE_HEADER_NORMAL: report_count = ts->buf[FW_HDR_COUNT]; if (report_count == 0 || report_count > 3) { @@ -998,7 +1099,12 @@ static irqreturn_t elants_i2c_irq(int irq, void *_dev) } report_len = ts->buf[FW_HDR_LENGTH] / report_count; - if (report_len != PACKET_SIZE) { + + if (report_len == PACKET_SIZE_OLD && + ts->chip_id == EKTF3624) { + dev_dbg_once(&client->dev, + "using old report format\n"); + } else if (report_len != PACKET_SIZE) { dev_err(&client->dev, "mismatching report length: %*ph\n", HEADER_SIZE, ts->buf); @@ -1007,8 +1113,8 @@ static irqreturn_t elants_i2c_irq(int irq, void *_dev) for (i = 0; i < report_count; i++) { u8 *buf = ts->buf + HEADER_SIZE + - i * PACKET_SIZE; - elants_i2c_event(ts, buf); + i * report_len; + elants_i2c_event(ts, buf, report_len); } break; @@ -1250,6 +1356,7 @@ static int elants_i2c_probe(struct i2c_client *client, init_completion(&ts->cmd_done); ts->client = client; + ts->chip_id = (enum elants_chip_id)id->driver_data; i2c_set_clientdata(client, ts); ts->vcc33 = devm_regulator_get(&client->dev, "vcc33"); @@ -1331,13 +1438,20 @@ static int elants_i2c_probe(struct i2c_client *client, input_set_abs_params(ts->input, ABS_MT_PRESSURE, 0, 255, 0, 0); input_set_abs_params(ts->input, ABS_MT_TOOL_TYPE, 0, MT_TOOL_PALM, 0, 0); + + touchscreen_parse_properties(ts->input, true, &ts->prop); + + if (ts->chip_id == EKTF3624) { + /* calculate resolution from size */ + ts->x_res = DIV_ROUND_CLOSEST(ts->prop.max_x, ts->phy_x); + ts->y_res = DIV_ROUND_CLOSEST(ts->prop.max_y, ts->phy_y); + } + input_abs_set_res(ts->input, ABS_MT_POSITION_X, ts->x_res); input_abs_set_res(ts->input, ABS_MT_POSITION_Y, ts->y_res); if (ts->major_res > 0) input_abs_set_res(ts->input, ABS_MT_TOUCH_MAJOR, ts->major_res); - touchscreen_parse_properties(ts->input, true, &ts->prop); - error = input_mt_init_slots(ts->input, MAX_CONTACT_NUM, INPUT_MT_DIRECT | INPUT_MT_DROP_UNUSED); if (error) { @@ -1466,14 +1580,16 @@ static SIMPLE_DEV_PM_OPS(elants_i2c_pm_ops, elants_i2c_suspend, elants_i2c_resume); static const struct i2c_device_id elants_i2c_id[] = { - { DEVICE_NAME, 0 }, + { DEVICE_NAME, EKTH3500 }, + { "ekth3500", EKTH3500 }, + { "ektf3624", EKTF3624 }, { } }; MODULE_DEVICE_TABLE(i2c, elants_i2c_id); #ifdef CONFIG_ACPI static const struct acpi_device_id elants_acpi_id[] = { - { "ELAN0001", 0 }, + { "ELAN0001", EKTH3500 }, { } }; MODULE_DEVICE_TABLE(acpi, elants_acpi_id); @@ -1482,6 +1598,7 @@ MODULE_DEVICE_TABLE(acpi, elants_acpi_id); #ifdef CONFIG_OF static const struct of_device_id elants_of_match[] = { { .compatible = "elan,ekth3500" }, + { .compatible = "elan,ektf3624" }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, elants_of_match); diff --git a/drivers/input/touchscreen/elo.c b/drivers/input/touchscreen/elo.c index e0bacd34866a..96173232e53f 100644 --- a/drivers/input/touchscreen/elo.c +++ b/drivers/input/touchscreen/elo.c @@ -341,8 +341,10 @@ static int elo_connect(struct serio *serio, struct serio_driver *drv) switch (elo->id) { case 0: /* 10-byte protocol */ - if (elo_setup_10(elo)) + if (elo_setup_10(elo)) { + err = -EIO; goto fail3; + } break; diff --git a/drivers/input/touchscreen/iqs5xx.c b/drivers/input/touchscreen/iqs5xx.c index 4fd21bc3ce0f..54f30038dca4 100644 --- a/drivers/input/touchscreen/iqs5xx.c +++ b/drivers/input/touchscreen/iqs5xx.c @@ -2,8 +2,7 @@ /* * Azoteq IQS550/572/525 Trackpad/Touchscreen Controller * - * Copyright (C) 2018 - * Author: Jeff LaBundy <jeff@labundy.com> + * Copyright (C) 2018 Jeff LaBundy <jeff@labundy.com> * * These devices require firmware exported from a PC-based configuration tool * made available by the vendor. Firmware files may be pushed to the device's @@ -12,6 +11,7 @@ * Link to PC-based configuration tool and data sheet: http://www.azoteq.com/ */ +#include <linux/bits.h> #include <linux/delay.h> #include <linux/device.h> #include <linux/err.h> @@ -30,9 +30,9 @@ #define IQS5XX_FW_FILE_LEN 64 #define IQS5XX_NUM_RETRIES 10 -#define IQS5XX_NUM_POINTS 256 #define IQS5XX_NUM_CONTACTS 5 #define IQS5XX_WR_BYTES_MAX 2 +#define IQS5XX_XY_RES_MAX 0xFFFE #define IQS5XX_PROD_NUM_IQS550 40 #define IQS5XX_PROD_NUM_IQS572 58 @@ -41,28 +41,27 @@ #define IQS5XX_PROJ_NUM_B000 15 #define IQS5XX_MAJOR_VER_MIN 2 -#define IQS5XX_RESUME 0x00 -#define IQS5XX_SUSPEND 0x01 +#define IQS5XX_SHOW_RESET BIT(7) +#define IQS5XX_ACK_RESET BIT(7) -#define IQS5XX_SW_INPUT_EVENT 0x10 -#define IQS5XX_SETUP_COMPLETE 0x40 -#define IQS5XX_EVENT_MODE 0x01 -#define IQS5XX_TP_EVENT 0x04 +#define IQS5XX_SUSPEND BIT(0) +#define IQS5XX_RESUME 0 -#define IQS5XX_FLIP_X 0x01 -#define IQS5XX_FLIP_Y 0x02 -#define IQS5XX_SWITCH_XY_AXIS 0x04 +#define IQS5XX_SETUP_COMPLETE BIT(6) +#define IQS5XX_WDT BIT(5) +#define IQS5XX_ALP_REATI BIT(3) +#define IQS5XX_REATI BIT(2) + +#define IQS5XX_TP_EVENT BIT(2) +#define IQS5XX_EVENT_MODE BIT(0) #define IQS5XX_PROD_NUM 0x0000 -#define IQS5XX_ABS_X 0x0016 -#define IQS5XX_ABS_Y 0x0018 +#define IQS5XX_SYS_INFO0 0x000F +#define IQS5XX_SYS_INFO1 0x0010 #define IQS5XX_SYS_CTRL0 0x0431 #define IQS5XX_SYS_CTRL1 0x0432 #define IQS5XX_SYS_CFG0 0x058E #define IQS5XX_SYS_CFG1 0x058F -#define IQS5XX_TOTAL_RX 0x063D -#define IQS5XX_TOTAL_TX 0x063E -#define IQS5XX_XY_CFG0 0x0669 #define IQS5XX_X_RES 0x066E #define IQS5XX_Y_RES 0x0670 #define IQS5XX_CHKSM 0x83C0 @@ -99,6 +98,7 @@ struct iqs5xx_private { struct i2c_client *client; struct input_dev *input; struct gpio_desc *reset_gpio; + struct touchscreen_properties prop; struct mutex lock; u8 bl_status; }; @@ -126,6 +126,14 @@ struct iqs5xx_touch_data { u8 area; } __packed; +struct iqs5xx_status { + u8 sys_info[2]; + u8 num_active; + __be16 rel_x; + __be16 rel_y; + struct iqs5xx_touch_data touch_data[IQS5XX_NUM_CONTACTS]; +} __packed; + static int iqs5xx_read_burst(struct i2c_client *client, u16 reg, void *val, u16 len) { @@ -182,11 +190,6 @@ static int iqs5xx_read_word(struct i2c_client *client, u16 reg, u16 *val) return 0; } -static int iqs5xx_read_byte(struct i2c_client *client, u16 reg, u8 *val) -{ - return iqs5xx_read_burst(client, reg, val, sizeof(*val)); -} - static int iqs5xx_write_burst(struct i2c_client *client, u16 reg, const void *val, u16 len) { @@ -337,11 +340,16 @@ static int iqs5xx_bl_open(struct i2c_client *client) */ for (i = 0; i < IQS5XX_BL_ATTEMPTS; i++) { iqs5xx_reset(client); + usleep_range(350, 400); for (j = 0; j < IQS5XX_NUM_RETRIES; j++) { error = iqs5xx_bl_cmd(client, IQS5XX_BL_CMD_VER, 0); - if (!error || error == -EINVAL) - return error; + if (!error) + usleep_range(10000, 10100); + else if (error != -EINVAL) + continue; + + return error; } } @@ -481,12 +489,10 @@ static void iqs5xx_close(struct input_dev *input) static int iqs5xx_axis_init(struct i2c_client *client) { struct iqs5xx_private *iqs5xx = i2c_get_clientdata(client); - struct touchscreen_properties prop; + struct touchscreen_properties *prop = &iqs5xx->prop; struct input_dev *input; + u16 max_x, max_y; int error; - u16 max_x, max_x_hw; - u16 max_y, max_y_hw; - u8 val; if (!iqs5xx->input) { input = devm_input_allocate_device(&client->dev); @@ -506,89 +512,39 @@ static int iqs5xx_axis_init(struct i2c_client *client) iqs5xx->input = input; } - touchscreen_parse_properties(iqs5xx->input, true, &prop); - - error = iqs5xx_read_byte(client, IQS5XX_TOTAL_RX, &val); - if (error) - return error; - max_x_hw = (val - 1) * IQS5XX_NUM_POINTS; - - error = iqs5xx_read_byte(client, IQS5XX_TOTAL_TX, &val); + error = iqs5xx_read_word(client, IQS5XX_X_RES, &max_x); if (error) return error; - max_y_hw = (val - 1) * IQS5XX_NUM_POINTS; - error = iqs5xx_read_byte(client, IQS5XX_XY_CFG0, &val); + error = iqs5xx_read_word(client, IQS5XX_Y_RES, &max_y); if (error) return error; - if (val & IQS5XX_SWITCH_XY_AXIS) - swap(max_x_hw, max_y_hw); + input_abs_set_max(iqs5xx->input, ABS_MT_POSITION_X, max_x); + input_abs_set_max(iqs5xx->input, ABS_MT_POSITION_Y, max_y); - if (prop.swap_x_y) - val ^= IQS5XX_SWITCH_XY_AXIS; - - if (prop.invert_x) - val ^= prop.swap_x_y ? IQS5XX_FLIP_Y : IQS5XX_FLIP_X; - - if (prop.invert_y) - val ^= prop.swap_x_y ? IQS5XX_FLIP_X : IQS5XX_FLIP_Y; - - error = iqs5xx_write_byte(client, IQS5XX_XY_CFG0, val); - if (error) - return error; + touchscreen_parse_properties(iqs5xx->input, true, prop); - if (prop.max_x > max_x_hw) { + if (prop->max_x > IQS5XX_XY_RES_MAX) { dev_err(&client->dev, "Invalid maximum x-coordinate: %u > %u\n", - prop.max_x, max_x_hw); + prop->max_x, IQS5XX_XY_RES_MAX); return -EINVAL; - } else if (prop.max_x == 0) { - error = iqs5xx_read_word(client, IQS5XX_X_RES, &max_x); + } else if (prop->max_x != max_x) { + error = iqs5xx_write_word(client, IQS5XX_X_RES, prop->max_x); if (error) return error; - - input_abs_set_max(iqs5xx->input, - prop.swap_x_y ? ABS_MT_POSITION_Y : - ABS_MT_POSITION_X, - max_x); - } else { - max_x = (u16)prop.max_x; } - if (prop.max_y > max_y_hw) { + if (prop->max_y > IQS5XX_XY_RES_MAX) { dev_err(&client->dev, "Invalid maximum y-coordinate: %u > %u\n", - prop.max_y, max_y_hw); + prop->max_y, IQS5XX_XY_RES_MAX); return -EINVAL; - } else if (prop.max_y == 0) { - error = iqs5xx_read_word(client, IQS5XX_Y_RES, &max_y); + } else if (prop->max_y != max_y) { + error = iqs5xx_write_word(client, IQS5XX_Y_RES, prop->max_y); if (error) return error; - - input_abs_set_max(iqs5xx->input, - prop.swap_x_y ? ABS_MT_POSITION_X : - ABS_MT_POSITION_Y, - max_y); - } else { - max_y = (u16)prop.max_y; } - /* - * Write horizontal and vertical resolution to the device in case its - * original defaults were overridden or swapped as per the properties - * specified in the device tree. - */ - error = iqs5xx_write_word(client, - prop.swap_x_y ? IQS5XX_Y_RES : IQS5XX_X_RES, - max_x); - if (error) - return error; - - error = iqs5xx_write_word(client, - prop.swap_x_y ? IQS5XX_X_RES : IQS5XX_Y_RES, - max_y); - if (error) - return error; - error = input_mt_init_slots(iqs5xx->input, IQS5XX_NUM_CONTACTS, INPUT_MT_DIRECT); if (error) @@ -603,7 +559,6 @@ static int iqs5xx_dev_init(struct i2c_client *client) struct iqs5xx_private *iqs5xx = i2c_get_clientdata(client); struct iqs5xx_dev_id_info *dev_id_info; int error; - u8 val; u8 buf[sizeof(*dev_id_info) + 1]; error = iqs5xx_read_burst(client, IQS5XX_PROD_NUM, @@ -666,18 +621,18 @@ static int iqs5xx_dev_init(struct i2c_client *client) if (error) return error; - error = iqs5xx_read_byte(client, IQS5XX_SYS_CFG0, &val); + error = iqs5xx_write_byte(client, IQS5XX_SYS_CTRL0, IQS5XX_ACK_RESET); if (error) return error; - val |= IQS5XX_SETUP_COMPLETE; - val &= ~IQS5XX_SW_INPUT_EVENT; - error = iqs5xx_write_byte(client, IQS5XX_SYS_CFG0, val); + error = iqs5xx_write_byte(client, IQS5XX_SYS_CFG0, + IQS5XX_SETUP_COMPLETE | IQS5XX_WDT | + IQS5XX_ALP_REATI | IQS5XX_REATI); if (error) return error; - val = IQS5XX_TP_EVENT | IQS5XX_EVENT_MODE; - error = iqs5xx_write_byte(client, IQS5XX_SYS_CFG1, val); + error = iqs5xx_write_byte(client, IQS5XX_SYS_CFG1, + IQS5XX_TP_EVENT | IQS5XX_EVENT_MODE); if (error) return error; @@ -688,13 +643,12 @@ static int iqs5xx_dev_init(struct i2c_client *client) iqs5xx->bl_status = dev_id_info->bl_status; /* - * Closure of the first communication window that appears following the - * release of reset appears to kick off an initialization period during - * which further communication is met with clock stretching. The return - * from this function is delayed so that further communication attempts - * avoid this period. + * The following delay allows ATI to complete before the open and close + * callbacks are free to elicit I2C communication. Any attempts to read + * from or write to the device during this time may face extended clock + * stretching and prompt the I2C controller to report an error. */ - msleep(100); + msleep(250); return 0; } @@ -702,7 +656,7 @@ static int iqs5xx_dev_init(struct i2c_client *client) static irqreturn_t iqs5xx_irq(int irq, void *data) { struct iqs5xx_private *iqs5xx = data; - struct iqs5xx_touch_data touch_data[IQS5XX_NUM_CONTACTS]; + struct iqs5xx_status status; struct i2c_client *client = iqs5xx->client; struct input_dev *input = iqs5xx->input; int error, i; @@ -715,21 +669,35 @@ static irqreturn_t iqs5xx_irq(int irq, void *data) if (iqs5xx->bl_status == IQS5XX_BL_STATUS_RESET) return IRQ_NONE; - error = iqs5xx_read_burst(client, IQS5XX_ABS_X, - touch_data, sizeof(touch_data)); + error = iqs5xx_read_burst(client, IQS5XX_SYS_INFO0, + &status, sizeof(status)); if (error) return IRQ_NONE; - for (i = 0; i < ARRAY_SIZE(touch_data); i++) { - u16 pressure = be16_to_cpu(touch_data[i].strength); + if (status.sys_info[0] & IQS5XX_SHOW_RESET) { + dev_err(&client->dev, "Unexpected device reset\n"); + + error = iqs5xx_dev_init(client); + if (error) { + dev_err(&client->dev, + "Failed to re-initialize device: %d\n", error); + return IRQ_NONE; + } + + return IRQ_HANDLED; + } + + for (i = 0; i < ARRAY_SIZE(status.touch_data); i++) { + struct iqs5xx_touch_data *touch_data = &status.touch_data[i]; + u16 pressure = be16_to_cpu(touch_data->strength); input_mt_slot(input, i); if (input_mt_report_slot_state(input, MT_TOOL_FINGER, pressure != 0)) { - input_report_abs(input, ABS_MT_POSITION_X, - be16_to_cpu(touch_data[i].abs_x)); - input_report_abs(input, ABS_MT_POSITION_Y, - be16_to_cpu(touch_data[i].abs_y)); + touchscreen_report_pos(iqs5xx->input, &iqs5xx->prop, + be16_to_cpu(touch_data->abs_x), + be16_to_cpu(touch_data->abs_y), + true); input_report_abs(input, ABS_MT_PRESSURE, pressure); } } @@ -884,7 +852,7 @@ static int iqs5xx_fw_file_parse(struct i2c_client *client, static int iqs5xx_fw_file_write(struct i2c_client *client, const char *fw_file) { struct iqs5xx_private *iqs5xx = i2c_get_clientdata(client); - int error; + int error, error_bl = 0; u8 *pmap; if (iqs5xx->bl_status == IQS5XX_BL_STATUS_NONE) @@ -938,6 +906,7 @@ err_reset: usleep_range(10000, 10100); } + error_bl = error; error = iqs5xx_dev_init(client); if (!error && iqs5xx->bl_status == IQS5XX_BL_STATUS_RESET) error = -EINVAL; @@ -949,11 +918,15 @@ err_reset: err_kfree: kfree(pmap); + if (error_bl) + return error_bl; + return error; } -static ssize_t fw_file_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) +static ssize_t fw_file_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t count) { struct iqs5xx_private *iqs5xx = dev_get_drvdata(dev); struct i2c_client *client = iqs5xx->client; @@ -1012,7 +985,7 @@ static int __maybe_unused iqs5xx_suspend(struct device *dev) struct input_dev *input = iqs5xx->input; int error = 0; - if (!input) + if (!input || device_may_wakeup(dev)) return error; mutex_lock(&input->mutex); @@ -1031,7 +1004,7 @@ static int __maybe_unused iqs5xx_resume(struct device *dev) struct input_dev *input = iqs5xx->input; int error = 0; - if (!input) + if (!input || device_may_wakeup(dev)) return error; mutex_lock(&input->mutex); diff --git a/drivers/input/touchscreen/melfas_mip4.c b/drivers/input/touchscreen/melfas_mip4.c index c0050044a5a9..225796a3f546 100644 --- a/drivers/input/touchscreen/melfas_mip4.c +++ b/drivers/input/touchscreen/melfas_mip4.c @@ -465,13 +465,13 @@ static void mip4_report_keys(struct mip4_ts *ts, u8 *packet) static void mip4_report_touch(struct mip4_ts *ts, u8 *packet) { int id; - bool hover; - bool palm; + bool __always_unused hover; + bool __always_unused palm; bool state; u16 x, y; - u8 pressure_stage = 0; + u8 __always_unused pressure_stage = 0; u8 pressure; - u8 size; + u8 __always_unused size; u8 touch_major; u8 touch_minor; diff --git a/drivers/input/touchscreen/raydium_i2c_ts.c b/drivers/input/touchscreen/raydium_i2c_ts.c index 603a948460d6..4d2d22a86977 100644 --- a/drivers/input/touchscreen/raydium_i2c_ts.c +++ b/drivers/input/touchscreen/raydium_i2c_ts.c @@ -445,6 +445,7 @@ static int raydium_i2c_write_object(struct i2c_client *client, enum raydium_bl_ack state) { int error; + static const u8 cmd[] = { 0xFF, 0x39 }; error = raydium_i2c_send(client, RM_CMD_BOOT_WRT, data, len); if (error) { @@ -453,7 +454,7 @@ static int raydium_i2c_write_object(struct i2c_client *client, return error; } - error = raydium_i2c_send(client, RM_CMD_BOOT_ACK, NULL, 0); + error = raydium_i2c_send(client, RM_CMD_BOOT_ACK, cmd, sizeof(cmd)); if (error) { dev_err(&client->dev, "Ack obj command failed: %d\n", error); return error; diff --git a/drivers/input/touchscreen/st1232.c b/drivers/input/touchscreen/st1232.c index b4e7bcbe9b91..6abae665ca71 100644 --- a/drivers/input/touchscreen/st1232.c +++ b/drivers/input/touchscreen/st1232.c @@ -94,8 +94,13 @@ static int st1232_ts_wait_ready(struct st1232_ts_data *ts) for (retries = 10; retries; retries--) { error = st1232_ts_read_data(ts, REG_STATUS, 1); - if (!error && ts->read_buf[0] == (STATUS_NORMAL | ERROR_NONE)) - return 0; + if (!error) { + switch (ts->read_buf[0]) { + case STATUS_NORMAL | ERROR_NONE: + case STATUS_IDLE | ERROR_NONE: + return 0; + } + } usleep_range(1000, 2000); } diff --git a/drivers/input/touchscreen/stmpe-ts.c b/drivers/input/touchscreen/stmpe-ts.c index cd747725589b..25c45c3a3561 100644 --- a/drivers/input/touchscreen/stmpe-ts.c +++ b/drivers/input/touchscreen/stmpe-ts.c @@ -52,6 +52,7 @@ * @idev: registered input device * @work: a work item used to scan the device * @dev: a pointer back to the MFD cell struct device* + * @prop: Touchscreen properties * @ave_ctrl: Sample average control * (0 -> 1 sample, 1 -> 2 samples, 2 -> 4 samples, 3 -> 8 samples) * @touch_det_delay: Touch detect interrupt delay diff --git a/drivers/input/touchscreen/sur40.c b/drivers/input/touchscreen/sur40.c index 620cdd7d214a..12f2562b0141 100644 --- a/drivers/input/touchscreen/sur40.c +++ b/drivers/input/touchscreen/sur40.c @@ -787,6 +787,7 @@ static int sur40_probe(struct usb_interface *interface, dev_err(&interface->dev, "Unable to register video controls."); v4l2_ctrl_handler_free(&sur40->hdl); + error = sur40->hdl.error; goto err_unreg_v4l2; } diff --git a/drivers/input/touchscreen/surface3_spi.c b/drivers/input/touchscreen/surface3_spi.c index 731454599fce..1da23e5585a0 100644 --- a/drivers/input/touchscreen/surface3_spi.c +++ b/drivers/input/touchscreen/surface3_spi.c @@ -94,9 +94,7 @@ static void surface3_spi_report_touch(struct surface3_ts_data *ts_data, static void surface3_spi_process_touch(struct surface3_ts_data *ts_data, u8 *data) { - u16 timestamp; unsigned int i; - timestamp = get_unaligned_le16(&data[15]); for (i = 0; i < 13; i++) { struct surface3_ts_data_finger *finger; diff --git a/drivers/input/touchscreen/usbtouchscreen.c b/drivers/input/touchscreen/usbtouchscreen.c index 397cb1d3f481..c847453a03c2 100644 --- a/drivers/input/touchscreen/usbtouchscreen.c +++ b/drivers/input/touchscreen/usbtouchscreen.c @@ -1044,6 +1044,7 @@ static void nexio_exit(struct usbtouch_usb *usbtouch) static int nexio_read_data(struct usbtouch_usb *usbtouch, unsigned char *pkt) { + struct device *dev = &usbtouch->interface->dev; struct nexio_touch_packet *packet = (void *) pkt; struct nexio_priv *priv = usbtouch->priv; unsigned int data_len = be16_to_cpu(packet->data_len); @@ -1062,6 +1063,8 @@ static int nexio_read_data(struct usbtouch_usb *usbtouch, unsigned char *pkt) /* send ACK */ ret = usb_submit_urb(priv->ack, GFP_ATOMIC); + if (ret) + dev_warn(dev, "Failed to submit ACK URB: %d\n", ret); if (!usbtouch->type->max_xc) { usbtouch->type->max_xc = 2 * x_len; diff --git a/drivers/input/touchscreen/zinitix.c b/drivers/input/touchscreen/zinitix.c index a3e3adbabc67..3b636beb583c 100644 --- a/drivers/input/touchscreen/zinitix.c +++ b/drivers/input/touchscreen/zinitix.c @@ -161,7 +161,7 @@ static int zinitix_read_data(struct i2c_client *client, ret = i2c_master_recv(client, (u8 *)values, length); if (ret != length) - return ret < 0 ? ret : -EIO; ; + return ret < 0 ? ret : -EIO; return 0; } @@ -190,7 +190,7 @@ static int zinitix_write_cmd(struct i2c_client *client, u16 reg) return 0; } -static bool zinitix_init_touch(struct bt541_ts_data *bt541) +static int zinitix_init_touch(struct bt541_ts_data *bt541) { struct i2c_client *client = bt541->client; int i; diff --git a/drivers/interconnect/qcom/Kconfig b/drivers/interconnect/qcom/Kconfig index b3fb5b02bcf1..ca52647f8955 100644 --- a/drivers/interconnect/qcom/Kconfig +++ b/drivers/interconnect/qcom/Kconfig @@ -17,6 +17,15 @@ config INTERCONNECT_QCOM_MSM8916 This is a driver for the Qualcomm Network-on-Chip on msm8916-based platforms. +config INTERCONNECT_QCOM_MSM8939 + tristate "Qualcomm MSM8939 interconnect driver" + depends on INTERCONNECT_QCOM + depends on QCOM_SMD_RPM + select INTERCONNECT_QCOM_SMD_RPM + help + This is a driver for the Qualcomm Network-on-Chip on msm8939-based + platforms. + config INTERCONNECT_QCOM_MSM8974 tristate "Qualcomm MSM8974 interconnect driver" depends on INTERCONNECT_QCOM @@ -74,6 +83,15 @@ config INTERCONNECT_QCOM_SDM845 This is a driver for the Qualcomm Network-on-Chip on sdm845-based platforms. +config INTERCONNECT_QCOM_SDX55 + tristate "Qualcomm SDX55 interconnect driver" + depends on INTERCONNECT_QCOM_RPMH_POSSIBLE + select INTERCONNECT_QCOM_RPMH + select INTERCONNECT_QCOM_BCM_VOTER + help + This is a driver for the Qualcomm Network-on-Chip on sdx55-based + platforms. + config INTERCONNECT_QCOM_SM8150 tristate "Qualcomm SM8150 interconnect driver" depends on INTERCONNECT_QCOM_RPMH_POSSIBLE diff --git a/drivers/interconnect/qcom/Makefile b/drivers/interconnect/qcom/Makefile index cf628f7990cd..c6a735df067e 100644 --- a/drivers/interconnect/qcom/Makefile +++ b/drivers/interconnect/qcom/Makefile @@ -2,24 +2,28 @@ icc-bcm-voter-objs := bcm-voter.o qnoc-msm8916-objs := msm8916.o +qnoc-msm8939-objs := msm8939.o qnoc-msm8974-objs := msm8974.o icc-osm-l3-objs := osm-l3.o qnoc-qcs404-objs := qcs404.o icc-rpmh-obj := icc-rpmh.o qnoc-sc7180-objs := sc7180.o qnoc-sdm845-objs := sdm845.o +qnoc-sdx55-objs := sdx55.o qnoc-sm8150-objs := sm8150.o qnoc-sm8250-objs := sm8250.o -icc-smd-rpm-objs := smd-rpm.o +icc-smd-rpm-objs := smd-rpm.o icc-rpm.o obj-$(CONFIG_INTERCONNECT_QCOM_BCM_VOTER) += icc-bcm-voter.o obj-$(CONFIG_INTERCONNECT_QCOM_MSM8916) += qnoc-msm8916.o +obj-$(CONFIG_INTERCONNECT_QCOM_MSM8939) += qnoc-msm8939.o obj-$(CONFIG_INTERCONNECT_QCOM_MSM8974) += qnoc-msm8974.o obj-$(CONFIG_INTERCONNECT_QCOM_OSM_L3) += icc-osm-l3.o obj-$(CONFIG_INTERCONNECT_QCOM_QCS404) += qnoc-qcs404.o obj-$(CONFIG_INTERCONNECT_QCOM_RPMH) += icc-rpmh.o obj-$(CONFIG_INTERCONNECT_QCOM_SC7180) += qnoc-sc7180.o obj-$(CONFIG_INTERCONNECT_QCOM_SDM845) += qnoc-sdm845.o +obj-$(CONFIG_INTERCONNECT_QCOM_SDX55) += qnoc-sdx55.o obj-$(CONFIG_INTERCONNECT_QCOM_SM8150) += qnoc-sm8150.o obj-$(CONFIG_INTERCONNECT_QCOM_SM8250) += qnoc-sm8250.o obj-$(CONFIG_INTERCONNECT_QCOM_SMD_RPM) += icc-smd-rpm.o diff --git a/drivers/interconnect/qcom/icc-rpm.c b/drivers/interconnect/qcom/icc-rpm.c new file mode 100644 index 000000000000..cc6095492cbe --- /dev/null +++ b/drivers/interconnect/qcom/icc-rpm.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Linaro Ltd + */ + +#include <linux/clk.h> +#include <linux/device.h> +#include <linux/interconnect-provider.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include "smd-rpm.h" +#include "icc-rpm.h" + +static int qcom_icc_set(struct icc_node *src, struct icc_node *dst) +{ + struct qcom_icc_provider *qp; + struct qcom_icc_node *qn; + struct icc_provider *provider; + struct icc_node *n; + u64 sum_bw; + u64 max_peak_bw; + u64 rate; + u32 agg_avg = 0; + u32 agg_peak = 0; + int ret, i; + + qn = src->data; + provider = src->provider; + qp = to_qcom_provider(provider); + + list_for_each_entry(n, &provider->nodes, node_list) + provider->aggregate(n, 0, n->avg_bw, n->peak_bw, + &agg_avg, &agg_peak); + + sum_bw = icc_units_to_bps(agg_avg); + max_peak_bw = icc_units_to_bps(agg_peak); + + /* send bandwidth request message to the RPM processor */ + if (qn->mas_rpm_id != -1) { + ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, + RPM_BUS_MASTER_REQ, + qn->mas_rpm_id, + sum_bw); + if (ret) { + pr_err("qcom_icc_rpm_smd_send mas %d error %d\n", + qn->mas_rpm_id, ret); + return ret; + } + } + + if (qn->slv_rpm_id != -1) { + ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, + RPM_BUS_SLAVE_REQ, + qn->slv_rpm_id, + sum_bw); + if (ret) { + pr_err("qcom_icc_rpm_smd_send slv error %d\n", + ret); + return ret; + } + } + + rate = max(sum_bw, max_peak_bw); + + do_div(rate, qn->buswidth); + + if (qn->rate == rate) + return 0; + + for (i = 0; i < qp->num_clks; i++) { + ret = clk_set_rate(qp->bus_clks[i].clk, rate); + if (ret) { + pr_err("%s clk_set_rate error: %d\n", + qp->bus_clks[i].id, ret); + return ret; + } + } + + qn->rate = rate; + + return 0; +} + +int qnoc_probe(struct platform_device *pdev, size_t cd_size, int cd_num, + const struct clk_bulk_data *cd) +{ + struct device *dev = &pdev->dev; + const struct qcom_icc_desc *desc; + struct icc_onecell_data *data; + struct icc_provider *provider; + struct qcom_icc_node **qnodes; + struct qcom_icc_provider *qp; + struct icc_node *node; + size_t num_nodes, i; + int ret; + + /* wait for the RPM proxy */ + if (!qcom_icc_rpm_smd_available()) + return -EPROBE_DEFER; + + desc = of_device_get_match_data(dev); + if (!desc) + return -EINVAL; + + qnodes = desc->nodes; + num_nodes = desc->num_nodes; + + qp = devm_kzalloc(dev, sizeof(*qp), GFP_KERNEL); + if (!qp) + return -ENOMEM; + + data = devm_kzalloc(dev, struct_size(data, nodes, num_nodes), + GFP_KERNEL); + if (!data) + return -ENOMEM; + + qp->bus_clks = devm_kmemdup(dev, cd, cd_size, + GFP_KERNEL); + if (!qp->bus_clks) + return -ENOMEM; + + qp->num_clks = cd_num; + ret = devm_clk_bulk_get(dev, qp->num_clks, qp->bus_clks); + if (ret) + return ret; + + ret = clk_bulk_prepare_enable(qp->num_clks, qp->bus_clks); + if (ret) + return ret; + + provider = &qp->provider; + INIT_LIST_HEAD(&provider->nodes); + provider->dev = dev; + provider->set = qcom_icc_set; + provider->aggregate = icc_std_aggregate; + provider->xlate = of_icc_xlate_onecell; + provider->data = data; + + ret = icc_provider_add(provider); + if (ret) { + dev_err(dev, "error adding interconnect provider: %d\n", ret); + clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); + return ret; + } + + for (i = 0; i < num_nodes; i++) { + size_t j; + + node = icc_node_create(qnodes[i]->id); + if (IS_ERR(node)) { + ret = PTR_ERR(node); + goto err; + } + + node->name = qnodes[i]->name; + node->data = qnodes[i]; + icc_node_add(node, provider); + + for (j = 0; j < qnodes[i]->num_links; j++) + icc_link_create(node, qnodes[i]->links[j]); + + data->nodes[i] = node; + } + data->num_nodes = num_nodes; + + platform_set_drvdata(pdev, qp); + + return 0; +err: + icc_nodes_remove(provider); + clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); + icc_provider_del(provider); + + return ret; +} +EXPORT_SYMBOL(qnoc_probe); + +int qnoc_remove(struct platform_device *pdev) +{ + struct qcom_icc_provider *qp = platform_get_drvdata(pdev); + + icc_nodes_remove(&qp->provider); + clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); + return icc_provider_del(&qp->provider); +} +EXPORT_SYMBOL(qnoc_remove); diff --git a/drivers/interconnect/qcom/icc-rpm.h b/drivers/interconnect/qcom/icc-rpm.h new file mode 100644 index 000000000000..79a6f68249c1 --- /dev/null +++ b/drivers/interconnect/qcom/icc-rpm.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Linaro Ltd + */ + +#ifndef __DRIVERS_INTERCONNECT_QCOM_ICC_RPM_H +#define __DRIVERS_INTERCONNECT_QCOM_ICC_RPM_H + +#define RPM_BUS_MASTER_REQ 0x73616d62 +#define RPM_BUS_SLAVE_REQ 0x766c7362 + +#define QCOM_MAX_LINKS 12 + +#define to_qcom_provider(_provider) \ + container_of(_provider, struct qcom_icc_provider, provider) + +/** + * struct qcom_icc_provider - Qualcomm specific interconnect provider + * @provider: generic interconnect provider + * @bus_clks: the clk_bulk_data table of bus clocks + * @num_clks: the total number of clk_bulk_data entries + */ +struct qcom_icc_provider { + struct icc_provider provider; + struct clk_bulk_data *bus_clks; + int num_clks; +}; + +/** + * struct qcom_icc_node - Qualcomm specific interconnect nodes + * @name: the node name used in debugfs + * @id: a unique node identifier + * @links: an array of nodes where we can go next while traversing + * @num_links: the total number of @links + * @buswidth: width of the interconnect between a node and the bus (bytes) + * @mas_rpm_id: RPM id for devices that are bus masters + * @slv_rpm_id: RPM id for devices that are bus slaves + * @rate: current bus clock rate in Hz + */ +struct qcom_icc_node { + unsigned char *name; + u16 id; + u16 links[QCOM_MAX_LINKS]; + u16 num_links; + u16 buswidth; + int mas_rpm_id; + int slv_rpm_id; + u64 rate; +}; + +struct qcom_icc_desc { + struct qcom_icc_node **nodes; + size_t num_nodes; +}; + +#define DEFINE_QNODE(_name, _id, _buswidth, _mas_rpm_id, _slv_rpm_id, \ + ...) \ + static struct qcom_icc_node _name = { \ + .name = #_name, \ + .id = _id, \ + .buswidth = _buswidth, \ + .mas_rpm_id = _mas_rpm_id, \ + .slv_rpm_id = _slv_rpm_id, \ + .num_links = ARRAY_SIZE(((int[]){ __VA_ARGS__ })), \ + .links = { __VA_ARGS__ }, \ + } + + +int qnoc_probe(struct platform_device *pdev, size_t cd_size, int cd_num, + const struct clk_bulk_data *cd); +int qnoc_remove(struct platform_device *pdev); + +#endif diff --git a/drivers/interconnect/qcom/msm8916.c b/drivers/interconnect/qcom/msm8916.c index e8371d40ab8d..fc3689c8947a 100644 --- a/drivers/interconnect/qcom/msm8916.c +++ b/drivers/interconnect/qcom/msm8916.c @@ -15,9 +15,7 @@ #include <dt-bindings/interconnect/qcom,msm8916.h> #include "smd-rpm.h" - -#define RPM_BUS_MASTER_REQ 0x73616d62 -#define RPM_BUS_SLAVE_REQ 0x766c7362 +#include "icc-rpm.h" enum { MSM8916_BIMC_SNOC_MAS = 1, @@ -107,67 +105,11 @@ enum { MSM8916_SNOC_PNOC_SLV, }; -#define to_msm8916_provider(_provider) \ - container_of(_provider, struct msm8916_icc_provider, provider) - static const struct clk_bulk_data msm8916_bus_clocks[] = { { .id = "bus" }, { .id = "bus_a" }, }; -/** - * struct msm8916_icc_provider - Qualcomm specific interconnect provider - * @provider: generic interconnect provider - * @bus_clks: the clk_bulk_data table of bus clocks - * @num_clks: the total number of clk_bulk_data entries - */ -struct msm8916_icc_provider { - struct icc_provider provider; - struct clk_bulk_data *bus_clks; - int num_clks; -}; - -#define MSM8916_MAX_LINKS 8 - -/** - * struct msm8916_icc_node - Qualcomm specific interconnect nodes - * @name: the node name used in debugfs - * @id: a unique node identifier - * @links: an array of nodes where we can go next while traversing - * @num_links: the total number of @links - * @buswidth: width of the interconnect between a node and the bus (bytes) - * @mas_rpm_id: RPM ID for devices that are bus masters - * @slv_rpm_id: RPM ID for devices that are bus slaves - * @rate: current bus clock rate in Hz - */ -struct msm8916_icc_node { - unsigned char *name; - u16 id; - u16 links[MSM8916_MAX_LINKS]; - u16 num_links; - u16 buswidth; - int mas_rpm_id; - int slv_rpm_id; - u64 rate; -}; - -struct msm8916_icc_desc { - struct msm8916_icc_node **nodes; - size_t num_nodes; -}; - -#define DEFINE_QNODE(_name, _id, _buswidth, _mas_rpm_id, _slv_rpm_id, \ - ...) \ - static struct msm8916_icc_node _name = { \ - .name = #_name, \ - .id = _id, \ - .buswidth = _buswidth, \ - .mas_rpm_id = _mas_rpm_id, \ - .slv_rpm_id = _slv_rpm_id, \ - .num_links = ARRAY_SIZE(((int[]){ __VA_ARGS__ })), \ - .links = { __VA_ARGS__ }, \ - } - DEFINE_QNODE(bimc_snoc_mas, MSM8916_BIMC_SNOC_MAS, 8, -1, -1, MSM8916_BIMC_SNOC_SLV); DEFINE_QNODE(bimc_snoc_slv, MSM8916_BIMC_SNOC_SLV, 8, -1, -1, MSM8916_SNOC_INT_0, MSM8916_SNOC_INT_1); DEFINE_QNODE(mas_apss, MSM8916_MASTER_AMPSS_M0, 8, -1, -1, MSM8916_SLAVE_EBI_CH0, MSM8916_BIMC_SNOC_MAS, MSM8916_SLAVE_AMPSS_L2); @@ -254,7 +196,7 @@ DEFINE_QNODE(snoc_int_bimc, MSM8916_SNOC_INT_BIMC, 8, 101, 132, MSM8916_SNOC_BIM DEFINE_QNODE(snoc_pcnoc_mas, MSM8916_SNOC_PNOC_MAS, 8, -1, -1, MSM8916_SNOC_PNOC_SLV); DEFINE_QNODE(snoc_pcnoc_slv, MSM8916_SNOC_PNOC_SLV, 8, -1, -1, MSM8916_PNOC_INT_0); -static struct msm8916_icc_node *msm8916_snoc_nodes[] = { +static struct qcom_icc_node *msm8916_snoc_nodes[] = { [BIMC_SNOC_SLV] = &bimc_snoc_slv, [MASTER_JPEG] = &mas_jpeg, [MASTER_MDP_PORT0] = &mas_mdp, @@ -283,12 +225,12 @@ static struct msm8916_icc_node *msm8916_snoc_nodes[] = { [SNOC_QDSS_INT] = &qdss_int, }; -static struct msm8916_icc_desc msm8916_snoc = { +static struct qcom_icc_desc msm8916_snoc = { .nodes = msm8916_snoc_nodes, .num_nodes = ARRAY_SIZE(msm8916_snoc_nodes), }; -static struct msm8916_icc_node *msm8916_bimc_nodes[] = { +static struct qcom_icc_node *msm8916_bimc_nodes[] = { [BIMC_SNOC_MAS] = &bimc_snoc_mas, [MASTER_AMPSS_M0] = &mas_apss, [MASTER_GRAPHICS_3D] = &mas_gfx, @@ -300,12 +242,12 @@ static struct msm8916_icc_node *msm8916_bimc_nodes[] = { [SNOC_BIMC_1_SLV] = &snoc_bimc_1_slv, }; -static struct msm8916_icc_desc msm8916_bimc = { +static struct qcom_icc_desc msm8916_bimc = { .nodes = msm8916_bimc_nodes, .num_nodes = ARRAY_SIZE(msm8916_bimc_nodes), }; -static struct msm8916_icc_node *msm8916_pcnoc_nodes[] = { +static struct qcom_icc_node *msm8916_pcnoc_nodes[] = { [MASTER_BLSP_1] = &mas_blsp_1, [MASTER_DEHR] = &mas_dehr, [MASTER_LPASS] = &mas_audio, @@ -358,178 +300,15 @@ static struct msm8916_icc_node *msm8916_pcnoc_nodes[] = { [SNOC_PCNOC_SLV] = &snoc_pcnoc_slv, }; -static struct msm8916_icc_desc msm8916_pcnoc = { +static struct qcom_icc_desc msm8916_pcnoc = { .nodes = msm8916_pcnoc_nodes, .num_nodes = ARRAY_SIZE(msm8916_pcnoc_nodes), }; -static int msm8916_icc_set(struct icc_node *src, struct icc_node *dst) -{ - struct msm8916_icc_provider *qp; - struct msm8916_icc_node *qn; - u64 sum_bw, max_peak_bw, rate; - u32 agg_avg = 0, agg_peak = 0; - struct icc_provider *provider; - struct icc_node *n; - int ret, i; - - qn = src->data; - provider = src->provider; - qp = to_msm8916_provider(provider); - - list_for_each_entry(n, &provider->nodes, node_list) - provider->aggregate(n, 0, n->avg_bw, n->peak_bw, - &agg_avg, &agg_peak); - - sum_bw = icc_units_to_bps(agg_avg); - max_peak_bw = icc_units_to_bps(agg_peak); - - /* send bandwidth request message to the RPM processor */ - if (qn->mas_rpm_id != -1) { - ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, - RPM_BUS_MASTER_REQ, - qn->mas_rpm_id, - sum_bw); - if (ret) { - pr_err("qcom_icc_rpm_smd_send mas %d error %d\n", - qn->mas_rpm_id, ret); - return ret; - } - } - - if (qn->slv_rpm_id != -1) { - ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, - RPM_BUS_SLAVE_REQ, - qn->slv_rpm_id, - sum_bw); - if (ret) { - pr_err("qcom_icc_rpm_smd_send slv error %d\n", - ret); - return ret; - } - } - - rate = max(sum_bw, max_peak_bw); - - do_div(rate, qn->buswidth); - - if (qn->rate == rate) - return 0; - - for (i = 0; i < qp->num_clks; i++) { - ret = clk_set_rate(qp->bus_clks[i].clk, rate); - if (ret) { - pr_err("%s clk_set_rate error: %d\n", - qp->bus_clks[i].id, ret); - return ret; - } - } - - qn->rate = rate; - - return 0; -} - static int msm8916_qnoc_probe(struct platform_device *pdev) { - const struct msm8916_icc_desc *desc; - struct msm8916_icc_node **qnodes; - struct msm8916_icc_provider *qp; - struct device *dev = &pdev->dev; - struct icc_onecell_data *data; - struct icc_provider *provider; - struct icc_node *node; - size_t num_nodes, i; - int ret; - - /* wait for the RPM proxy */ - if (!qcom_icc_rpm_smd_available()) - return -EPROBE_DEFER; - - desc = of_device_get_match_data(dev); - if (!desc) - return -EINVAL; - - qnodes = desc->nodes; - num_nodes = desc->num_nodes; - - qp = devm_kzalloc(dev, sizeof(*qp), GFP_KERNEL); - if (!qp) - return -ENOMEM; - - data = devm_kzalloc(dev, struct_size(data, nodes, num_nodes), - GFP_KERNEL); - if (!data) - return -ENOMEM; - - qp->bus_clks = devm_kmemdup(dev, msm8916_bus_clocks, - sizeof(msm8916_bus_clocks), GFP_KERNEL); - if (!qp->bus_clks) - return -ENOMEM; - - qp->num_clks = ARRAY_SIZE(msm8916_bus_clocks); - ret = devm_clk_bulk_get(dev, qp->num_clks, qp->bus_clks); - if (ret) - return ret; - - ret = clk_bulk_prepare_enable(qp->num_clks, qp->bus_clks); - if (ret) - return ret; - - provider = &qp->provider; - INIT_LIST_HEAD(&provider->nodes); - provider->dev = dev; - provider->set = msm8916_icc_set; - provider->aggregate = icc_std_aggregate; - provider->xlate = of_icc_xlate_onecell; - provider->data = data; - - ret = icc_provider_add(provider); - if (ret) { - dev_err(dev, "error adding interconnect provider: %d\n", ret); - clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - return ret; - } - - for (i = 0; i < num_nodes; i++) { - size_t j; - - node = icc_node_create(qnodes[i]->id); - if (IS_ERR(node)) { - ret = PTR_ERR(node); - goto err; - } - - node->name = qnodes[i]->name; - node->data = qnodes[i]; - icc_node_add(node, provider); - - for (j = 0; j < qnodes[i]->num_links; j++) - icc_link_create(node, qnodes[i]->links[j]); - - data->nodes[i] = node; - } - data->num_nodes = num_nodes; - - platform_set_drvdata(pdev, qp); - - return 0; - -err: - icc_nodes_remove(provider); - icc_provider_del(provider); - clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - - return ret; -} - -static int msm8916_qnoc_remove(struct platform_device *pdev) -{ - struct msm8916_icc_provider *qp = platform_get_drvdata(pdev); - - icc_nodes_remove(&qp->provider); - clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - return icc_provider_del(&qp->provider); + return qnoc_probe(pdev, sizeof(msm8916_bus_clocks), + ARRAY_SIZE(msm8916_bus_clocks), msm8916_bus_clocks); } static const struct of_device_id msm8916_noc_of_match[] = { @@ -542,7 +321,7 @@ MODULE_DEVICE_TABLE(of, msm8916_noc_of_match); static struct platform_driver msm8916_noc_driver = { .probe = msm8916_qnoc_probe, - .remove = msm8916_qnoc_remove, + .remove = qnoc_remove, .driver = { .name = "qnoc-msm8916", .of_match_table = msm8916_noc_of_match, diff --git a/drivers/interconnect/qcom/msm8939.c b/drivers/interconnect/qcom/msm8939.c new file mode 100644 index 000000000000..dfbec30ed149 --- /dev/null +++ b/drivers/interconnect/qcom/msm8939.c @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Linaro Ltd + * Author: Jun Nie <jun.nie@linaro.org> + * With reference of msm8916 interconnect driver of Georgi Djakov. + */ + +#include <linux/clk.h> +#include <linux/device.h> +#include <linux/interconnect-provider.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/of_device.h> + +#include <dt-bindings/interconnect/qcom,msm8939.h> + +#include "smd-rpm.h" +#include "icc-rpm.h" + +enum { + MSM8939_BIMC_SNOC_MAS = 1, + MSM8939_BIMC_SNOC_SLV, + MSM8939_MASTER_AMPSS_M0, + MSM8939_MASTER_LPASS, + MSM8939_MASTER_BLSP_1, + MSM8939_MASTER_DEHR, + MSM8939_MASTER_GRAPHICS_3D, + MSM8939_MASTER_JPEG, + MSM8939_MASTER_MDP_PORT0, + MSM8939_MASTER_MDP_PORT1, + MSM8939_MASTER_CPP, + MSM8939_MASTER_CRYPTO_CORE0, + MSM8939_MASTER_SDCC_1, + MSM8939_MASTER_SDCC_2, + MSM8939_MASTER_QDSS_BAM, + MSM8939_MASTER_QDSS_ETR, + MSM8939_MASTER_SNOC_CFG, + MSM8939_MASTER_SPDM, + MSM8939_MASTER_TCU0, + MSM8939_MASTER_USB_HS1, + MSM8939_MASTER_USB_HS2, + MSM8939_MASTER_VFE, + MSM8939_MASTER_VIDEO_P0, + MSM8939_SNOC_MM_INT_0, + MSM8939_SNOC_MM_INT_1, + MSM8939_SNOC_MM_INT_2, + MSM8939_PNOC_INT_0, + MSM8939_PNOC_INT_1, + MSM8939_PNOC_MAS_0, + MSM8939_PNOC_MAS_1, + MSM8939_PNOC_SLV_0, + MSM8939_PNOC_SLV_1, + MSM8939_PNOC_SLV_2, + MSM8939_PNOC_SLV_3, + MSM8939_PNOC_SLV_4, + MSM8939_PNOC_SLV_8, + MSM8939_PNOC_SLV_9, + MSM8939_PNOC_SNOC_MAS, + MSM8939_PNOC_SNOC_SLV, + MSM8939_SNOC_QDSS_INT, + MSM8939_SLAVE_AMPSS_L2, + MSM8939_SLAVE_APSS, + MSM8939_SLAVE_LPASS, + MSM8939_SLAVE_BIMC_CFG, + MSM8939_SLAVE_BLSP_1, + MSM8939_SLAVE_BOOT_ROM, + MSM8939_SLAVE_CAMERA_CFG, + MSM8939_SLAVE_CATS_128, + MSM8939_SLAVE_OCMEM_64, + MSM8939_SLAVE_CLK_CTL, + MSM8939_SLAVE_CRYPTO_0_CFG, + MSM8939_SLAVE_DEHR_CFG, + MSM8939_SLAVE_DISPLAY_CFG, + MSM8939_SLAVE_EBI_CH0, + MSM8939_SLAVE_GRAPHICS_3D_CFG, + MSM8939_SLAVE_IMEM_CFG, + MSM8939_SLAVE_IMEM, + MSM8939_SLAVE_MPM, + MSM8939_SLAVE_MSG_RAM, + MSM8939_SLAVE_MSS, + MSM8939_SLAVE_PDM, + MSM8939_SLAVE_PMIC_ARB, + MSM8939_SLAVE_PNOC_CFG, + MSM8939_SLAVE_PRNG, + MSM8939_SLAVE_QDSS_CFG, + MSM8939_SLAVE_QDSS_STM, + MSM8939_SLAVE_RBCPR_CFG, + MSM8939_SLAVE_SDCC_1, + MSM8939_SLAVE_SDCC_2, + MSM8939_SLAVE_SECURITY, + MSM8939_SLAVE_SNOC_CFG, + MSM8939_SLAVE_SPDM, + MSM8939_SLAVE_SRVC_SNOC, + MSM8939_SLAVE_TCSR, + MSM8939_SLAVE_TLMM, + MSM8939_SLAVE_USB_HS1, + MSM8939_SLAVE_USB_HS2, + MSM8939_SLAVE_VENUS_CFG, + MSM8939_SNOC_BIMC_0_MAS, + MSM8939_SNOC_BIMC_0_SLV, + MSM8939_SNOC_BIMC_1_MAS, + MSM8939_SNOC_BIMC_1_SLV, + MSM8939_SNOC_BIMC_2_MAS, + MSM8939_SNOC_BIMC_2_SLV, + MSM8939_SNOC_INT_0, + MSM8939_SNOC_INT_1, + MSM8939_SNOC_INT_BIMC, + MSM8939_SNOC_PNOC_MAS, + MSM8939_SNOC_PNOC_SLV, +}; + +static const struct clk_bulk_data msm8939_bus_clocks[] = { + { .id = "bus" }, + { .id = "bus_a" }, +}; + +DEFINE_QNODE(bimc_snoc_mas, MSM8939_BIMC_SNOC_MAS, 8, -1, -1, MSM8939_BIMC_SNOC_SLV); +DEFINE_QNODE(bimc_snoc_slv, MSM8939_BIMC_SNOC_SLV, 16, -1, 2, MSM8939_SNOC_INT_0, MSM8939_SNOC_INT_1); +DEFINE_QNODE(mas_apss, MSM8939_MASTER_AMPSS_M0, 16, -1, -1, MSM8939_SLAVE_EBI_CH0, MSM8939_BIMC_SNOC_MAS, MSM8939_SLAVE_AMPSS_L2); +DEFINE_QNODE(mas_audio, MSM8939_MASTER_LPASS, 4, -1, -1, MSM8939_PNOC_MAS_0); +DEFINE_QNODE(mas_blsp_1, MSM8939_MASTER_BLSP_1, 4, -1, -1, MSM8939_PNOC_MAS_1); +DEFINE_QNODE(mas_dehr, MSM8939_MASTER_DEHR, 4, -1, -1, MSM8939_PNOC_MAS_0); +DEFINE_QNODE(mas_gfx, MSM8939_MASTER_GRAPHICS_3D, 16, -1, -1, MSM8939_SLAVE_EBI_CH0, MSM8939_BIMC_SNOC_MAS, MSM8939_SLAVE_AMPSS_L2); +DEFINE_QNODE(mas_jpeg, MSM8939_MASTER_JPEG, 16, -1, -1, MSM8939_SNOC_MM_INT_0, MSM8939_SNOC_MM_INT_2); +DEFINE_QNODE(mas_mdp0, MSM8939_MASTER_MDP_PORT0, 16, -1, -1, MSM8939_SNOC_MM_INT_1, MSM8939_SNOC_MM_INT_2); +DEFINE_QNODE(mas_mdp1, MSM8939_MASTER_MDP_PORT1, 16, -1, -1, MSM8939_SNOC_MM_INT_0, MSM8939_SNOC_MM_INT_2); +DEFINE_QNODE(mas_cpp, MSM8939_MASTER_CPP, 16, -1, -1, MSM8939_SNOC_MM_INT_0, MSM8939_SNOC_MM_INT_2); +DEFINE_QNODE(mas_pcnoc_crypto_0, MSM8939_MASTER_CRYPTO_CORE0, 8, -1, -1, MSM8939_PNOC_INT_1); +DEFINE_QNODE(mas_pcnoc_sdcc_1, MSM8939_MASTER_SDCC_1, 8, -1, -1, MSM8939_PNOC_INT_1); +DEFINE_QNODE(mas_pcnoc_sdcc_2, MSM8939_MASTER_SDCC_2, 8, -1, -1, MSM8939_PNOC_INT_1); +DEFINE_QNODE(mas_qdss_bam, MSM8939_MASTER_QDSS_BAM, 8, -1, -1, MSM8939_SNOC_QDSS_INT); +DEFINE_QNODE(mas_qdss_etr, MSM8939_MASTER_QDSS_ETR, 8, -1, -1, MSM8939_SNOC_QDSS_INT); +DEFINE_QNODE(mas_snoc_cfg, MSM8939_MASTER_SNOC_CFG, 4, 20, -1, MSM8939_SLAVE_SRVC_SNOC); +DEFINE_QNODE(mas_spdm, MSM8939_MASTER_SPDM, 4, -1, -1, MSM8939_PNOC_MAS_0); +DEFINE_QNODE(mas_tcu0, MSM8939_MASTER_TCU0, 16, -1, -1, MSM8939_SLAVE_EBI_CH0, MSM8939_BIMC_SNOC_MAS, MSM8939_SLAVE_AMPSS_L2); +DEFINE_QNODE(mas_usb_hs1, MSM8939_MASTER_USB_HS1, 4, -1, -1, MSM8939_PNOC_MAS_1); +DEFINE_QNODE(mas_usb_hs2, MSM8939_MASTER_USB_HS2, 4, -1, -1, MSM8939_PNOC_MAS_1); +DEFINE_QNODE(mas_vfe, MSM8939_MASTER_VFE, 16, -1, -1, MSM8939_SNOC_MM_INT_1, MSM8939_SNOC_MM_INT_2); +DEFINE_QNODE(mas_video, MSM8939_MASTER_VIDEO_P0, 16, -1, -1, MSM8939_SNOC_MM_INT_0, MSM8939_SNOC_MM_INT_2); +DEFINE_QNODE(mm_int_0, MSM8939_SNOC_MM_INT_0, 16, -1, -1, MSM8939_SNOC_BIMC_2_MAS); +DEFINE_QNODE(mm_int_1, MSM8939_SNOC_MM_INT_1, 16, -1, -1, MSM8939_SNOC_BIMC_1_MAS); +DEFINE_QNODE(mm_int_2, MSM8939_SNOC_MM_INT_2, 16, -1, -1, MSM8939_SNOC_INT_0); +DEFINE_QNODE(pcnoc_int_0, MSM8939_PNOC_INT_0, 8, -1, -1, MSM8939_PNOC_SNOC_MAS, MSM8939_PNOC_SLV_0, MSM8939_PNOC_SLV_1, MSM8939_PNOC_SLV_2, MSM8939_PNOC_SLV_3, MSM8939_PNOC_SLV_4, MSM8939_PNOC_SLV_8, MSM8939_PNOC_SLV_9); +DEFINE_QNODE(pcnoc_int_1, MSM8939_PNOC_INT_1, 8, -1, -1, MSM8939_PNOC_SNOC_MAS); +DEFINE_QNODE(pcnoc_m_0, MSM8939_PNOC_MAS_0, 8, -1, -1, MSM8939_PNOC_INT_0); +DEFINE_QNODE(pcnoc_m_1, MSM8939_PNOC_MAS_1, 8, -1, -1, MSM8939_PNOC_SNOC_MAS); +DEFINE_QNODE(pcnoc_s_0, MSM8939_PNOC_SLV_0, 4, -1, -1, MSM8939_SLAVE_CLK_CTL, MSM8939_SLAVE_TLMM, MSM8939_SLAVE_TCSR, MSM8939_SLAVE_SECURITY, MSM8939_SLAVE_MSS); +DEFINE_QNODE(pcnoc_s_1, MSM8939_PNOC_SLV_1, 4, -1, -1, MSM8939_SLAVE_IMEM_CFG, MSM8939_SLAVE_CRYPTO_0_CFG, MSM8939_SLAVE_MSG_RAM, MSM8939_SLAVE_PDM, MSM8939_SLAVE_PRNG); +DEFINE_QNODE(pcnoc_s_2, MSM8939_PNOC_SLV_2, 4, -1, -1, MSM8939_SLAVE_SPDM, MSM8939_SLAVE_BOOT_ROM, MSM8939_SLAVE_BIMC_CFG, MSM8939_SLAVE_PNOC_CFG, MSM8939_SLAVE_PMIC_ARB); +DEFINE_QNODE(pcnoc_s_3, MSM8939_PNOC_SLV_3, 4, -1, -1, MSM8939_SLAVE_MPM, MSM8939_SLAVE_SNOC_CFG, MSM8939_SLAVE_RBCPR_CFG, MSM8939_SLAVE_QDSS_CFG, MSM8939_SLAVE_DEHR_CFG); +DEFINE_QNODE(pcnoc_s_4, MSM8939_PNOC_SLV_4, 4, -1, -1, MSM8939_SLAVE_VENUS_CFG, MSM8939_SLAVE_CAMERA_CFG, MSM8939_SLAVE_DISPLAY_CFG); +DEFINE_QNODE(pcnoc_s_8, MSM8939_PNOC_SLV_8, 4, -1, -1, MSM8939_SLAVE_USB_HS1, MSM8939_SLAVE_SDCC_1, MSM8939_SLAVE_BLSP_1); +DEFINE_QNODE(pcnoc_s_9, MSM8939_PNOC_SLV_9, 4, -1, -1, MSM8939_SLAVE_SDCC_2, MSM8939_SLAVE_LPASS, MSM8939_SLAVE_USB_HS2); +DEFINE_QNODE(pcnoc_snoc_mas, MSM8939_PNOC_SNOC_MAS, 8, 29, -1, MSM8939_PNOC_SNOC_SLV); +DEFINE_QNODE(pcnoc_snoc_slv, MSM8939_PNOC_SNOC_SLV, 8, -1, 45, MSM8939_SNOC_INT_0, MSM8939_SNOC_INT_BIMC, MSM8939_SNOC_INT_1); +DEFINE_QNODE(qdss_int, MSM8939_SNOC_QDSS_INT, 8, -1, -1, MSM8939_SNOC_INT_0, MSM8939_SNOC_INT_BIMC); +DEFINE_QNODE(slv_apps_l2, MSM8939_SLAVE_AMPSS_L2, 16, -1, -1, 0); +DEFINE_QNODE(slv_apss, MSM8939_SLAVE_APSS, 4, -1, 20, 0); +DEFINE_QNODE(slv_audio, MSM8939_SLAVE_LPASS, 4, -1, -1, 0); +DEFINE_QNODE(slv_bimc_cfg, MSM8939_SLAVE_BIMC_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_blsp_1, MSM8939_SLAVE_BLSP_1, 4, -1, -1, 0); +DEFINE_QNODE(slv_boot_rom, MSM8939_SLAVE_BOOT_ROM, 4, -1, -1, 0); +DEFINE_QNODE(slv_camera_cfg, MSM8939_SLAVE_CAMERA_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_cats_0, MSM8939_SLAVE_CATS_128, 16, -1, 106, 0); +DEFINE_QNODE(slv_cats_1, MSM8939_SLAVE_OCMEM_64, 8, -1, 107, 0); +DEFINE_QNODE(slv_clk_ctl, MSM8939_SLAVE_CLK_CTL, 4, -1, -1, 0); +DEFINE_QNODE(slv_crypto_0_cfg, MSM8939_SLAVE_CRYPTO_0_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_dehr_cfg, MSM8939_SLAVE_DEHR_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_display_cfg, MSM8939_SLAVE_DISPLAY_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_ebi_ch0, MSM8939_SLAVE_EBI_CH0, 16, -1, 0, 0); +DEFINE_QNODE(slv_gfx_cfg, MSM8939_SLAVE_GRAPHICS_3D_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_imem_cfg, MSM8939_SLAVE_IMEM_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_imem, MSM8939_SLAVE_IMEM, 8, -1, 26, 0); +DEFINE_QNODE(slv_mpm, MSM8939_SLAVE_MPM, 4, -1, -1, 0); +DEFINE_QNODE(slv_msg_ram, MSM8939_SLAVE_MSG_RAM, 4, -1, -1, 0); +DEFINE_QNODE(slv_mss, MSM8939_SLAVE_MSS, 4, -1, -1, 0); +DEFINE_QNODE(slv_pdm, MSM8939_SLAVE_PDM, 4, -1, -1, 0); +DEFINE_QNODE(slv_pmic_arb, MSM8939_SLAVE_PMIC_ARB, 4, -1, -1, 0); +DEFINE_QNODE(slv_pcnoc_cfg, MSM8939_SLAVE_PNOC_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_prng, MSM8939_SLAVE_PRNG, 4, -1, -1, 0); +DEFINE_QNODE(slv_qdss_cfg, MSM8939_SLAVE_QDSS_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_qdss_stm, MSM8939_SLAVE_QDSS_STM, 4, -1, 30, 0); +DEFINE_QNODE(slv_rbcpr_cfg, MSM8939_SLAVE_RBCPR_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_sdcc_1, MSM8939_SLAVE_SDCC_1, 4, -1, -1, 0); +DEFINE_QNODE(slv_sdcc_2, MSM8939_SLAVE_SDCC_2, 4, -1, -1, 0); +DEFINE_QNODE(slv_security, MSM8939_SLAVE_SECURITY, 4, -1, -1, 0); +DEFINE_QNODE(slv_snoc_cfg, MSM8939_SLAVE_SNOC_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_spdm, MSM8939_SLAVE_SPDM, 4, -1, -1, 0); +DEFINE_QNODE(slv_srvc_snoc, MSM8939_SLAVE_SRVC_SNOC, 8, -1, 29, 0); +DEFINE_QNODE(slv_tcsr, MSM8939_SLAVE_TCSR, 4, -1, -1, 0); +DEFINE_QNODE(slv_tlmm, MSM8939_SLAVE_TLMM, 4, -1, -1, 0); +DEFINE_QNODE(slv_usb_hs1, MSM8939_SLAVE_USB_HS1, 4, -1, -1, 0); +DEFINE_QNODE(slv_usb_hs2, MSM8939_SLAVE_USB_HS2, 4, -1, -1, 0); +DEFINE_QNODE(slv_venus_cfg, MSM8939_SLAVE_VENUS_CFG, 4, -1, -1, 0); +DEFINE_QNODE(snoc_bimc_0_mas, MSM8939_SNOC_BIMC_0_MAS, 16, 3, -1, MSM8939_SNOC_BIMC_0_SLV); +DEFINE_QNODE(snoc_bimc_0_slv, MSM8939_SNOC_BIMC_0_SLV, 16, -1, 24, MSM8939_SLAVE_EBI_CH0); +DEFINE_QNODE(snoc_bimc_1_mas, MSM8939_SNOC_BIMC_1_MAS, 16, 76, -1, MSM8939_SNOC_BIMC_1_SLV); +DEFINE_QNODE(snoc_bimc_1_slv, MSM8939_SNOC_BIMC_1_SLV, 16, -1, 104, MSM8939_SLAVE_EBI_CH0); +DEFINE_QNODE(snoc_bimc_2_mas, MSM8939_SNOC_BIMC_2_MAS, 16, -1, -1, MSM8939_SNOC_BIMC_2_SLV); +DEFINE_QNODE(snoc_bimc_2_slv, MSM8939_SNOC_BIMC_2_SLV, 16, -1, -1, MSM8939_SLAVE_EBI_CH0); +DEFINE_QNODE(snoc_int_0, MSM8939_SNOC_INT_0, 8, 99, 130, MSM8939_SLAVE_QDSS_STM, MSM8939_SLAVE_IMEM, MSM8939_SNOC_PNOC_MAS); +DEFINE_QNODE(snoc_int_1, MSM8939_SNOC_INT_1, 8, 100, 131, MSM8939_SLAVE_APSS, MSM8939_SLAVE_CATS_128, MSM8939_SLAVE_OCMEM_64); +DEFINE_QNODE(snoc_int_bimc, MSM8939_SNOC_INT_BIMC, 8, 101, 132, MSM8939_SNOC_BIMC_1_MAS); +DEFINE_QNODE(snoc_pcnoc_mas, MSM8939_SNOC_PNOC_MAS, 8, -1, -1, MSM8939_SNOC_PNOC_SLV); +DEFINE_QNODE(snoc_pcnoc_slv, MSM8939_SNOC_PNOC_SLV, 8, -1, -1, MSM8939_PNOC_INT_0); + +static struct qcom_icc_node *msm8939_snoc_nodes[] = { + [BIMC_SNOC_SLV] = &bimc_snoc_slv, + [MASTER_QDSS_BAM] = &mas_qdss_bam, + [MASTER_QDSS_ETR] = &mas_qdss_etr, + [MASTER_SNOC_CFG] = &mas_snoc_cfg, + [PCNOC_SNOC_SLV] = &pcnoc_snoc_slv, + [SLAVE_APSS] = &slv_apss, + [SLAVE_CATS_128] = &slv_cats_0, + [SLAVE_OCMEM_64] = &slv_cats_1, + [SLAVE_IMEM] = &slv_imem, + [SLAVE_QDSS_STM] = &slv_qdss_stm, + [SLAVE_SRVC_SNOC] = &slv_srvc_snoc, + [SNOC_BIMC_0_MAS] = &snoc_bimc_0_mas, + [SNOC_BIMC_1_MAS] = &snoc_bimc_1_mas, + [SNOC_BIMC_2_MAS] = &snoc_bimc_2_mas, + [SNOC_INT_0] = &snoc_int_0, + [SNOC_INT_1] = &snoc_int_1, + [SNOC_INT_BIMC] = &snoc_int_bimc, + [SNOC_PCNOC_MAS] = &snoc_pcnoc_mas, + [SNOC_QDSS_INT] = &qdss_int, +}; + +static struct qcom_icc_desc msm8939_snoc = { + .nodes = msm8939_snoc_nodes, + .num_nodes = ARRAY_SIZE(msm8939_snoc_nodes), +}; + +static struct qcom_icc_node *msm8939_snoc_mm_nodes[] = { + [MASTER_VIDEO_P0] = &mas_video, + [MASTER_JPEG] = &mas_jpeg, + [MASTER_VFE] = &mas_vfe, + [MASTER_MDP_PORT0] = &mas_mdp0, + [MASTER_MDP_PORT1] = &mas_mdp1, + [MASTER_CPP] = &mas_cpp, + [SNOC_MM_INT_0] = &mm_int_0, + [SNOC_MM_INT_1] = &mm_int_1, + [SNOC_MM_INT_2] = &mm_int_2, +}; + +static struct qcom_icc_desc msm8939_snoc_mm = { + .nodes = msm8939_snoc_mm_nodes, + .num_nodes = ARRAY_SIZE(msm8939_snoc_mm_nodes), +}; + +static struct qcom_icc_node *msm8939_bimc_nodes[] = { + [BIMC_SNOC_MAS] = &bimc_snoc_mas, + [MASTER_AMPSS_M0] = &mas_apss, + [MASTER_GRAPHICS_3D] = &mas_gfx, + [MASTER_TCU0] = &mas_tcu0, + [SLAVE_AMPSS_L2] = &slv_apps_l2, + [SLAVE_EBI_CH0] = &slv_ebi_ch0, + [SNOC_BIMC_0_SLV] = &snoc_bimc_0_slv, + [SNOC_BIMC_1_SLV] = &snoc_bimc_1_slv, + [SNOC_BIMC_2_SLV] = &snoc_bimc_2_slv, +}; + +static struct qcom_icc_desc msm8939_bimc = { + .nodes = msm8939_bimc_nodes, + .num_nodes = ARRAY_SIZE(msm8939_bimc_nodes), +}; + +static struct qcom_icc_node *msm8939_pcnoc_nodes[] = { + [MASTER_BLSP_1] = &mas_blsp_1, + [MASTER_DEHR] = &mas_dehr, + [MASTER_LPASS] = &mas_audio, + [MASTER_CRYPTO_CORE0] = &mas_pcnoc_crypto_0, + [MASTER_SDCC_1] = &mas_pcnoc_sdcc_1, + [MASTER_SDCC_2] = &mas_pcnoc_sdcc_2, + [MASTER_SPDM] = &mas_spdm, + [MASTER_USB_HS1] = &mas_usb_hs1, + [MASTER_USB_HS2] = &mas_usb_hs2, + [PCNOC_INT_0] = &pcnoc_int_0, + [PCNOC_INT_1] = &pcnoc_int_1, + [PCNOC_MAS_0] = &pcnoc_m_0, + [PCNOC_MAS_1] = &pcnoc_m_1, + [PCNOC_SLV_0] = &pcnoc_s_0, + [PCNOC_SLV_1] = &pcnoc_s_1, + [PCNOC_SLV_2] = &pcnoc_s_2, + [PCNOC_SLV_3] = &pcnoc_s_3, + [PCNOC_SLV_4] = &pcnoc_s_4, + [PCNOC_SLV_8] = &pcnoc_s_8, + [PCNOC_SLV_9] = &pcnoc_s_9, + [PCNOC_SNOC_MAS] = &pcnoc_snoc_mas, + [SLAVE_BIMC_CFG] = &slv_bimc_cfg, + [SLAVE_BLSP_1] = &slv_blsp_1, + [SLAVE_BOOT_ROM] = &slv_boot_rom, + [SLAVE_CAMERA_CFG] = &slv_camera_cfg, + [SLAVE_CLK_CTL] = &slv_clk_ctl, + [SLAVE_CRYPTO_0_CFG] = &slv_crypto_0_cfg, + [SLAVE_DEHR_CFG] = &slv_dehr_cfg, + [SLAVE_DISPLAY_CFG] = &slv_display_cfg, + [SLAVE_GRAPHICS_3D_CFG] = &slv_gfx_cfg, + [SLAVE_IMEM_CFG] = &slv_imem_cfg, + [SLAVE_LPASS] = &slv_audio, + [SLAVE_MPM] = &slv_mpm, + [SLAVE_MSG_RAM] = &slv_msg_ram, + [SLAVE_MSS] = &slv_mss, + [SLAVE_PDM] = &slv_pdm, + [SLAVE_PMIC_ARB] = &slv_pmic_arb, + [SLAVE_PCNOC_CFG] = &slv_pcnoc_cfg, + [SLAVE_PRNG] = &slv_prng, + [SLAVE_QDSS_CFG] = &slv_qdss_cfg, + [SLAVE_RBCPR_CFG] = &slv_rbcpr_cfg, + [SLAVE_SDCC_1] = &slv_sdcc_1, + [SLAVE_SDCC_2] = &slv_sdcc_2, + [SLAVE_SECURITY] = &slv_security, + [SLAVE_SNOC_CFG] = &slv_snoc_cfg, + [SLAVE_SPDM] = &slv_spdm, + [SLAVE_TCSR] = &slv_tcsr, + [SLAVE_TLMM] = &slv_tlmm, + [SLAVE_USB_HS1] = &slv_usb_hs1, + [SLAVE_USB_HS2] = &slv_usb_hs2, + [SLAVE_VENUS_CFG] = &slv_venus_cfg, + [SNOC_PCNOC_SLV] = &snoc_pcnoc_slv, +}; + +static struct qcom_icc_desc msm8939_pcnoc = { + .nodes = msm8939_pcnoc_nodes, + .num_nodes = ARRAY_SIZE(msm8939_pcnoc_nodes), +}; + +static int msm8939_qnoc_probe(struct platform_device *pdev) +{ + return qnoc_probe(pdev, sizeof(msm8939_bus_clocks), + ARRAY_SIZE(msm8939_bus_clocks), msm8939_bus_clocks); +} + +static const struct of_device_id msm8939_noc_of_match[] = { + { .compatible = "qcom,msm8939-bimc", .data = &msm8939_bimc }, + { .compatible = "qcom,msm8939-pcnoc", .data = &msm8939_pcnoc }, + { .compatible = "qcom,msm8939-snoc", .data = &msm8939_snoc }, + { .compatible = "qcom,msm8939-snoc-mm", .data = &msm8939_snoc_mm }, + { } +}; +MODULE_DEVICE_TABLE(of, msm8939_noc_of_match); + +static struct platform_driver msm8939_noc_driver = { + .probe = msm8939_qnoc_probe, + .remove = qnoc_remove, + .driver = { + .name = "qnoc-msm8939", + .of_match_table = msm8939_noc_of_match, + }, +}; +module_platform_driver(msm8939_noc_driver); +MODULE_AUTHOR("Jun Nie <jun.nie@linaro.org>"); +MODULE_DESCRIPTION("Qualcomm MSM8939 NoC driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/interconnect/qcom/qcs404.c b/drivers/interconnect/qcom/qcs404.c index 9820709b43db..36a7e30a00be 100644 --- a/drivers/interconnect/qcom/qcs404.c +++ b/drivers/interconnect/qcom/qcs404.c @@ -9,15 +9,12 @@ #include <linux/interconnect-provider.h> #include <linux/io.h> #include <linux/module.h> -#include <linux/of_device.h> -#include <linux/of_platform.h> #include <linux/platform_device.h> -#include <linux/slab.h> +#include <linux/of_device.h> -#include "smd-rpm.h" -#define RPM_BUS_MASTER_REQ 0x73616d62 -#define RPM_BUS_SLAVE_REQ 0x766c7362 +#include "smd-rpm.h" +#include "icc-rpm.h" enum { QCS404_MASTER_AMPSS_M0 = 1, @@ -95,67 +92,11 @@ enum { QCS404_SLAVE_LPASS, }; -#define to_qcom_provider(_provider) \ - container_of(_provider, struct qcom_icc_provider, provider) - -static const struct clk_bulk_data bus_clocks[] = { +static const struct clk_bulk_data qcs404_bus_clocks[] = { { .id = "bus" }, { .id = "bus_a" }, }; -/** - * struct qcom_icc_provider - Qualcomm specific interconnect provider - * @provider: generic interconnect provider - * @bus_clks: the clk_bulk_data table of bus clocks - * @num_clks: the total number of clk_bulk_data entries - */ -struct qcom_icc_provider { - struct icc_provider provider; - struct clk_bulk_data *bus_clks; - int num_clks; -}; - -#define QCS404_MAX_LINKS 12 - -/** - * struct qcom_icc_node - Qualcomm specific interconnect nodes - * @name: the node name used in debugfs - * @id: a unique node identifier - * @links: an array of nodes where we can go next while traversing - * @num_links: the total number of @links - * @buswidth: width of the interconnect between a node and the bus (bytes) - * @mas_rpm_id: RPM id for devices that are bus masters - * @slv_rpm_id: RPM id for devices that are bus slaves - * @rate: current bus clock rate in Hz - */ -struct qcom_icc_node { - unsigned char *name; - u16 id; - u16 links[QCS404_MAX_LINKS]; - u16 num_links; - u16 buswidth; - int mas_rpm_id; - int slv_rpm_id; - u64 rate; -}; - -struct qcom_icc_desc { - struct qcom_icc_node **nodes; - size_t num_nodes; -}; - -#define DEFINE_QNODE(_name, _id, _buswidth, _mas_rpm_id, _slv_rpm_id, \ - ...) \ - static struct qcom_icc_node _name = { \ - .name = #_name, \ - .id = _id, \ - .buswidth = _buswidth, \ - .mas_rpm_id = _mas_rpm_id, \ - .slv_rpm_id = _slv_rpm_id, \ - .num_links = ARRAY_SIZE(((int[]){ __VA_ARGS__ })), \ - .links = { __VA_ARGS__ }, \ - } - DEFINE_QNODE(mas_apps_proc, QCS404_MASTER_AMPSS_M0, 8, 0, -1, QCS404_SLAVE_EBI_CH0, QCS404_BIMC_SNOC_SLV); DEFINE_QNODE(mas_oxili, QCS404_MASTER_GRAPHICS_3D, 8, -1, -1, QCS404_SLAVE_EBI_CH0, QCS404_BIMC_SNOC_SLV); DEFINE_QNODE(mas_mdp, QCS404_MASTER_MDP_PORT0, 8, -1, -1, QCS404_SLAVE_EBI_CH0, QCS404_BIMC_SNOC_SLV); @@ -327,178 +268,11 @@ static struct qcom_icc_desc qcs404_snoc = { .num_nodes = ARRAY_SIZE(qcs404_snoc_nodes), }; -static int qcom_icc_set(struct icc_node *src, struct icc_node *dst) -{ - struct qcom_icc_provider *qp; - struct qcom_icc_node *qn; - struct icc_provider *provider; - struct icc_node *n; - u64 sum_bw; - u64 max_peak_bw; - u64 rate; - u32 agg_avg = 0; - u32 agg_peak = 0; - int ret, i; - - qn = src->data; - provider = src->provider; - qp = to_qcom_provider(provider); - - list_for_each_entry(n, &provider->nodes, node_list) - provider->aggregate(n, 0, n->avg_bw, n->peak_bw, - &agg_avg, &agg_peak); - - sum_bw = icc_units_to_bps(agg_avg); - max_peak_bw = icc_units_to_bps(agg_peak); - - /* send bandwidth request message to the RPM processor */ - if (qn->mas_rpm_id != -1) { - ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, - RPM_BUS_MASTER_REQ, - qn->mas_rpm_id, - sum_bw); - if (ret) { - pr_err("qcom_icc_rpm_smd_send mas %d error %d\n", - qn->mas_rpm_id, ret); - return ret; - } - } - - if (qn->slv_rpm_id != -1) { - ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, - RPM_BUS_SLAVE_REQ, - qn->slv_rpm_id, - sum_bw); - if (ret) { - pr_err("qcom_icc_rpm_smd_send slv error %d\n", - ret); - return ret; - } - } - - rate = max(sum_bw, max_peak_bw); - - do_div(rate, qn->buswidth); - - if (qn->rate == rate) - return 0; - - for (i = 0; i < qp->num_clks; i++) { - ret = clk_set_rate(qp->bus_clks[i].clk, rate); - if (ret) { - pr_err("%s clk_set_rate error: %d\n", - qp->bus_clks[i].id, ret); - return ret; - } - } - - qn->rate = rate; - return 0; -} - -static int qnoc_probe(struct platform_device *pdev) +static int qcs404_qnoc_probe(struct platform_device *pdev) { - struct device *dev = &pdev->dev; - const struct qcom_icc_desc *desc; - struct icc_onecell_data *data; - struct icc_provider *provider; - struct qcom_icc_node **qnodes; - struct qcom_icc_provider *qp; - struct icc_node *node; - size_t num_nodes, i; - int ret; - - /* wait for the RPM proxy */ - if (!qcom_icc_rpm_smd_available()) - return -EPROBE_DEFER; - - desc = of_device_get_match_data(dev); - if (!desc) - return -EINVAL; - - qnodes = desc->nodes; - num_nodes = desc->num_nodes; - - qp = devm_kzalloc(dev, sizeof(*qp), GFP_KERNEL); - if (!qp) - return -ENOMEM; - - data = devm_kzalloc(dev, struct_size(data, nodes, num_nodes), - GFP_KERNEL); - if (!data) - return -ENOMEM; - - qp->bus_clks = devm_kmemdup(dev, bus_clocks, sizeof(bus_clocks), - GFP_KERNEL); - if (!qp->bus_clks) - return -ENOMEM; - - qp->num_clks = ARRAY_SIZE(bus_clocks); - ret = devm_clk_bulk_get(dev, qp->num_clks, qp->bus_clks); - if (ret) - return ret; - - ret = clk_bulk_prepare_enable(qp->num_clks, qp->bus_clks); - if (ret) - return ret; - - provider = &qp->provider; - INIT_LIST_HEAD(&provider->nodes); - provider->dev = dev; - provider->set = qcom_icc_set; - provider->aggregate = icc_std_aggregate; - provider->xlate = of_icc_xlate_onecell; - provider->data = data; - - ret = icc_provider_add(provider); - if (ret) { - dev_err(dev, "error adding interconnect provider: %d\n", ret); - clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - return ret; - } - - for (i = 0; i < num_nodes; i++) { - size_t j; - - node = icc_node_create(qnodes[i]->id); - if (IS_ERR(node)) { - ret = PTR_ERR(node); - goto err; - } - - node->name = qnodes[i]->name; - node->data = qnodes[i]; - icc_node_add(node, provider); - - dev_dbg(dev, "registered node %s\n", node->name); - - /* populate links */ - for (j = 0; j < qnodes[i]->num_links; j++) - icc_link_create(node, qnodes[i]->links[j]); - - data->nodes[i] = node; - } - data->num_nodes = num_nodes; - - platform_set_drvdata(pdev, qp); - - return 0; -err: - icc_nodes_remove(provider); - clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - icc_provider_del(provider); - - return ret; -} - -static int qnoc_remove(struct platform_device *pdev) -{ - struct qcom_icc_provider *qp = platform_get_drvdata(pdev); - - icc_nodes_remove(&qp->provider); - clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - return icc_provider_del(&qp->provider); + return qnoc_probe(pdev, sizeof(qcs404_bus_clocks), + ARRAY_SIZE(qcs404_bus_clocks), qcs404_bus_clocks); } static const struct of_device_id qcs404_noc_of_match[] = { @@ -510,7 +284,7 @@ static const struct of_device_id qcs404_noc_of_match[] = { MODULE_DEVICE_TABLE(of, qcs404_noc_of_match); static struct platform_driver qcs404_noc_driver = { - .probe = qnoc_probe, + .probe = qcs404_qnoc_probe, .remove = qnoc_remove, .driver = { .name = "qnoc-qcs404", diff --git a/drivers/interconnect/qcom/sdx55.c b/drivers/interconnect/qcom/sdx55.c new file mode 100644 index 000000000000..a5a122ee3d21 --- /dev/null +++ b/drivers/interconnect/qcom/sdx55.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Qualcomm SDX55 interconnect driver + * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> + * + * Copyright (c) 2021, Linaro Ltd. + * + */ + +#include <linux/device.h> +#include <linux/interconnect.h> +#include <linux/interconnect-provider.h> +#include <linux/module.h> +#include <linux/of_platform.h> +#include <dt-bindings/interconnect/qcom,sdx55.h> + +#include "bcm-voter.h" +#include "icc-rpmh.h" +#include "sdx55.h" + +DEFINE_QNODE(ipa_core_master, SDX55_MASTER_IPA_CORE, 1, 8, SDX55_SLAVE_IPA_CORE); +DEFINE_QNODE(llcc_mc, SDX55_MASTER_LLCC, 4, 4, SDX55_SLAVE_EBI_CH0); +DEFINE_QNODE(acm_tcu, SDX55_MASTER_TCU_0, 1, 8, SDX55_SLAVE_LLCC, SDX55_SLAVE_MEM_NOC_SNOC, SDX55_SLAVE_MEM_NOC_PCIE_SNOC); +DEFINE_QNODE(qnm_snoc_gc, SDX55_MASTER_SNOC_GC_MEM_NOC, 1, 8, SDX55_SLAVE_LLCC); +DEFINE_QNODE(xm_apps_rdwr, SDX55_MASTER_AMPSS_M0, 1, 16, SDX55_SLAVE_LLCC, SDX55_SLAVE_MEM_NOC_SNOC, SDX55_SLAVE_MEM_NOC_PCIE_SNOC); +DEFINE_QNODE(qhm_audio, SDX55_MASTER_AUDIO, 1, 4, SDX55_SLAVE_ANOC_SNOC); +DEFINE_QNODE(qhm_blsp1, SDX55_MASTER_BLSP_1, 1, 4, SDX55_SLAVE_ANOC_SNOC); +DEFINE_QNODE(qhm_qdss_bam, SDX55_MASTER_QDSS_BAM, 1, 4, SDX55_SLAVE_SNOC_CFG, SDX55_SLAVE_EMAC_CFG, SDX55_SLAVE_USB3, SDX55_SLAVE_TLMM, SDX55_SLAVE_SPMI_FETCHER, SDX55_SLAVE_QDSS_CFG, SDX55_SLAVE_PDM, SDX55_SLAVE_SNOC_MEM_NOC_GC, SDX55_SLAVE_TCSR, SDX55_SLAVE_CNOC_DDRSS, SDX55_SLAVE_SPMI_VGI_COEX, SDX55_SLAVE_QPIC, SDX55_SLAVE_OCIMEM, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_USB3_PHY_CFG, SDX55_SLAVE_AOP, SDX55_SLAVE_BLSP_1, SDX55_SLAVE_SDCC_1, SDX55_SLAVE_CNOC_MSS, SDX55_SLAVE_PCIE_PARF, SDX55_SLAVE_ECC_CFG, SDX55_SLAVE_AUDIO, SDX55_SLAVE_AOSS, SDX55_SLAVE_PRNG, SDX55_SLAVE_CRYPTO_0_CFG, SDX55_SLAVE_TCU, SDX55_SLAVE_CLK_CTL, SDX55_SLAVE_IMEM_CFG); +DEFINE_QNODE(qhm_qpic, SDX55_MASTER_QPIC, 1, 4, SDX55_SLAVE_AOSS, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_ANOC_SNOC, SDX55_SLAVE_AOP, SDX55_SLAVE_AUDIO); +DEFINE_QNODE(qhm_snoc_cfg, SDX55_MASTER_SNOC_CFG, 1, 4, SDX55_SLAVE_SERVICE_SNOC); +DEFINE_QNODE(qhm_spmi_fetcher1, SDX55_MASTER_SPMI_FETCHER, 1, 4, SDX55_SLAVE_AOSS, SDX55_SLAVE_ANOC_SNOC, SDX55_SLAVE_AOP); +DEFINE_QNODE(qnm_aggre_noc, SDX55_MASTER_ANOC_SNOC, 1, 8, SDX55_SLAVE_PCIE_0, SDX55_SLAVE_SNOC_CFG, SDX55_SLAVE_SDCC_1, SDX55_SLAVE_TLMM, SDX55_SLAVE_SPMI_FETCHER, SDX55_SLAVE_QDSS_CFG, SDX55_SLAVE_PDM, SDX55_SLAVE_SNOC_MEM_NOC_GC, SDX55_SLAVE_TCSR, SDX55_SLAVE_CNOC_DDRSS, SDX55_SLAVE_SPMI_VGI_COEX, SDX55_SLAVE_QDSS_STM, SDX55_SLAVE_QPIC, SDX55_SLAVE_OCIMEM, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_USB3_PHY_CFG, SDX55_SLAVE_AOP, SDX55_SLAVE_BLSP_1, SDX55_SLAVE_USB3, SDX55_SLAVE_CNOC_MSS, SDX55_SLAVE_PCIE_PARF, SDX55_SLAVE_ECC_CFG, SDX55_SLAVE_APPSS, SDX55_SLAVE_AUDIO, SDX55_SLAVE_AOSS, SDX55_SLAVE_PRNG, SDX55_SLAVE_CRYPTO_0_CFG, SDX55_SLAVE_TCU, SDX55_SLAVE_CLK_CTL, SDX55_SLAVE_IMEM_CFG); +DEFINE_QNODE(qnm_ipa, SDX55_MASTER_IPA, 1, 8, SDX55_SLAVE_SNOC_CFG, SDX55_SLAVE_EMAC_CFG, SDX55_SLAVE_USB3, SDX55_SLAVE_AOSS, SDX55_SLAVE_SPMI_FETCHER, SDX55_SLAVE_QDSS_CFG, SDX55_SLAVE_PDM, SDX55_SLAVE_SNOC_MEM_NOC_GC, SDX55_SLAVE_TCSR, SDX55_SLAVE_CNOC_DDRSS, SDX55_SLAVE_QDSS_STM, SDX55_SLAVE_QPIC, SDX55_SLAVE_OCIMEM, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_USB3_PHY_CFG, SDX55_SLAVE_AOP, SDX55_SLAVE_BLSP_1, SDX55_SLAVE_SDCC_1, SDX55_SLAVE_CNOC_MSS, SDX55_SLAVE_PCIE_PARF, SDX55_SLAVE_ECC_CFG, SDX55_SLAVE_AUDIO, SDX55_SLAVE_TLMM, SDX55_SLAVE_PRNG, SDX55_SLAVE_CRYPTO_0_CFG, SDX55_SLAVE_CLK_CTL, SDX55_SLAVE_IMEM_CFG); +DEFINE_QNODE(qnm_memnoc, SDX55_MASTER_MEM_NOC_SNOC, 1, 8, SDX55_SLAVE_SNOC_CFG, SDX55_SLAVE_EMAC_CFG, SDX55_SLAVE_USB3, SDX55_SLAVE_TLMM, SDX55_SLAVE_SPMI_FETCHER, SDX55_SLAVE_QDSS_CFG, SDX55_SLAVE_PDM, SDX55_SLAVE_TCSR, SDX55_SLAVE_CNOC_DDRSS, SDX55_SLAVE_SPMI_VGI_COEX, SDX55_SLAVE_QDSS_STM, SDX55_SLAVE_QPIC, SDX55_SLAVE_OCIMEM, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_USB3_PHY_CFG, SDX55_SLAVE_AOP, SDX55_SLAVE_BLSP_1, SDX55_SLAVE_SDCC_1, SDX55_SLAVE_CNOC_MSS, SDX55_SLAVE_PCIE_PARF, SDX55_SLAVE_ECC_CFG, SDX55_SLAVE_APPSS, SDX55_SLAVE_AUDIO, SDX55_SLAVE_AOSS, SDX55_SLAVE_PRNG, SDX55_SLAVE_CRYPTO_0_CFG, SDX55_SLAVE_TCU, SDX55_SLAVE_CLK_CTL, SDX55_SLAVE_IMEM_CFG); +DEFINE_QNODE(qnm_memnoc_pcie, SDX55_MASTER_MEM_NOC_PCIE_SNOC, 1, 8, SDX55_SLAVE_PCIE_0); +DEFINE_QNODE(qxm_crypto, SDX55_MASTER_CRYPTO_CORE_0, 1, 8, SDX55_SLAVE_AOSS, SDX55_SLAVE_ANOC_SNOC, SDX55_SLAVE_AOP); +DEFINE_QNODE(xm_emac, SDX55_MASTER_EMAC, 1, 8, SDX55_SLAVE_ANOC_SNOC); +DEFINE_QNODE(xm_ipa2pcie_slv, SDX55_MASTER_IPA_PCIE, 1, 8, SDX55_SLAVE_PCIE_0); +DEFINE_QNODE(xm_pcie, SDX55_MASTER_PCIE, 1, 8, SDX55_SLAVE_ANOC_SNOC); +DEFINE_QNODE(xm_qdss_etr, SDX55_MASTER_QDSS_ETR, 1, 8, SDX55_SLAVE_SNOC_CFG, SDX55_SLAVE_EMAC_CFG, SDX55_SLAVE_USB3, SDX55_SLAVE_AOSS, SDX55_SLAVE_SPMI_FETCHER, SDX55_SLAVE_QDSS_CFG, SDX55_SLAVE_PDM, SDX55_SLAVE_SNOC_MEM_NOC_GC, SDX55_SLAVE_TCSR, SDX55_SLAVE_CNOC_DDRSS, SDX55_SLAVE_SPMI_VGI_COEX, SDX55_SLAVE_QPIC, SDX55_SLAVE_OCIMEM, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_USB3_PHY_CFG, SDX55_SLAVE_AOP, SDX55_SLAVE_BLSP_1, SDX55_SLAVE_SDCC_1, SDX55_SLAVE_CNOC_MSS, SDX55_SLAVE_PCIE_PARF, SDX55_SLAVE_ECC_CFG, SDX55_SLAVE_AUDIO, SDX55_SLAVE_AOSS, SDX55_SLAVE_PRNG, SDX55_SLAVE_CRYPTO_0_CFG, SDX55_SLAVE_TCU, SDX55_SLAVE_CLK_CTL, SDX55_SLAVE_IMEM_CFG); +DEFINE_QNODE(xm_sdc1, SDX55_MASTER_SDCC_1, 1, 8, SDX55_SLAVE_AOSS, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_ANOC_SNOC, SDX55_SLAVE_AOP, SDX55_SLAVE_AUDIO); +DEFINE_QNODE(xm_usb3, SDX55_MASTER_USB3, 1, 8, SDX55_SLAVE_ANOC_SNOC); +DEFINE_QNODE(ipa_core_slave, SDX55_SLAVE_IPA_CORE, 1, 8); +DEFINE_QNODE(ebi, SDX55_SLAVE_EBI_CH0, 1, 4); +DEFINE_QNODE(qns_llcc, SDX55_SLAVE_LLCC, 1, 16, SDX55_SLAVE_EBI_CH0); +DEFINE_QNODE(qns_memnoc_snoc, SDX55_SLAVE_MEM_NOC_SNOC, 1, 8, SDX55_MASTER_MEM_NOC_SNOC); +DEFINE_QNODE(qns_sys_pcie, SDX55_SLAVE_MEM_NOC_PCIE_SNOC, 1, 8, SDX55_MASTER_MEM_NOC_PCIE_SNOC); +DEFINE_QNODE(qhs_aop, SDX55_SLAVE_AOP, 1, 4); +DEFINE_QNODE(qhs_aoss, SDX55_SLAVE_AOSS, 1, 4); +DEFINE_QNODE(qhs_apss, SDX55_SLAVE_APPSS, 1, 4); +DEFINE_QNODE(qhs_audio, SDX55_SLAVE_AUDIO, 1, 4); +DEFINE_QNODE(qhs_blsp1, SDX55_SLAVE_BLSP_1, 1, 4); +DEFINE_QNODE(qhs_clk_ctl, SDX55_SLAVE_CLK_CTL, 1, 4); +DEFINE_QNODE(qhs_crypto0_cfg, SDX55_SLAVE_CRYPTO_0_CFG, 1, 4); +DEFINE_QNODE(qhs_ddrss_cfg, SDX55_SLAVE_CNOC_DDRSS, 1, 4); +DEFINE_QNODE(qhs_ecc_cfg, SDX55_SLAVE_ECC_CFG, 1, 4); +DEFINE_QNODE(qhs_emac_cfg, SDX55_SLAVE_EMAC_CFG, 1, 4); +DEFINE_QNODE(qhs_imem_cfg, SDX55_SLAVE_IMEM_CFG, 1, 4); +DEFINE_QNODE(qhs_ipa, SDX55_SLAVE_IPA_CFG, 1, 4); +DEFINE_QNODE(qhs_mss_cfg, SDX55_SLAVE_CNOC_MSS, 1, 4); +DEFINE_QNODE(qhs_pcie_parf, SDX55_SLAVE_PCIE_PARF, 1, 4); +DEFINE_QNODE(qhs_pdm, SDX55_SLAVE_PDM, 1, 4); +DEFINE_QNODE(qhs_prng, SDX55_SLAVE_PRNG, 1, 4); +DEFINE_QNODE(qhs_qdss_cfg, SDX55_SLAVE_QDSS_CFG, 1, 4); +DEFINE_QNODE(qhs_qpic, SDX55_SLAVE_QPIC, 1, 4); +DEFINE_QNODE(qhs_sdc1, SDX55_SLAVE_SDCC_1, 1, 4); +DEFINE_QNODE(qhs_snoc_cfg, SDX55_SLAVE_SNOC_CFG, 1, 4, SDX55_MASTER_SNOC_CFG); +DEFINE_QNODE(qhs_spmi_fetcher, SDX55_SLAVE_SPMI_FETCHER, 1, 4); +DEFINE_QNODE(qhs_spmi_vgi_coex, SDX55_SLAVE_SPMI_VGI_COEX, 1, 4); +DEFINE_QNODE(qhs_tcsr, SDX55_SLAVE_TCSR, 1, 4); +DEFINE_QNODE(qhs_tlmm, SDX55_SLAVE_TLMM, 1, 4); +DEFINE_QNODE(qhs_usb3, SDX55_SLAVE_USB3, 1, 4); +DEFINE_QNODE(qhs_usb3_phy, SDX55_SLAVE_USB3_PHY_CFG, 1, 4); +DEFINE_QNODE(qns_aggre_noc, SDX55_SLAVE_ANOC_SNOC, 1, 8, SDX55_MASTER_ANOC_SNOC); +DEFINE_QNODE(qns_snoc_memnoc, SDX55_SLAVE_SNOC_MEM_NOC_GC, 1, 8, SDX55_MASTER_SNOC_GC_MEM_NOC); +DEFINE_QNODE(qxs_imem, SDX55_SLAVE_OCIMEM, 1, 8); +DEFINE_QNODE(srvc_snoc, SDX55_SLAVE_SERVICE_SNOC, 1, 4); +DEFINE_QNODE(xs_pcie, SDX55_SLAVE_PCIE_0, 1, 8); +DEFINE_QNODE(xs_qdss_stm, SDX55_SLAVE_QDSS_STM, 1, 4); +DEFINE_QNODE(xs_sys_tcu_cfg, SDX55_SLAVE_TCU, 1, 8); + +DEFINE_QBCM(bcm_mc0, "MC0", true, &ebi); +DEFINE_QBCM(bcm_sh0, "SH0", true, &qns_llcc); +DEFINE_QBCM(bcm_ce0, "CE0", false, &qxm_crypto); +DEFINE_QBCM(bcm_ip0, "IP0", false, &ipa_core_slave); +DEFINE_QBCM(bcm_pn0, "PN0", false, &qhm_snoc_cfg); +DEFINE_QBCM(bcm_sh3, "SH3", false, &xm_apps_rdwr); +DEFINE_QBCM(bcm_sh4, "SH4", false, &qns_memnoc_snoc, &qns_sys_pcie); +DEFINE_QBCM(bcm_sn0, "SN0", true, &qns_snoc_memnoc); +DEFINE_QBCM(bcm_sn1, "SN1", false, &qxs_imem); +DEFINE_QBCM(bcm_pn1, "PN1", false, &xm_sdc1); +DEFINE_QBCM(bcm_pn2, "PN2", false, &qhm_audio, &qhm_spmi_fetcher1); +DEFINE_QBCM(bcm_sn3, "SN3", false, &xs_qdss_stm); +DEFINE_QBCM(bcm_pn3, "PN3", false, &qhm_blsp1, &qhm_qpic); +DEFINE_QBCM(bcm_sn4, "SN4", false, &xs_sys_tcu_cfg); +DEFINE_QBCM(bcm_pn5, "PN5", false, &qxm_crypto); +DEFINE_QBCM(bcm_sn6, "SN6", false, &xs_pcie); +DEFINE_QBCM(bcm_sn7, "SN7", false, &qnm_aggre_noc, &xm_emac, &xm_emac, &xm_usb3, + &qns_aggre_noc); +DEFINE_QBCM(bcm_sn8, "SN8", false, &qhm_qdss_bam, &xm_qdss_etr); +DEFINE_QBCM(bcm_sn9, "SN9", false, &qnm_memnoc); +DEFINE_QBCM(bcm_sn10, "SN10", false, &qnm_memnoc_pcie); +DEFINE_QBCM(bcm_sn11, "SN11", false, &qnm_ipa, &xm_ipa2pcie_slv); + +static struct qcom_icc_bcm *mc_virt_bcms[] = { + &bcm_mc0, +}; + +static struct qcom_icc_node *mc_virt_nodes[] = { + [MASTER_LLCC] = &llcc_mc, + [SLAVE_EBI_CH0] = &ebi, +}; + +static const struct qcom_icc_desc sdx55_mc_virt = { + .nodes = mc_virt_nodes, + .num_nodes = ARRAY_SIZE(mc_virt_nodes), + .bcms = mc_virt_bcms, + .num_bcms = ARRAY_SIZE(mc_virt_bcms), +}; + +static struct qcom_icc_bcm *mem_noc_bcms[] = { + &bcm_sh0, + &bcm_sh3, + &bcm_sh4, +}; + +static struct qcom_icc_node *mem_noc_nodes[] = { + [MASTER_TCU_0] = &acm_tcu, + [MASTER_SNOC_GC_MEM_NOC] = &qnm_snoc_gc, + [MASTER_AMPSS_M0] = &xm_apps_rdwr, + [SLAVE_LLCC] = &qns_llcc, + [SLAVE_MEM_NOC_SNOC] = &qns_memnoc_snoc, + [SLAVE_MEM_NOC_PCIE_SNOC] = &qns_sys_pcie, +}; + +static const struct qcom_icc_desc sdx55_mem_noc = { + .nodes = mem_noc_nodes, + .num_nodes = ARRAY_SIZE(mem_noc_nodes), + .bcms = mem_noc_bcms, + .num_bcms = ARRAY_SIZE(mem_noc_bcms), +}; + +static struct qcom_icc_bcm *system_noc_bcms[] = { + &bcm_ce0, + &bcm_pn0, + &bcm_pn1, + &bcm_pn2, + &bcm_pn3, + &bcm_pn5, + &bcm_sn0, + &bcm_sn1, + &bcm_sn3, + &bcm_sn4, + &bcm_sn6, + &bcm_sn7, + &bcm_sn8, + &bcm_sn9, + &bcm_sn10, + &bcm_sn11, +}; + +static struct qcom_icc_node *system_noc_nodes[] = { + [MASTER_AUDIO] = &qhm_audio, + [MASTER_BLSP_1] = &qhm_blsp1, + [MASTER_QDSS_BAM] = &qhm_qdss_bam, + [MASTER_QPIC] = &qhm_qpic, + [MASTER_SNOC_CFG] = &qhm_snoc_cfg, + [MASTER_SPMI_FETCHER] = &qhm_spmi_fetcher1, + [MASTER_ANOC_SNOC] = &qnm_aggre_noc, + [MASTER_IPA] = &qnm_ipa, + [MASTER_MEM_NOC_SNOC] = &qnm_memnoc, + [MASTER_MEM_NOC_PCIE_SNOC] = &qnm_memnoc_pcie, + [MASTER_CRYPTO_CORE_0] = &qxm_crypto, + [MASTER_EMAC] = &xm_emac, + [MASTER_IPA_PCIE] = &xm_ipa2pcie_slv, + [MASTER_PCIE] = &xm_pcie, + [MASTER_QDSS_ETR] = &xm_qdss_etr, + [MASTER_SDCC_1] = &xm_sdc1, + [MASTER_USB3] = &xm_usb3, + [SLAVE_AOP] = &qhs_aop, + [SLAVE_AOSS] = &qhs_aoss, + [SLAVE_APPSS] = &qhs_apss, + [SLAVE_AUDIO] = &qhs_audio, + [SLAVE_BLSP_1] = &qhs_blsp1, + [SLAVE_CLK_CTL] = &qhs_clk_ctl, + [SLAVE_CRYPTO_0_CFG] = &qhs_crypto0_cfg, + [SLAVE_CNOC_DDRSS] = &qhs_ddrss_cfg, + [SLAVE_ECC_CFG] = &qhs_ecc_cfg, + [SLAVE_EMAC_CFG] = &qhs_emac_cfg, + [SLAVE_IMEM_CFG] = &qhs_imem_cfg, + [SLAVE_IPA_CFG] = &qhs_ipa, + [SLAVE_CNOC_MSS] = &qhs_mss_cfg, + [SLAVE_PCIE_PARF] = &qhs_pcie_parf, + [SLAVE_PDM] = &qhs_pdm, + [SLAVE_PRNG] = &qhs_prng, + [SLAVE_QDSS_CFG] = &qhs_qdss_cfg, + [SLAVE_QPIC] = &qhs_qpic, + [SLAVE_SDCC_1] = &qhs_sdc1, + [SLAVE_SNOC_CFG] = &qhs_snoc_cfg, + [SLAVE_SPMI_FETCHER] = &qhs_spmi_fetcher, + [SLAVE_SPMI_VGI_COEX] = &qhs_spmi_vgi_coex, + [SLAVE_TCSR] = &qhs_tcsr, + [SLAVE_TLMM] = &qhs_tlmm, + [SLAVE_USB3] = &qhs_usb3, + [SLAVE_USB3_PHY_CFG] = &qhs_usb3_phy, + [SLAVE_ANOC_SNOC] = &qns_aggre_noc, + [SLAVE_SNOC_MEM_NOC_GC] = &qns_snoc_memnoc, + [SLAVE_OCIMEM] = &qxs_imem, + [SLAVE_SERVICE_SNOC] = &srvc_snoc, + [SLAVE_PCIE_0] = &xs_pcie, + [SLAVE_QDSS_STM] = &xs_qdss_stm, + [SLAVE_TCU] = &xs_sys_tcu_cfg, +}; + +static const struct qcom_icc_desc sdx55_system_noc = { + .nodes = system_noc_nodes, + .num_nodes = ARRAY_SIZE(system_noc_nodes), + .bcms = system_noc_bcms, + .num_bcms = ARRAY_SIZE(system_noc_bcms), +}; + +static struct qcom_icc_bcm *ipa_virt_bcms[] = { + &bcm_ip0, +}; + +static struct qcom_icc_node *ipa_virt_nodes[] = { + [MASTER_IPA_CORE] = &ipa_core_master, + [SLAVE_IPA_CORE] = &ipa_core_slave, +}; + +static const struct qcom_icc_desc sdx55_ipa_virt = { + .nodes = ipa_virt_nodes, + .num_nodes = ARRAY_SIZE(ipa_virt_nodes), + .bcms = ipa_virt_bcms, + .num_bcms = ARRAY_SIZE(ipa_virt_bcms), +}; + +static int qnoc_probe(struct platform_device *pdev) +{ + const struct qcom_icc_desc *desc; + struct icc_onecell_data *data; + struct icc_provider *provider; + struct qcom_icc_node **qnodes; + struct qcom_icc_provider *qp; + struct icc_node *node; + size_t num_nodes, i; + int ret; + + desc = device_get_match_data(&pdev->dev); + if (!desc) + return -EINVAL; + + qnodes = desc->nodes; + num_nodes = desc->num_nodes; + + qp = devm_kzalloc(&pdev->dev, sizeof(*qp), GFP_KERNEL); + if (!qp) + return -ENOMEM; + + data = devm_kcalloc(&pdev->dev, num_nodes, sizeof(*node), GFP_KERNEL); + if (!data) + return -ENOMEM; + + provider = &qp->provider; + provider->dev = &pdev->dev; + provider->set = qcom_icc_set; + provider->pre_aggregate = qcom_icc_pre_aggregate; + provider->aggregate = qcom_icc_aggregate; + provider->xlate = of_icc_xlate_onecell; + INIT_LIST_HEAD(&provider->nodes); + provider->data = data; + + qp->dev = &pdev->dev; + qp->bcms = desc->bcms; + qp->num_bcms = desc->num_bcms; + + qp->voter = of_bcm_voter_get(qp->dev, NULL); + if (IS_ERR(qp->voter)) + return PTR_ERR(qp->voter); + + ret = icc_provider_add(provider); + if (ret) { + dev_err(&pdev->dev, "error adding interconnect provider\n"); + return ret; + } + + for (i = 0; i < qp->num_bcms; i++) + qcom_icc_bcm_init(qp->bcms[i], &pdev->dev); + + for (i = 0; i < num_nodes; i++) { + size_t j; + + if (!qnodes[i]) + continue; + + node = icc_node_create(qnodes[i]->id); + if (IS_ERR(node)) { + ret = PTR_ERR(node); + goto err; + } + + node->name = qnodes[i]->name; + node->data = qnodes[i]; + icc_node_add(node, provider); + + for (j = 0; j < qnodes[i]->num_links; j++) + icc_link_create(node, qnodes[i]->links[j]); + + data->nodes[i] = node; + } + data->num_nodes = num_nodes; + + platform_set_drvdata(pdev, qp); + + return 0; +err: + icc_nodes_remove(provider); + icc_provider_del(provider); + return ret; +} + +static int qnoc_remove(struct platform_device *pdev) +{ + struct qcom_icc_provider *qp = platform_get_drvdata(pdev); + + icc_nodes_remove(&qp->provider); + return icc_provider_del(&qp->provider); +} + +static const struct of_device_id qnoc_of_match[] = { + { .compatible = "qcom,sdx55-mc-virt", + .data = &sdx55_mc_virt}, + { .compatible = "qcom,sdx55-mem-noc", + .data = &sdx55_mem_noc}, + { .compatible = "qcom,sdx55-system-noc", + .data = &sdx55_system_noc}, + { .compatible = "qcom,sdx55-ipa-virt", + .data = &sdx55_ipa_virt}, + { } +}; +MODULE_DEVICE_TABLE(of, qnoc_of_match); + +static struct platform_driver qnoc_driver = { + .probe = qnoc_probe, + .remove = qnoc_remove, + .driver = { + .name = "qnoc-sdx55", + .of_match_table = qnoc_of_match, + .sync_state = icc_sync_state, + }, +}; +module_platform_driver(qnoc_driver); + +MODULE_DESCRIPTION("Qualcomm SDX55 NoC driver"); +MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/interconnect/qcom/sdx55.h b/drivers/interconnect/qcom/sdx55.h new file mode 100644 index 000000000000..deff8afe0631 --- /dev/null +++ b/drivers/interconnect/qcom/sdx55.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021, Linaro Ltd. + */ + +#ifndef __DRIVERS_INTERCONNECT_QCOM_SDX55_H +#define __DRIVERS_INTERCONNECT_QCOM_SDX55_H + +#define SDX55_MASTER_IPA_CORE 0 +#define SDX55_MASTER_LLCC 1 +#define SDX55_MASTER_TCU_0 2 +#define SDX55_MASTER_SNOC_GC_MEM_NOC 3 +#define SDX55_MASTER_AMPSS_M0 4 +#define SDX55_MASTER_AUDIO 5 +#define SDX55_MASTER_BLSP_1 6 +#define SDX55_MASTER_QDSS_BAM 7 +#define SDX55_MASTER_QPIC 8 +#define SDX55_MASTER_SNOC_CFG 9 +#define SDX55_MASTER_SPMI_FETCHER 10 +#define SDX55_MASTER_ANOC_SNOC 11 +#define SDX55_MASTER_IPA 12 +#define SDX55_MASTER_MEM_NOC_SNOC 13 +#define SDX55_MASTER_MEM_NOC_PCIE_SNOC 14 +#define SDX55_MASTER_CRYPTO_CORE_0 15 +#define SDX55_MASTER_EMAC 16 +#define SDX55_MASTER_IPA_PCIE 17 +#define SDX55_MASTER_PCIE 18 +#define SDX55_MASTER_QDSS_ETR 19 +#define SDX55_MASTER_SDCC_1 20 +#define SDX55_MASTER_USB3 21 +#define SDX55_SLAVE_IPA_CORE 22 +#define SDX55_SLAVE_EBI_CH0 23 +#define SDX55_SLAVE_LLCC 24 +#define SDX55_SLAVE_MEM_NOC_SNOC 25 +#define SDX55_SLAVE_MEM_NOC_PCIE_SNOC 26 +#define SDX55_SLAVE_ANOC_SNOC 27 +#define SDX55_SLAVE_SNOC_CFG 28 +#define SDX55_SLAVE_EMAC_CFG 29 +#define SDX55_SLAVE_USB3 30 +#define SDX55_SLAVE_TLMM 31 +#define SDX55_SLAVE_SPMI_FETCHER 32 +#define SDX55_SLAVE_QDSS_CFG 33 +#define SDX55_SLAVE_PDM 34 +#define SDX55_SLAVE_SNOC_MEM_NOC_GC 35 +#define SDX55_SLAVE_TCSR 36 +#define SDX55_SLAVE_CNOC_DDRSS 37 +#define SDX55_SLAVE_SPMI_VGI_COEX 38 +#define SDX55_SLAVE_QPIC 39 +#define SDX55_SLAVE_OCIMEM 40 +#define SDX55_SLAVE_IPA_CFG 41 +#define SDX55_SLAVE_USB3_PHY_CFG 42 +#define SDX55_SLAVE_AOP 43 +#define SDX55_SLAVE_BLSP_1 44 +#define SDX55_SLAVE_SDCC_1 45 +#define SDX55_SLAVE_CNOC_MSS 46 +#define SDX55_SLAVE_PCIE_PARF 47 +#define SDX55_SLAVE_ECC_CFG 48 +#define SDX55_SLAVE_AUDIO 49 +#define SDX55_SLAVE_AOSS 51 +#define SDX55_SLAVE_PRNG 52 +#define SDX55_SLAVE_CRYPTO_0_CFG 53 +#define SDX55_SLAVE_TCU 54 +#define SDX55_SLAVE_CLK_CTL 55 +#define SDX55_SLAVE_IMEM_CFG 56 +#define SDX55_SLAVE_SERVICE_SNOC 57 +#define SDX55_SLAVE_PCIE_0 58 +#define SDX55_SLAVE_QDSS_STM 59 +#define SDX55_SLAVE_APPSS 60 + +#endif diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index 1c4961e05c12..bb0ee5c9fde7 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -182,6 +182,10 @@ static bool increase_address_space(struct protection_domain *domain, bool ret = true; u64 *pte; + pte = (void *)get_zeroed_page(gfp); + if (!pte) + return false; + spin_lock_irqsave(&domain->lock, flags); if (address <= PM_LEVEL_SIZE(domain->iop.mode)) @@ -191,10 +195,6 @@ static bool increase_address_space(struct protection_domain *domain, if (WARN_ON_ONCE(domain->iop.mode == PAGE_MODE_6_LEVEL)) goto out; - pte = (void *)get_zeroed_page(gfp); - if (!pte) - goto out; - *pte = PM_LEVEL_PDE(domain->iop.mode, iommu_virt_to_phys(domain->iop.root)); domain->iop.root = pte; @@ -208,10 +208,12 @@ static bool increase_address_space(struct protection_domain *domain, */ amd_iommu_domain_set_pgtable(domain, pte, domain->iop.mode); + pte = NULL; ret = true; out: spin_unlock_irqrestore(&domain->lock, flags); + free_page((unsigned long)pte); return ret; } diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index f659395e7959..af765c813cc8 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -311,6 +311,11 @@ static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad) domain->ops->flush_iotlb_all(domain); } +static bool dev_is_untrusted(struct device *dev) +{ + return dev_is_pci(dev) && to_pci_dev(dev)->untrusted; +} + /** * iommu_dma_init_domain - Initialise a DMA mapping domain * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() @@ -365,8 +370,9 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, init_iova_domain(iovad, 1UL << order, base_pfn); - if (!cookie->fq_domain && !iommu_domain_get_attr(domain, - DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, &attr) && attr) { + if (!cookie->fq_domain && (!dev || !dev_is_untrusted(dev)) && + !iommu_domain_get_attr(domain, DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, &attr) && + attr) { if (init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all, iommu_dma_entry_dtor)) pr_warn("iova flush queue initialization failed\n"); @@ -508,11 +514,6 @@ static void __iommu_dma_unmap_swiotlb(struct device *dev, dma_addr_t dma_addr, iova_align(iovad, size), dir, attrs); } -static bool dev_is_untrusted(struct device *dev) -{ - return dev_is_pci(dev) && to_pci_dev(dev)->untrusted; -} - static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, size_t size, int prot, u64 dma_mask) { @@ -1187,34 +1188,6 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, return cpu_addr; } -#ifdef CONFIG_DMA_REMAP -static void *iommu_dma_alloc_noncoherent(struct device *dev, size_t size, - dma_addr_t *handle, enum dma_data_direction dir, gfp_t gfp) -{ - if (!gfpflags_allow_blocking(gfp)) { - struct page *page; - - page = dma_common_alloc_pages(dev, size, handle, dir, gfp); - if (!page) - return NULL; - return page_address(page); - } - - return iommu_dma_alloc_remap(dev, size, handle, gfp | __GFP_ZERO, - PAGE_KERNEL, 0); -} - -static void iommu_dma_free_noncoherent(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t handle, enum dma_data_direction dir) -{ - __iommu_dma_unmap(dev, handle, size); - __iommu_dma_free(dev, size, cpu_addr); -} -#else -#define iommu_dma_alloc_noncoherent NULL -#define iommu_dma_free_noncoherent NULL -#endif /* CONFIG_DMA_REMAP */ - static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs) @@ -1285,8 +1258,6 @@ static const struct dma_map_ops iommu_dma_ops = { .free = iommu_dma_free, .alloc_pages = dma_common_alloc_pages, .free_pages = dma_common_free_pages, - .alloc_noncoherent = iommu_dma_alloc_noncoherent, - .free_noncoherent = iommu_dma_free_noncoherent, .mmap = iommu_dma_mmap, .get_sgtable = iommu_dma_get_sgtable, .map_page = iommu_dma_map_page, diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index 97dfcffbf495..444c0bec221a 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -30,8 +30,8 @@ #define VCMD_VRSP_IP 0x1 #define VCMD_VRSP_SC(e) (((e) >> 1) & 0x3) #define VCMD_VRSP_SC_SUCCESS 0 -#define VCMD_VRSP_SC_NO_PASID_AVAIL 1 -#define VCMD_VRSP_SC_INVALID_PASID 1 +#define VCMD_VRSP_SC_NO_PASID_AVAIL 2 +#define VCMD_VRSP_SC_INVALID_PASID 2 #define VCMD_VRSP_RESULT_PASID(e) (((e) >> 8) & 0xfffff) #define VCMD_CMD_OPERAND(e) ((e) << 8) /* diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 4a3f095a1c26..97eb62f667d2 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -798,10 +798,70 @@ static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain, return SMMU_PFN_PHYS(pfn) + SMMU_OFFSET_IN_PAGE(iova); } +static struct tegra_smmu *tegra_smmu_find(struct device_node *np) +{ + struct platform_device *pdev; + struct tegra_mc *mc; + + pdev = of_find_device_by_node(np); + if (!pdev) + return NULL; + + mc = platform_get_drvdata(pdev); + if (!mc) + return NULL; + + return mc->smmu; +} + +static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev, + struct of_phandle_args *args) +{ + const struct iommu_ops *ops = smmu->iommu.ops; + int err; + + err = iommu_fwspec_init(dev, &dev->of_node->fwnode, ops); + if (err < 0) { + dev_err(dev, "failed to initialize fwspec: %d\n", err); + return err; + } + + err = ops->of_xlate(dev, args); + if (err < 0) { + dev_err(dev, "failed to parse SW group ID: %d\n", err); + iommu_fwspec_free(dev); + return err; + } + + return 0; +} + static struct iommu_device *tegra_smmu_probe_device(struct device *dev) { - struct tegra_smmu *smmu = dev_iommu_priv_get(dev); + struct device_node *np = dev->of_node; + struct tegra_smmu *smmu = NULL; + struct of_phandle_args args; + unsigned int index = 0; + int err; + + while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, + &args) == 0) { + smmu = tegra_smmu_find(args.np); + if (smmu) { + err = tegra_smmu_configure(smmu, dev, &args); + of_node_put(args.np); + if (err < 0) + return ERR_PTR(err); + + break; + } + + of_node_put(args.np); + index++; + } + + smmu = dev_iommu_priv_get(dev); if (!smmu) return ERR_PTR(-ENODEV); @@ -1028,6 +1088,16 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, if (!smmu) return ERR_PTR(-ENOMEM); + /* + * This is a bit of a hack. Ideally we'd want to simply return this + * value. However the IOMMU registration process will attempt to add + * all devices to the IOMMU when bus_set_iommu() is called. In order + * not to rely on global variables to track the IOMMU instance, we + * set it here so that it can be looked up from the .probe_device() + * callback via the IOMMU device's .drvdata field. + */ + mc->smmu = smmu; + size = BITS_TO_LONGS(soc->num_asids) * sizeof(long); smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL); diff --git a/drivers/ipack/ipack.c b/drivers/ipack/ipack.c index 9267a85fee18..7de9605cac4f 100644 --- a/drivers/ipack/ipack.c +++ b/drivers/ipack/ipack.c @@ -64,9 +64,6 @@ static int ipack_bus_probe(struct device *device) struct ipack_device *dev = to_ipack_dev(device); struct ipack_driver *drv = to_ipack_driver(device->driver); - if (!drv->ops->probe) - return -EINVAL; - return drv->ops->probe(dev); } @@ -75,10 +72,9 @@ static int ipack_bus_remove(struct device *device) struct ipack_device *dev = to_ipack_dev(device); struct ipack_driver *drv = to_ipack_driver(device->driver); - if (!drv->ops->remove) - return -EINVAL; + if (drv->ops->remove) + drv->ops->remove(dev); - drv->ops->remove(dev); return 0; } @@ -252,6 +248,9 @@ EXPORT_SYMBOL_GPL(ipack_bus_unregister); int ipack_driver_register(struct ipack_driver *edrv, struct module *owner, const char *name) { + if (!edrv->ops->probe) + return -EINVAL; + edrv->driver.owner = owner; edrv->driver.name = name; edrv->driver.bus = &ipack_bus_type; diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig index 6c1d8b69a465..b6742b4231bf 100644 --- a/drivers/leds/Kconfig +++ b/drivers/leds/Kconfig @@ -934,4 +934,7 @@ source "drivers/leds/flash/Kconfig" comment "LED Triggers" source "drivers/leds/trigger/Kconfig" +comment "LED Blink" +source "drivers/leds/blink/Kconfig" + endif # NEW_LEDS diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile index 156c0b4e60d9..2a698df9da57 100644 --- a/drivers/leds/Makefile +++ b/drivers/leds/Makefile @@ -108,3 +108,6 @@ obj-$(CONFIG_LEDS_CLASS_FLASH) += flash/ # LED Triggers obj-$(CONFIG_LEDS_TRIGGERS) += trigger/ + +# LED Blink +obj-$(CONFIG_LEDS_BLINK) += blink/ diff --git a/drivers/leds/blink/Kconfig b/drivers/leds/blink/Kconfig new file mode 100644 index 000000000000..265b53476a80 --- /dev/null +++ b/drivers/leds/blink/Kconfig @@ -0,0 +1,20 @@ +menuconfig LEDS_BLINK + bool "LED Blink support" + depends on LEDS_CLASS + help + This option enables blink support for the leds class. + If unsure, say Y. + +if LEDS_BLINK + +config LEDS_BLINK_LGM + tristate "LED support for Intel LGM SoC series" + depends on LEDS_CLASS + depends on MFD_SYSCON + depends on OF + help + Parallel to serial conversion, which is also called SSO controller, + can drive external shift register for LED outputs. + This enables LED support for Serial Shift Output controller(SSO). + +endif # LEDS_BLINK diff --git a/drivers/leds/blink/Makefile b/drivers/leds/blink/Makefile new file mode 100644 index 000000000000..2fa6c7b7b67e --- /dev/null +++ b/drivers/leds/blink/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_LEDS_BLINK_LGM) += leds-lgm-sso.o diff --git a/drivers/leds/blink/leds-lgm-sso.c b/drivers/leds/blink/leds-lgm-sso.c new file mode 100644 index 000000000000..7d5c9ca007d6 --- /dev/null +++ b/drivers/leds/blink/leds-lgm-sso.c @@ -0,0 +1,888 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Intel Lightning Mountain SoC LED Serial Shift Output Controller driver + * + * Copyright (c) 2020 Intel Corporation. + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/gpio.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/leds.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/property.h> +#include <linux/regmap.h> +#include <linux/sizes.h> +#include <linux/uaccess.h> + +#define SSO_DEV_NAME "lgm-sso" + +#define LED_BLINK_H8_0 0x0 +#define LED_BLINK_H8_1 0x4 +#define GET_FREQ_OFFSET(pin, src) (((pin) * 6) + ((src) * 2)) +#define GET_SRC_OFFSET(pinc) (((pin) * 6) + 4) + +#define DUTY_CYCLE(x) (0x8 + ((x) * 4)) +#define SSO_CON0 0x2B0 +#define SSO_CON0_RZFL BIT(26) +#define SSO_CON0_BLINK_R BIT(30) +#define SSO_CON0_SWU BIT(31) + +#define SSO_CON1 0x2B4 +#define SSO_CON1_FCDSC GENMASK(21, 20) /* Fixed Divider Shift Clock */ +#define SSO_CON1_FPID GENMASK(24, 23) +#define SSO_CON1_GPTD GENMASK(26, 25) +#define SSO_CON1_US GENMASK(31, 30) + +#define SSO_CPU 0x2B8 +#define SSO_CON2 0x2C4 +#define SSO_CON3 0x2C8 + +/* Driver MACRO */ +#define MAX_PIN_NUM_PER_BANK SZ_32 +#define MAX_GROUP_NUM SZ_4 +#define PINS_PER_GROUP SZ_8 +#define FPID_FREQ_RANK_MAX SZ_4 +#define SSO_LED_MAX_NUM SZ_32 +#define MAX_FREQ_RANK 10 +#define DEF_GPTC_CLK_RATE 200000000 +#define SSO_DEF_BRIGHTNESS LED_HALF +#define DATA_CLK_EDGE 0 /* 0-rising, 1-falling */ + +static const u32 freq_div_tbl[] = {4000, 2000, 1000, 800}; +static const int freq_tbl[] = {2, 4, 8, 10, 50000, 100000, 200000, 250000}; +static const int shift_clk_freq_tbl[] = {25000000, 12500000, 6250000, 3125000}; + +/* + * Update Source to update the SOUTs + * SW - Software has to update the SWU bit + * GPTC - General Purpose timer is used as clock source + * FPID - Divided FSC clock (FPID) is used as clock source + */ +enum { + US_SW = 0, + US_GPTC = 1, + US_FPID = 2 +}; + +enum { + MAX_FPID_FREQ_RANK = 5, /* 1 to 4 */ + MAX_GPTC_FREQ_RANK = 9, /* 5 to 8 */ + MAX_GPTC_HS_FREQ_RANK = 10, /* 9 to 10 */ +}; + +enum { + LED_GRP0_PIN_MAX = 24, + LED_GRP1_PIN_MAX = 29, + LED_GRP2_PIN_MAX = 32, +}; + +enum { + LED_GRP0_0_23, + LED_GRP1_24_28, + LED_GRP2_29_31, + LED_GROUP_MAX, +}; + +enum { + CLK_SRC_FPID = 0, + CLK_SRC_GPTC = 1, + CLK_SRC_GPTC_HS = 2, +}; + +struct sso_led_priv; + +struct sso_led_desc { + const char *name; + const char *default_trigger; + unsigned int brightness; + unsigned int blink_rate; + unsigned int retain_state_suspended:1; + unsigned int retain_state_shutdown:1; + unsigned int panic_indicator:1; + unsigned int hw_blink:1; + unsigned int hw_trig:1; + unsigned int blinking:1; + int freq_idx; + u32 pin; +}; + +struct sso_led { + struct list_head list; + struct led_classdev cdev; + struct gpio_desc *gpiod; + struct sso_led_desc desc; + struct sso_led_priv *priv; +}; + +struct sso_gpio { + struct gpio_chip chip; + int shift_clk_freq; + int edge; + int freq; + u32 pins; + u32 alloc_bitmap; +}; + +struct sso_led_priv { + struct regmap *mmap; + struct device *dev; + struct platform_device *pdev; + struct clk *gclk; + struct clk *fpid_clk; + u32 fpid_clkrate; + u32 gptc_clkrate; + u32 freq[MAX_FREQ_RANK]; + struct list_head led_list; + struct sso_gpio gpio; +}; + +static int sso_get_blink_rate_idx(struct sso_led_priv *priv, u32 rate) +{ + int i; + + for (i = 0; i < MAX_FREQ_RANK; i++) { + if (rate <= priv->freq[i]) + return i; + } + + return -1; +} + +static unsigned int sso_led_pin_to_group(u32 pin) +{ + if (pin < LED_GRP0_PIN_MAX) + return LED_GRP0_0_23; + else if (pin < LED_GRP1_PIN_MAX) + return LED_GRP1_24_28; + else + return LED_GRP2_29_31; +} + +static u32 sso_led_get_freq_src(int freq_idx) +{ + if (freq_idx < MAX_FPID_FREQ_RANK) + return CLK_SRC_FPID; + else if (freq_idx < MAX_GPTC_FREQ_RANK) + return CLK_SRC_GPTC; + else + return CLK_SRC_GPTC_HS; +} + +static u32 sso_led_pin_blink_off(u32 pin, unsigned int group) +{ + if (group == LED_GRP2_29_31) + return pin - LED_GRP1_PIN_MAX; + else if (group == LED_GRP1_24_28) + return pin - LED_GRP0_PIN_MAX; + else /* led 0 - 23 in led 32 location */ + return SSO_LED_MAX_NUM - LED_GRP1_PIN_MAX; +} + +static struct sso_led +*cdev_to_sso_led_data(struct led_classdev *led_cdev) +{ + return container_of(led_cdev, struct sso_led, cdev); +} + +static void sso_led_freq_set(struct sso_led_priv *priv, u32 pin, int freq_idx) +{ + u32 reg, off, freq_src, val_freq; + u32 low, high, val; + unsigned int group; + + if (!freq_idx) + return; + + group = sso_led_pin_to_group(pin); + freq_src = sso_led_get_freq_src(freq_idx); + off = sso_led_pin_blink_off(pin, group); + + if (group == LED_GRP0_0_23) + return; + else if (group == LED_GRP1_24_28) + reg = LED_BLINK_H8_0; + else + reg = LED_BLINK_H8_1; + + if (freq_src == CLK_SRC_FPID) + val_freq = freq_idx - 1; + else if (freq_src == CLK_SRC_GPTC) + val_freq = freq_idx - MAX_FPID_FREQ_RANK; + + /* set blink rate idx */ + if (freq_src != CLK_SRC_GPTC_HS) { + low = GET_FREQ_OFFSET(off, freq_src); + high = low + 2; + val = val_freq << high; + regmap_update_bits(priv->mmap, reg, GENMASK(high, low), val); + } + + /* select clock source */ + low = GET_SRC_OFFSET(off); + high = low + 2; + val = freq_src << high; + regmap_update_bits(priv->mmap, reg, GENMASK(high, low), val); +} + +static void sso_led_brightness_set(struct led_classdev *led_cdev, + enum led_brightness brightness) +{ + struct sso_led_priv *priv; + struct sso_led_desc *desc; + struct sso_led *led; + int val; + + led = cdev_to_sso_led_data(led_cdev); + priv = led->priv; + desc = &led->desc; + + desc->brightness = brightness; + regmap_write(priv->mmap, DUTY_CYCLE(desc->pin), brightness); + + if (brightness == LED_OFF) + val = 0; + else + val = 1; + + /* HW blink off */ + if (desc->hw_blink && !val && desc->blinking) { + desc->blinking = 0; + regmap_update_bits(priv->mmap, SSO_CON2, BIT(desc->pin), 0); + } else if (desc->hw_blink && val && !desc->blinking) { + desc->blinking = 1; + regmap_update_bits(priv->mmap, SSO_CON2, BIT(desc->pin), + 1 << desc->pin); + } + + if (!desc->hw_trig && led->gpiod) + gpiod_set_value(led->gpiod, val); +} + +static enum led_brightness sso_led_brightness_get(struct led_classdev *led_cdev) +{ + struct sso_led *led = cdev_to_sso_led_data(led_cdev); + + return (enum led_brightness)led->desc.brightness; +} + +static int +delay_to_freq_idx(struct sso_led *led, unsigned long *delay_on, + unsigned long *delay_off) +{ + struct sso_led_priv *priv = led->priv; + unsigned long delay; + int freq_idx; + u32 freq; + + if (!*delay_on && !*delay_off) { + *delay_on = *delay_off = (1000 / priv->freq[0]) / 2; + return 0; + } + + delay = *delay_on + *delay_off; + freq = 1000 / delay; + + freq_idx = sso_get_blink_rate_idx(priv, freq); + if (freq_idx == -1) + freq_idx = MAX_FREQ_RANK - 1; + + delay = 1000 / priv->freq[freq_idx]; + *delay_on = *delay_off = delay / 2; + + if (!*delay_on) + *delay_on = *delay_off = 1; + + return freq_idx; +} + +static int +sso_led_blink_set(struct led_classdev *led_cdev, unsigned long *delay_on, + unsigned long *delay_off) +{ + struct sso_led_priv *priv; + struct sso_led *led; + int freq_idx; + + led = cdev_to_sso_led_data(led_cdev); + priv = led->priv; + freq_idx = delay_to_freq_idx(led, delay_on, delay_off); + + sso_led_freq_set(priv, led->desc.pin, freq_idx); + regmap_update_bits(priv->mmap, SSO_CON2, BIT(led->desc.pin), + 1 << led->desc.pin); + led->desc.freq_idx = freq_idx; + led->desc.blink_rate = priv->freq[freq_idx]; + led->desc.blinking = 1; + + return 1; +} + +static void sso_led_hw_cfg(struct sso_led_priv *priv, struct sso_led *led) +{ + struct sso_led_desc *desc = &led->desc; + + /* set freq */ + if (desc->hw_blink) { + sso_led_freq_set(priv, desc->pin, desc->freq_idx); + regmap_update_bits(priv->mmap, SSO_CON2, BIT(desc->pin), + 1 << desc->pin); + } + + if (desc->hw_trig) + regmap_update_bits(priv->mmap, SSO_CON3, BIT(desc->pin), + 1 << desc->pin); + + /* set brightness */ + regmap_write(priv->mmap, DUTY_CYCLE(desc->pin), desc->brightness); + + /* enable output */ + if (!desc->hw_trig && desc->brightness) + gpiod_set_value(led->gpiod, 1); +} + +static int sso_create_led(struct sso_led_priv *priv, struct sso_led *led, + struct fwnode_handle *child) +{ + struct sso_led_desc *desc = &led->desc; + struct led_init_data init_data; + int err; + + init_data.fwnode = child; + init_data.devicename = SSO_DEV_NAME; + init_data.default_label = ":"; + + led->cdev.default_trigger = desc->default_trigger; + led->cdev.brightness_set = sso_led_brightness_set; + led->cdev.brightness_get = sso_led_brightness_get; + led->cdev.brightness = desc->brightness; + led->cdev.max_brightness = LED_FULL; + + if (desc->retain_state_shutdown) + led->cdev.flags |= LED_RETAIN_AT_SHUTDOWN; + if (desc->retain_state_suspended) + led->cdev.flags |= LED_CORE_SUSPENDRESUME; + if (desc->panic_indicator) + led->cdev.flags |= LED_PANIC_INDICATOR; + + if (desc->hw_blink) + led->cdev.blink_set = sso_led_blink_set; + + sso_led_hw_cfg(priv, led); + + err = devm_led_classdev_register_ext(priv->dev, &led->cdev, &init_data); + if (err) + return err; + + list_add(&led->list, &priv->led_list); + + return 0; +} + +static void sso_init_freq(struct sso_led_priv *priv) +{ + int i; + + priv->freq[0] = 0; + for (i = 1; i < MAX_FREQ_RANK; i++) { + if (i < MAX_FPID_FREQ_RANK) { + priv->freq[i] = priv->fpid_clkrate / freq_div_tbl[i - 1]; + } else if (i < MAX_GPTC_FREQ_RANK) { + priv->freq[i] = priv->gptc_clkrate / + freq_div_tbl[i - MAX_FPID_FREQ_RANK]; + } else if (i < MAX_GPTC_HS_FREQ_RANK) { + priv->freq[i] = priv->gptc_clkrate; + } + } +} + +static int sso_gpio_request(struct gpio_chip *chip, unsigned int offset) +{ + struct sso_led_priv *priv = gpiochip_get_data(chip); + + if (priv->gpio.alloc_bitmap & BIT(offset)) + return -EINVAL; + + priv->gpio.alloc_bitmap |= BIT(offset); + regmap_write(priv->mmap, DUTY_CYCLE(offset), 0xFF); + + return 0; +} + +static void sso_gpio_free(struct gpio_chip *chip, unsigned int offset) +{ + struct sso_led_priv *priv = gpiochip_get_data(chip); + + priv->gpio.alloc_bitmap &= ~BIT(offset); + regmap_write(priv->mmap, DUTY_CYCLE(offset), 0x0); +} + +static int sso_gpio_get_dir(struct gpio_chip *chip, unsigned int offset) +{ + return GPIOF_DIR_OUT; +} + +static int +sso_gpio_dir_out(struct gpio_chip *chip, unsigned int offset, int value) +{ + struct sso_led_priv *priv = gpiochip_get_data(chip); + bool bit = !!value; + + regmap_update_bits(priv->mmap, SSO_CPU, BIT(offset), bit << offset); + if (!priv->gpio.freq) + regmap_update_bits(priv->mmap, SSO_CON0, SSO_CON0_SWU, + SSO_CON0_SWU); + + return 0; +} + +static int sso_gpio_get(struct gpio_chip *chip, unsigned int offset) +{ + struct sso_led_priv *priv = gpiochip_get_data(chip); + u32 reg_val; + + regmap_read(priv->mmap, SSO_CPU, ®_val); + + return !!(reg_val & BIT(offset)); +} + +static void sso_gpio_set(struct gpio_chip *chip, unsigned int offset, int value) +{ + struct sso_led_priv *priv = gpiochip_get_data(chip); + + regmap_update_bits(priv->mmap, SSO_CPU, BIT(offset), value << offset); + if (!priv->gpio.freq) + regmap_update_bits(priv->mmap, SSO_CON0, SSO_CON0_SWU, + SSO_CON0_SWU); +} + +static int sso_gpio_gc_init(struct device *dev, struct sso_led_priv *priv) +{ + struct gpio_chip *gc = &priv->gpio.chip; + + gc->request = sso_gpio_request; + gc->free = sso_gpio_free; + gc->get_direction = sso_gpio_get_dir; + gc->direction_output = sso_gpio_dir_out; + gc->get = sso_gpio_get; + gc->set = sso_gpio_set; + + gc->label = "lgm-sso"; + gc->base = -1; + /* To exclude pins from control, use "gpio-reserved-ranges" */ + gc->ngpio = priv->gpio.pins; + gc->parent = dev; + gc->owner = THIS_MODULE; + gc->of_node = dev->of_node; + + return devm_gpiochip_add_data(dev, gc, priv); +} + +static int sso_gpio_get_freq_idx(int freq) +{ + int idx; + + for (idx = 0; idx < ARRAY_SIZE(freq_tbl); idx++) { + if (freq <= freq_tbl[idx]) + return idx; + } + + return -1; +} + +static void sso_register_shift_clk(struct sso_led_priv *priv) +{ + int idx, size = ARRAY_SIZE(shift_clk_freq_tbl); + u32 val = 0; + + for (idx = 0; idx < size; idx++) { + if (shift_clk_freq_tbl[idx] <= priv->gpio.shift_clk_freq) { + val = idx; + break; + } + } + + if (idx == size) + dev_warn(priv->dev, "%s: Invalid freq %d\n", + __func__, priv->gpio.shift_clk_freq); + + regmap_update_bits(priv->mmap, SSO_CON1, SSO_CON1_FCDSC, + FIELD_PREP(SSO_CON1_FCDSC, val)); +} + +static int sso_gpio_freq_set(struct sso_led_priv *priv) +{ + int freq_idx; + u32 val; + + freq_idx = sso_gpio_get_freq_idx(priv->gpio.freq); + if (freq_idx == -1) + freq_idx = ARRAY_SIZE(freq_tbl) - 1; + + val = freq_idx % FPID_FREQ_RANK_MAX; + + if (!priv->gpio.freq) { + regmap_update_bits(priv->mmap, SSO_CON0, SSO_CON0_BLINK_R, 0); + regmap_update_bits(priv->mmap, SSO_CON1, SSO_CON1_US, + FIELD_PREP(SSO_CON1_US, US_SW)); + } else if (freq_idx < FPID_FREQ_RANK_MAX) { + regmap_update_bits(priv->mmap, SSO_CON0, SSO_CON0_BLINK_R, + SSO_CON0_BLINK_R); + regmap_update_bits(priv->mmap, SSO_CON1, SSO_CON1_US, + FIELD_PREP(SSO_CON1_US, US_FPID)); + regmap_update_bits(priv->mmap, SSO_CON1, SSO_CON1_FPID, + FIELD_PREP(SSO_CON1_FPID, val)); + } else { + regmap_update_bits(priv->mmap, SSO_CON0, SSO_CON0_BLINK_R, + SSO_CON0_BLINK_R); + regmap_update_bits(priv->mmap, SSO_CON1, SSO_CON1_US, + FIELD_PREP(SSO_CON1_US, US_GPTC)); + regmap_update_bits(priv->mmap, SSO_CON1, SSO_CON1_GPTD, + FIELD_PREP(SSO_CON1_GPTD, val)); + } + + return 0; +} + +static int sso_gpio_hw_init(struct sso_led_priv *priv) +{ + u32 activate; + int i, err; + + /* Clear all duty cycles */ + for (i = 0; i < priv->gpio.pins; i++) { + err = regmap_write(priv->mmap, DUTY_CYCLE(i), 0); + if (err) + return err; + } + + /* 4 groups for total 32 pins */ + for (i = 1; i <= MAX_GROUP_NUM; i++) { + activate = !!(i * PINS_PER_GROUP <= priv->gpio.pins || + priv->gpio.pins > (i - 1) * PINS_PER_GROUP); + err = regmap_update_bits(priv->mmap, SSO_CON1, BIT(i - 1), + activate << (i - 1)); + if (err) + return err; + } + + /* NO HW directly controlled pin by default */ + err = regmap_write(priv->mmap, SSO_CON3, 0); + if (err) + return err; + + /* NO BLINK for all pins */ + err = regmap_write(priv->mmap, SSO_CON2, 0); + if (err) + return err; + + /* OUTPUT 0 by default */ + err = regmap_write(priv->mmap, SSO_CPU, 0); + if (err) + return err; + + /* update edge */ + err = regmap_update_bits(priv->mmap, SSO_CON0, SSO_CON0_RZFL, + FIELD_PREP(SSO_CON0_RZFL, priv->gpio.edge)); + if (err) + return err; + + /* Set GPIO update rate */ + sso_gpio_freq_set(priv); + + /* Register shift clock */ + sso_register_shift_clk(priv); + + return 0; +} + +static void sso_led_shutdown(struct sso_led *led) +{ + struct sso_led_priv *priv = led->priv; + + /* unregister led */ + devm_led_classdev_unregister(priv->dev, &led->cdev); + + /* clear HW control bit */ + if (led->desc.hw_trig) + regmap_update_bits(priv->mmap, SSO_CON3, BIT(led->desc.pin), 0); + + if (led->gpiod) + devm_gpiod_put(priv->dev, led->gpiod); + + led->priv = NULL; +} + +static int +__sso_led_dt_parse(struct sso_led_priv *priv, struct fwnode_handle *fw_ssoled) +{ + struct fwnode_handle *fwnode_child; + struct device *dev = priv->dev; + struct sso_led_desc *desc; + struct sso_led *led; + struct list_head *p; + const char *tmp; + u32 prop; + int ret; + + fwnode_for_each_child_node(fw_ssoled, fwnode_child) { + led = devm_kzalloc(dev, sizeof(*led), GFP_KERNEL); + if (!led) + return -ENOMEM; + + INIT_LIST_HEAD(&led->list); + led->priv = priv; + desc = &led->desc; + + led->gpiod = devm_fwnode_get_gpiod_from_child(dev, NULL, + fwnode_child, + GPIOD_ASIS, NULL); + if (IS_ERR(led->gpiod)) { + dev_err(dev, "led: get gpio fail!\n"); + goto __dt_err; + } + + fwnode_property_read_string(fwnode_child, + "linux,default-trigger", + &desc->default_trigger); + + if (fwnode_property_present(fwnode_child, + "retain-state-suspended")) + desc->retain_state_suspended = 1; + + if (fwnode_property_present(fwnode_child, + "retain-state-shutdown")) + desc->retain_state_shutdown = 1; + + if (fwnode_property_present(fwnode_child, "panic-indicator")) + desc->panic_indicator = 1; + + ret = fwnode_property_read_u32(fwnode_child, "reg", &prop); + if (ret != 0 || prop >= SSO_LED_MAX_NUM) { + dev_err(dev, "invalid LED pin:%u\n", prop); + goto __dt_err; + } + desc->pin = prop; + + if (fwnode_property_present(fwnode_child, "intel,sso-hw-blink")) + desc->hw_blink = 1; + + desc->hw_trig = fwnode_property_read_bool(fwnode_child, + "intel,sso-hw-trigger"); + if (desc->hw_trig) { + desc->default_trigger = NULL; + desc->retain_state_shutdown = 0; + desc->retain_state_suspended = 0; + desc->panic_indicator = 0; + desc->hw_blink = 0; + } + + if (fwnode_property_read_u32(fwnode_child, + "intel,sso-blink-rate-hz", &prop)) { + /* default first freq rate */ + desc->freq_idx = 0; + desc->blink_rate = priv->freq[desc->freq_idx]; + } else { + desc->freq_idx = sso_get_blink_rate_idx(priv, prop); + if (desc->freq_idx == -1) + desc->freq_idx = MAX_FREQ_RANK - 1; + + desc->blink_rate = priv->freq[desc->freq_idx]; + } + + if (!fwnode_property_read_string(fwnode_child, "default-state", &tmp)) { + if (!strcmp(tmp, "on")) + desc->brightness = LED_FULL; + } + + if (sso_create_led(priv, led, fwnode_child)) + goto __dt_err; + } + fwnode_handle_put(fw_ssoled); + + return 0; +__dt_err: + fwnode_handle_put(fw_ssoled); + /* unregister leds */ + list_for_each(p, &priv->led_list) { + led = list_entry(p, struct sso_led, list); + sso_led_shutdown(led); + } + + return -EINVAL; +} + +static int sso_led_dt_parse(struct sso_led_priv *priv) +{ + struct fwnode_handle *fwnode = dev_fwnode(priv->dev); + struct fwnode_handle *fw_ssoled; + struct device *dev = priv->dev; + int count; + int ret; + + count = device_get_child_node_count(dev); + if (!count) + return 0; + + fw_ssoled = fwnode_get_named_child_node(fwnode, "ssoled"); + if (fw_ssoled) { + ret = __sso_led_dt_parse(priv, fw_ssoled); + if (ret) + return ret; + } + + return 0; +} + +static int sso_probe_gpios(struct sso_led_priv *priv) +{ + struct device *dev = priv->dev; + int ret; + + if (device_property_read_u32(dev, "ngpios", &priv->gpio.pins)) + priv->gpio.pins = MAX_PIN_NUM_PER_BANK; + + if (priv->gpio.pins > MAX_PIN_NUM_PER_BANK) + return -EINVAL; + + if (device_property_read_u32(dev, "intel,sso-update-rate-hz", + &priv->gpio.freq)) + priv->gpio.freq = 0; + + priv->gpio.edge = DATA_CLK_EDGE; + priv->gpio.shift_clk_freq = -1; + + ret = sso_gpio_hw_init(priv); + if (ret) + return ret; + + return sso_gpio_gc_init(dev, priv); +} + +static void sso_clk_disable(void *data) +{ + struct sso_led_priv *priv = data; + + clk_disable_unprepare(priv->fpid_clk); + clk_disable_unprepare(priv->gclk); +} + +static int intel_sso_led_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct sso_led_priv *priv; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->pdev = pdev; + priv->dev = dev; + + /* gate clock */ + priv->gclk = devm_clk_get(dev, "sso"); + if (IS_ERR(priv->gclk)) { + dev_err(dev, "get sso gate clock failed!\n"); + return PTR_ERR(priv->gclk); + } + + ret = clk_prepare_enable(priv->gclk); + if (ret) { + dev_err(dev, "Failed to prepate/enable sso gate clock!\n"); + return ret; + } + + priv->fpid_clk = devm_clk_get(dev, "fpid"); + if (IS_ERR(priv->fpid_clk)) { + dev_err(dev, "Failed to get fpid clock!\n"); + return PTR_ERR(priv->fpid_clk); + } + + ret = clk_prepare_enable(priv->fpid_clk); + if (ret) { + dev_err(dev, "Failed to prepare/enable fpid clock!\n"); + return ret; + } + priv->fpid_clkrate = clk_get_rate(priv->fpid_clk); + + ret = devm_add_action_or_reset(dev, sso_clk_disable, priv); + if (ret) { + dev_err(dev, "Failed to devm_add_action_or_reset, %d\n", ret); + return ret; + } + + priv->mmap = syscon_node_to_regmap(dev->of_node); + if (IS_ERR(priv->mmap)) { + dev_err(dev, "Failed to map iomem!\n"); + return PTR_ERR(priv->mmap); + } + + ret = sso_probe_gpios(priv); + if (ret) { + regmap_exit(priv->mmap); + return ret; + } + + INIT_LIST_HEAD(&priv->led_list); + + platform_set_drvdata(pdev, priv); + sso_init_freq(priv); + + priv->gptc_clkrate = DEF_GPTC_CLK_RATE; + + ret = sso_led_dt_parse(priv); + if (ret) { + regmap_exit(priv->mmap); + return ret; + } + dev_info(priv->dev, "sso LED init success!\n"); + + return 0; +} + +static int intel_sso_led_remove(struct platform_device *pdev) +{ + struct sso_led_priv *priv; + struct list_head *pos, *n; + struct sso_led *led; + + priv = platform_get_drvdata(pdev); + + list_for_each_safe(pos, n, &priv->led_list) { + list_del(pos); + led = list_entry(pos, struct sso_led, list); + sso_led_shutdown(led); + } + + clk_disable_unprepare(priv->fpid_clk); + clk_disable_unprepare(priv->gclk); + regmap_exit(priv->mmap); + + return 0; +} + +static const struct of_device_id of_sso_led_match[] = { + { .compatible = "intel,lgm-ssoled" }, + {} +}; + +MODULE_DEVICE_TABLE(of, of_sso_led_match); + +static struct platform_driver intel_sso_led_driver = { + .probe = intel_sso_led_probe, + .remove = intel_sso_led_remove, + .driver = { + .name = "lgm-ssoled", + .of_match_table = of_match_ptr(of_sso_led_match), + }, +}; + +module_platform_driver(intel_sso_led_driver); + +MODULE_DESCRIPTION("Intel SSO LED/GPIO driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/leds/led-class.c b/drivers/leds/led-class.c index 131ca83f5fb3..2e495ff67856 100644 --- a/drivers/leds/led-class.c +++ b/drivers/leds/led-class.c @@ -145,8 +145,7 @@ static void led_remove_brightness_hw_changed(struct led_classdev *led_cdev) device_remove_file(led_cdev->dev, &dev_attr_brightness_hw_changed); } -void led_classdev_notify_brightness_hw_changed(struct led_classdev *led_cdev, - enum led_brightness brightness) +void led_classdev_notify_brightness_hw_changed(struct led_classdev *led_cdev, unsigned int brightness) { if (WARN_ON(!led_cdev->brightness_hw_changed_kn)) return; diff --git a/drivers/leds/led-core.c b/drivers/leds/led-core.c index c4e780bdb385..8eb8054ef9c6 100644 --- a/drivers/leds/led-core.c +++ b/drivers/leds/led-core.c @@ -39,8 +39,7 @@ const char * const led_colors[LED_COLOR_ID_MAX] = { }; EXPORT_SYMBOL_GPL(led_colors); -static int __led_set_brightness(struct led_classdev *led_cdev, - enum led_brightness value) +static int __led_set_brightness(struct led_classdev *led_cdev, unsigned int value) { if (!led_cdev->brightness_set) return -ENOTSUPP; @@ -50,8 +49,7 @@ static int __led_set_brightness(struct led_classdev *led_cdev, return 0; } -static int __led_set_brightness_blocking(struct led_classdev *led_cdev, - enum led_brightness value) +static int __led_set_brightness_blocking(struct led_classdev *led_cdev, unsigned int value) { if (!led_cdev->brightness_set_blocking) return -ENOTSUPP; @@ -240,8 +238,7 @@ void led_stop_software_blink(struct led_classdev *led_cdev) } EXPORT_SYMBOL_GPL(led_stop_software_blink); -void led_set_brightness(struct led_classdev *led_cdev, - enum led_brightness brightness) +void led_set_brightness(struct led_classdev *led_cdev, unsigned int brightness) { /* * If software blink is active, delay brightness setting @@ -253,7 +250,7 @@ void led_set_brightness(struct led_classdev *led_cdev, * work queue task to avoid problems in case we are called * from hard irq context. */ - if (brightness == LED_OFF) { + if (!brightness) { set_bit(LED_BLINK_DISABLE, &led_cdev->work_flags); schedule_work(&led_cdev->set_brightness_work); } else { @@ -268,8 +265,7 @@ void led_set_brightness(struct led_classdev *led_cdev, } EXPORT_SYMBOL_GPL(led_set_brightness); -void led_set_brightness_nopm(struct led_classdev *led_cdev, - enum led_brightness value) +void led_set_brightness_nopm(struct led_classdev *led_cdev, unsigned int value) { /* Use brightness_set op if available, it is guaranteed not to sleep */ if (!__led_set_brightness(led_cdev, value)) @@ -281,8 +277,7 @@ void led_set_brightness_nopm(struct led_classdev *led_cdev, } EXPORT_SYMBOL_GPL(led_set_brightness_nopm); -void led_set_brightness_nosleep(struct led_classdev *led_cdev, - enum led_brightness value) +void led_set_brightness_nosleep(struct led_classdev *led_cdev, unsigned int value) { led_cdev->brightness = min(value, led_cdev->max_brightness); @@ -293,8 +288,7 @@ void led_set_brightness_nosleep(struct led_classdev *led_cdev, } EXPORT_SYMBOL_GPL(led_set_brightness_nosleep); -int led_set_brightness_sync(struct led_classdev *led_cdev, - enum led_brightness value) +int led_set_brightness_sync(struct led_classdev *led_cdev, unsigned int value) { if (led_cdev->blink_delay_on || led_cdev->blink_delay_off) return -EBUSY; diff --git a/drivers/leds/leds-apu.c b/drivers/leds/leds-apu.c index 7fd557aceff6..c409b80c236d 100644 --- a/drivers/leds/leds-apu.c +++ b/drivers/leds/leds-apu.c @@ -83,6 +83,7 @@ static const struct apu_led_profile apu1_led_profile[] = { }; static const struct dmi_system_id apu_led_dmi_table[] __initconst = { + /* PC Engines APU with factory bios "SageBios_PCEngines_APU-45" */ { .ident = "apu", .matches = { @@ -90,6 +91,14 @@ static const struct dmi_system_id apu_led_dmi_table[] __initconst = { DMI_MATCH(DMI_PRODUCT_NAME, "APU") } }, + /* PC Engines APU with "Mainline" bios >= 4.6.8 */ + { + .ident = "apu", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "PC Engines"), + DMI_MATCH(DMI_PRODUCT_NAME, "apu1") + } + }, {} }; MODULE_DEVICE_TABLE(dmi, apu_led_dmi_table); @@ -173,7 +182,7 @@ static int __init apu_led_init(void) int err; if (!(dmi_match(DMI_SYS_VENDOR, "PC Engines") && - dmi_match(DMI_PRODUCT_NAME, "APU"))) { + (dmi_match(DMI_PRODUCT_NAME, "APU") || dmi_match(DMI_PRODUCT_NAME, "apu1")))) { pr_err("No PC Engines APUv1 board detected. For APUv2,3 support, enable CONFIG_PCENGINES_APU2\n"); return -ENODEV; } diff --git a/drivers/leds/leds-blinkm.c b/drivers/leds/leds-blinkm.c index e11fe1788242..b4e1fdff4186 100644 --- a/drivers/leds/leds-blinkm.c +++ b/drivers/leds/leds-blinkm.c @@ -192,13 +192,13 @@ static int store_color_common(struct device *dev, const char *buf, int color) return 0; } -static ssize_t show_red(struct device *dev, struct device_attribute *attr, +static ssize_t red_show(struct device *dev, struct device_attribute *attr, char *buf) { return show_color_common(dev, buf, RED); } -static ssize_t store_red(struct device *dev, struct device_attribute *attr, +static ssize_t red_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { int ret; @@ -209,15 +209,15 @@ static ssize_t store_red(struct device *dev, struct device_attribute *attr, return count; } -static DEVICE_ATTR(red, S_IRUGO | S_IWUSR, show_red, store_red); +static DEVICE_ATTR_RW(red); -static ssize_t show_green(struct device *dev, struct device_attribute *attr, +static ssize_t green_show(struct device *dev, struct device_attribute *attr, char *buf) { return show_color_common(dev, buf, GREEN); } -static ssize_t store_green(struct device *dev, struct device_attribute *attr, +static ssize_t green_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { @@ -229,15 +229,15 @@ static ssize_t store_green(struct device *dev, struct device_attribute *attr, return count; } -static DEVICE_ATTR(green, S_IRUGO | S_IWUSR, show_green, store_green); +static DEVICE_ATTR_RW(green); -static ssize_t show_blue(struct device *dev, struct device_attribute *attr, +static ssize_t blue_show(struct device *dev, struct device_attribute *attr, char *buf) { return show_color_common(dev, buf, BLUE); } -static ssize_t store_blue(struct device *dev, struct device_attribute *attr, +static ssize_t blue_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { int ret; @@ -248,16 +248,16 @@ static ssize_t store_blue(struct device *dev, struct device_attribute *attr, return count; } -static DEVICE_ATTR(blue, S_IRUGO | S_IWUSR, show_blue, store_blue); +static DEVICE_ATTR_RW(blue); -static ssize_t show_test(struct device *dev, struct device_attribute *attr, +static ssize_t test_show(struct device *dev, struct device_attribute *attr, char *buf) { return scnprintf(buf, PAGE_SIZE, "#Write into test to start test sequence!#\n"); } -static ssize_t store_test(struct device *dev, struct device_attribute *attr, +static ssize_t test_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { @@ -273,7 +273,7 @@ static ssize_t store_test(struct device *dev, struct device_attribute *attr, return count; } -static DEVICE_ATTR(test, S_IRUGO | S_IWUSR, show_test, store_test); +static DEVICE_ATTR_RW(test); /* TODO: HSB, fade, timeadj, script ... */ diff --git a/drivers/leds/leds-gpio.c b/drivers/leds/leds-gpio.c index 93f5b1b60fde..b5d5e22d2d1e 100644 --- a/drivers/leds/leds-gpio.c +++ b/drivers/leds/leds-gpio.c @@ -96,7 +96,8 @@ static int create_gpio_led(const struct gpio_led *template, } else { state = (template->default_state == LEDS_GPIO_DEFSTATE_ON); } - led_dat->cdev.brightness = state ? LED_FULL : LED_OFF; + led_dat->cdev.brightness = state; + led_dat->cdev.max_brightness = 1; if (!template->retain_state_suspended) led_dat->cdev.flags |= LED_CORE_SUSPENDRESUME; if (template->panic_indicator) diff --git a/drivers/leds/leds-lm3530.c b/drivers/leds/leds-lm3530.c index 2f8362f6bf75..2db455efd4b1 100644 --- a/drivers/leds/leds-lm3530.c +++ b/drivers/leds/leds-lm3530.c @@ -346,8 +346,8 @@ static void lm3530_brightness_set(struct led_classdev *led_cdev, } } -static ssize_t lm3530_mode_get(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t mode_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct led_classdev *led_cdev = dev_get_drvdata(dev); struct lm3530_data *drvdata; @@ -365,8 +365,8 @@ static ssize_t lm3530_mode_get(struct device *dev, return len; } -static ssize_t lm3530_mode_set(struct device *dev, struct device_attribute - *attr, const char *buf, size_t size) +static ssize_t mode_store(struct device *dev, struct device_attribute + *attr, const char *buf, size_t size) { struct led_classdev *led_cdev = dev_get_drvdata(dev); struct lm3530_data *drvdata; @@ -397,7 +397,7 @@ static ssize_t lm3530_mode_set(struct device *dev, struct device_attribute return sizeof(drvdata->mode); } -static DEVICE_ATTR(mode, 0644, lm3530_mode_get, lm3530_mode_set); +static DEVICE_ATTR_RW(mode); static struct attribute *lm3530_attrs[] = { &dev_attr_mode.attr, diff --git a/drivers/leds/leds-lm3533.c b/drivers/leds/leds-lm3533.c index 9dd205870525..43d5970d96aa 100644 --- a/drivers/leds/leds-lm3533.c +++ b/drivers/leds/leds-lm3533.c @@ -608,7 +608,7 @@ static struct attribute *lm3533_led_attributes[] = { static umode_t lm3533_led_attr_is_visible(struct kobject *kobj, struct attribute *attr, int n) { - struct device *dev = container_of(kobj, struct device, kobj); + struct device *dev = kobj_to_dev(kobj); struct led_classdev *led_cdev = dev_get_drvdata(dev); struct lm3533_led *led = to_lm3533_led(led_cdev); umode_t mode = attr->mode; diff --git a/drivers/leds/leds-lm355x.c b/drivers/leds/leds-lm355x.c index 1505521249b5..2d3e11845ba5 100644 --- a/drivers/leds/leds-lm355x.c +++ b/drivers/leds/leds-lm355x.c @@ -349,9 +349,9 @@ static int lm355x_indicator_brightness_set(struct led_classdev *cdev, } /* indicator pattern only for lm3556*/ -static ssize_t lm3556_indicator_pattern_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t size) +static ssize_t pattern_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) { ssize_t ret; struct led_classdev *led_cdev = dev_get_drvdata(dev); @@ -381,7 +381,7 @@ out: return ret; } -static DEVICE_ATTR(pattern, S_IWUSR, NULL, lm3556_indicator_pattern_store); +static DEVICE_ATTR_WO(pattern); static struct attribute *lm355x_indicator_attrs[] = { &dev_attr_pattern.attr, diff --git a/drivers/leds/leds-lm3642.c b/drivers/leds/leds-lm3642.c index 62c14872caf7..8007b82985a8 100644 --- a/drivers/leds/leds-lm3642.c +++ b/drivers/leds/leds-lm3642.c @@ -165,9 +165,9 @@ static int lm3642_control(struct lm3642_chip_data *chip, /* torch */ /* torch pin config for lm3642 */ -static ssize_t lm3642_torch_pin_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t size) +static ssize_t torch_pin_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) { ssize_t ret; struct led_classdev *led_cdev = dev_get_drvdata(dev); @@ -193,7 +193,7 @@ static ssize_t lm3642_torch_pin_store(struct device *dev, return size; } -static DEVICE_ATTR(torch_pin, S_IWUSR, NULL, lm3642_torch_pin_store); +static DEVICE_ATTR_WO(torch_pin); static int lm3642_torch_brightness_set(struct led_classdev *cdev, enum led_brightness brightness) @@ -212,9 +212,9 @@ static int lm3642_torch_brightness_set(struct led_classdev *cdev, /* flash */ /* strobe pin config for lm3642*/ -static ssize_t lm3642_strobe_pin_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t size) +static ssize_t strobe_pin_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) { ssize_t ret; struct led_classdev *led_cdev = dev_get_drvdata(dev); @@ -240,7 +240,7 @@ static ssize_t lm3642_strobe_pin_store(struct device *dev, return size; } -static DEVICE_ATTR(strobe_pin, S_IWUSR, NULL, lm3642_strobe_pin_store); +static DEVICE_ATTR_WO(strobe_pin); static int lm3642_strobe_brightness_set(struct led_classdev *cdev, enum led_brightness brightness) diff --git a/drivers/leds/leds-lp50xx.c b/drivers/leds/leds-lp50xx.c index f13117eed976..06230614fdc5 100644 --- a/drivers/leds/leds-lp50xx.c +++ b/drivers/leds/leds-lp50xx.c @@ -6,10 +6,9 @@ #include <linux/i2c.h> #include <linux/init.h> #include <linux/leds.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/mutex.h> -#include <linux/of.h> -#include <linux/of_gpio.h> #include <linux/regmap.h> #include <linux/regulator/consumer.h> #include <linux/slab.h> @@ -322,7 +321,7 @@ static int lp50xx_brightness_set(struct led_classdev *cdev, ret = regmap_write(led->priv->regmap, reg_val, brightness); if (ret) { - dev_err(&led->priv->client->dev, + dev_err(led->priv->dev, "Cannot write brightness value %d\n", ret); goto out; } @@ -338,7 +337,7 @@ static int lp50xx_brightness_set(struct led_classdev *cdev, ret = regmap_write(led->priv->regmap, reg_val, mc_dev->subled_info[i].intensity); if (ret) { - dev_err(&led->priv->client->dev, + dev_err(led->priv->dev, "Cannot write intensity value %d\n", ret); goto out; } @@ -360,8 +359,8 @@ static int lp50xx_set_banks(struct lp50xx *priv, u32 led_banks[]) bank_enable_mask |= (1 << led_banks[i]); } - led_config_lo = (u8)(bank_enable_mask & 0xff); - led_config_hi = (u8)(bank_enable_mask >> 8) & 0xff; + led_config_lo = bank_enable_mask; + led_config_hi = bank_enable_mask >> 8; ret = regmap_write(priv->regmap, LP50XX_LED_CFG0, led_config_lo); if (ret) @@ -382,11 +381,9 @@ static int lp50xx_enable_disable(struct lp50xx *priv, int enable_disable) { int ret; - if (priv->enable_gpio) { - ret = gpiod_direction_output(priv->enable_gpio, enable_disable); - if (ret) - return ret; - } + ret = gpiod_direction_output(priv->enable_gpio, enable_disable); + if (ret) + return ret; if (enable_disable) return regmap_write(priv->regmap, LP50XX_DEV_CFG0, LP50XX_CHIP_EN); @@ -404,7 +401,7 @@ static int lp50xx_probe_leds(struct fwnode_handle *child, struct lp50xx *priv, if (num_leds > 1) { if (num_leds > priv->chip_info->max_modules) { - dev_err(&priv->client->dev, "reg property is invalid\n"); + dev_err(priv->dev, "reg property is invalid\n"); return -EINVAL; } @@ -412,13 +409,13 @@ static int lp50xx_probe_leds(struct fwnode_handle *child, struct lp50xx *priv, ret = fwnode_property_read_u32_array(child, "reg", led_banks, num_leds); if (ret) { - dev_err(&priv->client->dev, "reg property is missing\n"); + dev_err(priv->dev, "reg property is missing\n"); return ret; } ret = lp50xx_set_banks(priv, led_banks); if (ret) { - dev_err(&priv->client->dev, "Cannot setup banked LEDs\n"); + dev_err(priv->dev, "Cannot setup banked LEDs\n"); return ret; } @@ -426,12 +423,12 @@ static int lp50xx_probe_leds(struct fwnode_handle *child, struct lp50xx *priv, } else { ret = fwnode_property_read_u32(child, "reg", &led_number); if (ret) { - dev_err(&priv->client->dev, "led reg property missing\n"); + dev_err(priv->dev, "led reg property missing\n"); return ret; } if (led_number > priv->chip_info->num_leds) { - dev_err(&priv->client->dev, "led-sources property is invalid\n"); + dev_err(priv->dev, "led-sources property is invalid\n"); return -EINVAL; } @@ -455,12 +452,9 @@ static int lp50xx_probe_dt(struct lp50xx *priv) int i = 0; priv->enable_gpio = devm_gpiod_get_optional(priv->dev, "enable", GPIOD_OUT_LOW); - if (IS_ERR(priv->enable_gpio)) { - ret = PTR_ERR(priv->enable_gpio); - dev_err(&priv->client->dev, "Failed to get enable gpio: %d\n", - ret); - return ret; - } + if (IS_ERR(priv->enable_gpio)) + return dev_err_probe(priv->dev, PTR_ERR(priv->enable_gpio), + "Failed to get enable GPIO\n"); priv->regulator = devm_regulator_get(priv->dev, "vled"); if (IS_ERR(priv->regulator)) @@ -470,7 +464,7 @@ static int lp50xx_probe_dt(struct lp50xx *priv) led = &priv->leds[i]; ret = fwnode_property_count_u32(child, "reg"); if (ret < 0) { - dev_err(&priv->client->dev, "reg property is invalid\n"); + dev_err(priv->dev, "reg property is invalid\n"); goto child_out; } @@ -510,12 +504,11 @@ static int lp50xx_probe_dt(struct lp50xx *priv) led_cdev = &led->mc_cdev.led_cdev; led_cdev->brightness_set_blocking = lp50xx_brightness_set; - ret = devm_led_classdev_multicolor_register_ext(&priv->client->dev, + ret = devm_led_classdev_multicolor_register_ext(priv->dev, &led->mc_cdev, &init_data); if (ret) { - dev_err(&priv->client->dev, "led register err: %d\n", - ret); + dev_err(priv->dev, "led register err: %d\n", ret); goto child_out; } i++; @@ -529,8 +522,7 @@ child_out: return ret; } -static int lp50xx_probe(struct i2c_client *client, - const struct i2c_device_id *id) +static int lp50xx_probe(struct i2c_client *client) { struct lp50xx *led; int count; @@ -550,7 +542,7 @@ static int lp50xx_probe(struct i2c_client *client, mutex_init(&led->lock); led->client = client; led->dev = &client->dev; - led->chip_info = &lp50xx_chip_info_tbl[id->driver_data]; + led->chip_info = device_get_match_data(&client->dev); i2c_set_clientdata(client, led); led->regmap = devm_regmap_init_i2c(client, led->chip_info->lp50xx_regmap_config); @@ -579,15 +571,14 @@ static int lp50xx_remove(struct i2c_client *client) ret = lp50xx_enable_disable(led, 0); if (ret) { - dev_err(&led->client->dev, "Failed to disable chip\n"); + dev_err(led->dev, "Failed to disable chip\n"); return ret; } if (led->regulator) { ret = regulator_disable(led->regulator); if (ret) - dev_err(&led->client->dev, - "Failed to disable regulator\n"); + dev_err(led->dev, "Failed to disable regulator\n"); } mutex_destroy(&led->lock); @@ -596,24 +587,24 @@ static int lp50xx_remove(struct i2c_client *client) } static const struct i2c_device_id lp50xx_id[] = { - { "lp5009", LP5009 }, - { "lp5012", LP5012 }, - { "lp5018", LP5018 }, - { "lp5024", LP5024 }, - { "lp5030", LP5030 }, - { "lp5036", LP5036 }, + { "lp5009", (kernel_ulong_t)&lp50xx_chip_info_tbl[LP5009] }, + { "lp5012", (kernel_ulong_t)&lp50xx_chip_info_tbl[LP5012] }, + { "lp5018", (kernel_ulong_t)&lp50xx_chip_info_tbl[LP5018] }, + { "lp5024", (kernel_ulong_t)&lp50xx_chip_info_tbl[LP5024] }, + { "lp5030", (kernel_ulong_t)&lp50xx_chip_info_tbl[LP5030] }, + { "lp5036", (kernel_ulong_t)&lp50xx_chip_info_tbl[LP5036] }, { } }; MODULE_DEVICE_TABLE(i2c, lp50xx_id); static const struct of_device_id of_lp50xx_leds_match[] = { - { .compatible = "ti,lp5009", .data = (void *)LP5009 }, - { .compatible = "ti,lp5012", .data = (void *)LP5012 }, - { .compatible = "ti,lp5018", .data = (void *)LP5018 }, - { .compatible = "ti,lp5024", .data = (void *)LP5024 }, - { .compatible = "ti,lp5030", .data = (void *)LP5030 }, - { .compatible = "ti,lp5036", .data = (void *)LP5036 }, - {}, + { .compatible = "ti,lp5009", .data = &lp50xx_chip_info_tbl[LP5009] }, + { .compatible = "ti,lp5012", .data = &lp50xx_chip_info_tbl[LP5012] }, + { .compatible = "ti,lp5018", .data = &lp50xx_chip_info_tbl[LP5018] }, + { .compatible = "ti,lp5024", .data = &lp50xx_chip_info_tbl[LP5024] }, + { .compatible = "ti,lp5030", .data = &lp50xx_chip_info_tbl[LP5030] }, + { .compatible = "ti,lp5036", .data = &lp50xx_chip_info_tbl[LP5036] }, + {} }; MODULE_DEVICE_TABLE(of, of_lp50xx_leds_match); @@ -622,7 +613,7 @@ static struct i2c_driver lp50xx_driver = { .name = "lp50xx", .of_match_table = of_lp50xx_leds_match, }, - .probe = lp50xx_probe, + .probe_new = lp50xx_probe, .remove = lp50xx_remove, .id_table = lp50xx_id, }; diff --git a/drivers/leds/leds-max8997.c b/drivers/leds/leds-max8997.c index 512a11d142d0..c0bddb33888d 100644 --- a/drivers/leds/leds-max8997.c +++ b/drivers/leds/leds-max8997.c @@ -160,8 +160,8 @@ static void max8997_led_brightness_set(struct led_classdev *led_cdev, } } -static ssize_t max8997_led_show_mode(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t mode_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct led_classdev *led_cdev = dev_get_drvdata(dev); struct max8997_led *led = @@ -193,9 +193,9 @@ static ssize_t max8997_led_show_mode(struct device *dev, return ret; } -static ssize_t max8997_led_store_mode(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t size) +static ssize_t mode_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) { struct led_classdev *led_cdev = dev_get_drvdata(dev); struct max8997_led *led = @@ -222,7 +222,7 @@ static ssize_t max8997_led_store_mode(struct device *dev, return size; } -static DEVICE_ATTR(mode, 0644, max8997_led_show_mode, max8997_led_store_mode); +static DEVICE_ATTR_RW(mode); static struct attribute *max8997_attrs[] = { &dev_attr_mode.attr, diff --git a/drivers/leds/leds-netxbig.c b/drivers/leds/leds-netxbig.c index 68fbf0b66fad..77213b79f84d 100644 --- a/drivers/leds/leds-netxbig.c +++ b/drivers/leds/leds-netxbig.c @@ -204,9 +204,9 @@ static void netxbig_led_set(struct led_classdev *led_cdev, spin_unlock_irqrestore(&led_dat->lock, flags); } -static ssize_t netxbig_led_sata_store(struct device *dev, - struct device_attribute *attr, - const char *buff, size_t count) +static ssize_t sata_store(struct device *dev, + struct device_attribute *attr, + const char *buff, size_t count) { struct led_classdev *led_cdev = dev_get_drvdata(dev); struct netxbig_led_data *led_dat = @@ -255,8 +255,8 @@ exit_unlock: return ret; } -static ssize_t netxbig_led_sata_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t sata_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct led_classdev *led_cdev = dev_get_drvdata(dev); struct netxbig_led_data *led_dat = @@ -265,7 +265,7 @@ static ssize_t netxbig_led_sata_show(struct device *dev, return sprintf(buf, "%d\n", led_dat->sata); } -static DEVICE_ATTR(sata, 0644, netxbig_led_sata_show, netxbig_led_sata_store); +static DEVICE_ATTR_RW(sata); static struct attribute *netxbig_led_attrs[] = { &dev_attr_sata.attr, diff --git a/drivers/leds/leds-ss4200.c b/drivers/leds/leds-ss4200.c index 245de443fe9c..fcaa34706b6c 100644 --- a/drivers/leds/leds-ss4200.c +++ b/drivers/leds/leds-ss4200.c @@ -441,8 +441,8 @@ static void set_power_light_amber_noblink(void) nasgpio_led_set_brightness(&amber->led_cdev, LED_FULL); } -static ssize_t nas_led_blink_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t blink_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct led_classdev *led = dev_get_drvdata(dev); int blinking = 0; @@ -451,9 +451,9 @@ static ssize_t nas_led_blink_show(struct device *dev, return sprintf(buf, "%u\n", blinking); } -static ssize_t nas_led_blink_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t size) +static ssize_t blink_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) { int ret; struct led_classdev *led = dev_get_drvdata(dev); @@ -468,7 +468,7 @@ static ssize_t nas_led_blink_store(struct device *dev, return size; } -static DEVICE_ATTR(blink, 0644, nas_led_blink_show, nas_led_blink_store); +static DEVICE_ATTR_RW(blink); static struct attribute *nasgpio_led_attrs[] = { &dev_attr_blink.attr, @@ -478,7 +478,6 @@ ATTRIBUTE_GROUPS(nasgpio_led); static int register_nasgpio_led(int led_nr) { - int ret; struct nasgpio_led *nas_led = &nasgpio_leds[led_nr]; struct led_classdev *led = get_classdev_for_led_nr(led_nr); @@ -489,11 +488,8 @@ static int register_nasgpio_led(int led_nr) led->brightness_set = nasgpio_led_set_brightness; led->blink_set = nasgpio_led_set_blink; led->groups = nasgpio_led_groups; - ret = led_classdev_register(&nas_gpio_pci_dev->dev, led); - if (ret) - return ret; - return 0; + return led_classdev_register(&nas_gpio_pci_dev->dev, led); } static void unregister_nasgpio_led(int led_nr) diff --git a/drivers/leds/leds-wm831x-status.c b/drivers/leds/leds-wm831x-status.c index 67f4235cb28a..c48b80574f02 100644 --- a/drivers/leds/leds-wm831x-status.c +++ b/drivers/leds/leds-wm831x-status.c @@ -155,8 +155,8 @@ static const char * const led_src_texts[] = { "soft", }; -static ssize_t wm831x_status_src_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t src_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct led_classdev *led_cdev = dev_get_drvdata(dev); struct wm831x_status *led = to_wm831x_status(led_cdev); @@ -178,9 +178,9 @@ static ssize_t wm831x_status_src_show(struct device *dev, return ret; } -static ssize_t wm831x_status_src_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t size) +static ssize_t src_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) { struct led_classdev *led_cdev = dev_get_drvdata(dev); struct wm831x_status *led = to_wm831x_status(led_cdev); @@ -197,7 +197,7 @@ static ssize_t wm831x_status_src_store(struct device *dev, return size; } -static DEVICE_ATTR(src, 0644, wm831x_status_src_show, wm831x_status_src_store); +static DEVICE_ATTR_RW(src); static struct attribute *wm831x_status_attrs[] = { &dev_attr_src.attr, diff --git a/drivers/leds/leds.h b/drivers/leds/leds.h index 2d9eb48bbed9..345062ccabda 100644 --- a/drivers/leds/leds.h +++ b/drivers/leds/leds.h @@ -19,10 +19,8 @@ static inline int led_get_brightness(struct led_classdev *led_cdev) void led_init_core(struct led_classdev *led_cdev); void led_stop_software_blink(struct led_classdev *led_cdev); -void led_set_brightness_nopm(struct led_classdev *led_cdev, - enum led_brightness value); -void led_set_brightness_nosleep(struct led_classdev *led_cdev, - enum led_brightness value); +void led_set_brightness_nopm(struct led_classdev *led_cdev, unsigned int value); +void led_set_brightness_nosleep(struct led_classdev *led_cdev, unsigned int value); ssize_t led_trigger_read(struct file *filp, struct kobject *kobj, struct bin_attribute *attr, char *buf, loff_t pos, size_t count); diff --git a/drivers/mailbox/arm_mhuv2.c b/drivers/mailbox/arm_mhuv2.c index 6cf1991a5c9c..d997f8ebfa98 100644 --- a/drivers/mailbox/arm_mhuv2.c +++ b/drivers/mailbox/arm_mhuv2.c @@ -238,19 +238,19 @@ struct mhuv2_mbox_chan_priv { }; /* Macro for reading a bitfield within a physically mapped packed struct */ -#define readl_relaxed_bitfield(_regptr, _field) \ +#define readl_relaxed_bitfield(_regptr, _type, _field) \ ({ \ u32 _regval; \ _regval = readl_relaxed((_regptr)); \ - (*(typeof((_regptr)))(&_regval))._field; \ + (*(_type *)(&_regval))._field; \ }) /* Macro for writing a bitfield within a physically mapped packed struct */ -#define writel_relaxed_bitfield(_value, _regptr, _field) \ +#define writel_relaxed_bitfield(_value, _regptr, _type, _field) \ ({ \ u32 _regval; \ _regval = readl_relaxed(_regptr); \ - (*(typeof(_regptr))(&_regval))._field = _value; \ + (*(_type *)(&_regval))._field = _value; \ writel_relaxed(_regval, _regptr); \ }) @@ -496,7 +496,7 @@ static const struct mhuv2_protocol_ops mhuv2_data_transfer_ops = { /* Interrupt handlers */ -static struct mbox_chan *get_irq_chan_comb(struct mhuv2 *mhu, u32 *reg) +static struct mbox_chan *get_irq_chan_comb(struct mhuv2 *mhu, u32 __iomem *reg) { struct mbox_chan *chans = mhu->mbox.chans; int channel = 0, i, offset = 0, windows, protocol, ch_wn; @@ -699,7 +699,9 @@ static irqreturn_t mhuv2_receiver_interrupt(int irq, void *arg) ret = IRQ_HANDLED; } - kfree(data); + if (!IS_ERR(data)) + kfree(data); + return ret; } @@ -969,8 +971,8 @@ static int mhuv2_tx_init(struct amba_device *adev, struct mhuv2 *mhu, mhu->mbox.ops = &mhuv2_sender_ops; mhu->send = reg; - mhu->windows = readl_relaxed_bitfield(&mhu->send->mhu_cfg, num_ch); - mhu->minor = readl_relaxed_bitfield(&mhu->send->aidr, arch_minor_rev); + mhu->windows = readl_relaxed_bitfield(&mhu->send->mhu_cfg, struct mhu_cfg_t, num_ch); + mhu->minor = readl_relaxed_bitfield(&mhu->send->aidr, struct aidr_t, arch_minor_rev); spin_lock_init(&mhu->doorbell_pending_lock); @@ -990,7 +992,7 @@ static int mhuv2_tx_init(struct amba_device *adev, struct mhuv2 *mhu, mhu->mbox.txdone_poll = false; mhu->irq = adev->irq[0]; - writel_relaxed_bitfield(1, &mhu->send->int_en, chcomb); + writel_relaxed_bitfield(1, &mhu->send->int_en, struct int_en_t, chcomb); /* Disable all channel interrupts */ for (i = 0; i < mhu->windows; i++) @@ -1023,8 +1025,8 @@ static int mhuv2_rx_init(struct amba_device *adev, struct mhuv2 *mhu, mhu->mbox.ops = &mhuv2_receiver_ops; mhu->recv = reg; - mhu->windows = readl_relaxed_bitfield(&mhu->recv->mhu_cfg, num_ch); - mhu->minor = readl_relaxed_bitfield(&mhu->recv->aidr, arch_minor_rev); + mhu->windows = readl_relaxed_bitfield(&mhu->recv->mhu_cfg, struct mhu_cfg_t, num_ch); + mhu->minor = readl_relaxed_bitfield(&mhu->recv->aidr, struct aidr_t, arch_minor_rev); mhu->irq = adev->irq[0]; if (!mhu->irq) { @@ -1045,7 +1047,7 @@ static int mhuv2_rx_init(struct amba_device *adev, struct mhuv2 *mhu, writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[i].mask_set); if (mhu->minor) - writel_relaxed_bitfield(1, &mhu->recv->int_en, chcomb); + writel_relaxed_bitfield(1, &mhu->recv->int_en, struct int_en_t, chcomb); return 0; } diff --git a/drivers/mailbox/omap-mailbox.c b/drivers/mailbox/omap-mailbox.c index 93fe08aef3ca..7295e3835e30 100644 --- a/drivers/mailbox/omap-mailbox.c +++ b/drivers/mailbox/omap-mailbox.c @@ -3,7 +3,7 @@ * OMAP mailbox driver * * Copyright (C) 2006-2009 Nokia Corporation. All rights reserved. - * Copyright (C) 2013-2019 Texas Instruments Incorporated - https://www.ti.com + * Copyright (C) 2013-2021 Texas Instruments Incorporated - https://www.ti.com * * Contact: Hiroshi DOYU <Hiroshi.DOYU@nokia.com> * Suman Anna <s-anna@ti.com> @@ -664,6 +664,10 @@ static const struct of_device_id omap_mailbox_of_match[] = { .data = &omap4_data, }, { + .compatible = "ti,am64-mailbox", + .data = &omap4_data, + }, + { /* end */ }, }; diff --git a/drivers/mailbox/qcom-apcs-ipc-mailbox.c b/drivers/mailbox/qcom-apcs-ipc-mailbox.c index 077e5c6a9ef7..f25324d03842 100644 --- a/drivers/mailbox/qcom-apcs-ipc-mailbox.c +++ b/drivers/mailbox/qcom-apcs-ipc-mailbox.c @@ -61,11 +61,15 @@ static const struct qcom_apcs_ipc_data apps_shared_apcs_data = { .offset = 12, .clk_name = NULL }; +static const struct qcom_apcs_ipc_data sdx55_apcs_data = { + .offset = 0x1008, .clk_name = "qcom-sdx55-acps-clk" +}; + static const struct regmap_config apcs_regmap_config = { .reg_bits = 32, .reg_stride = 4, .val_bits = 32, - .max_register = 0xFFC, + .max_register = 0x1008, .fast_io = true, }; @@ -159,9 +163,11 @@ static const struct of_device_id qcom_apcs_ipc_of_match[] = { { .compatible = "qcom,msm8998-apcs-hmss-global", .data = &msm8998_apcs_data }, { .compatible = "qcom,qcs404-apcs-apps-global", .data = &msm8916_apcs_data }, { .compatible = "qcom,sc7180-apss-shared", .data = &apps_shared_apcs_data }, + { .compatible = "qcom,sc8180x-apss-shared", .data = &apps_shared_apcs_data }, { .compatible = "qcom,sdm660-apcs-hmss-global", .data = &sdm660_apcs_data }, { .compatible = "qcom,sdm845-apss-shared", .data = &apps_shared_apcs_data }, { .compatible = "qcom,sm8150-apss-shared", .data = &apps_shared_apcs_data }, + { .compatible = "qcom,sdx55-apcs-gcc", .data = &sdx55_apcs_data }, {} }; MODULE_DEVICE_TABLE(of, qcom_apcs_ipc_of_match); diff --git a/drivers/mailbox/sprd-mailbox.c b/drivers/mailbox/sprd-mailbox.c index f6fab24ae8a9..4c325301a2fe 100644 --- a/drivers/mailbox/sprd-mailbox.c +++ b/drivers/mailbox/sprd-mailbox.c @@ -35,7 +35,7 @@ #define SPRD_MBOX_IRQ_CLR BIT(0) /* Bit and mask definiation for outbox's SPRD_MBOX_FIFO_STS register */ -#define SPRD_OUTBOX_FIFO_FULL BIT(0) +#define SPRD_OUTBOX_FIFO_FULL BIT(2) #define SPRD_OUTBOX_FIFO_WR_SHIFT 16 #define SPRD_OUTBOX_FIFO_RD_SHIFT 24 #define SPRD_OUTBOX_FIFO_POS_MASK GENMASK(7, 0) diff --git a/drivers/mailbox/tegra-hsp.c b/drivers/mailbox/tegra-hsp.c index e07091d71986..acd0675da681 100644 --- a/drivers/mailbox/tegra-hsp.c +++ b/drivers/mailbox/tegra-hsp.c @@ -98,7 +98,9 @@ struct tegra_hsp { unsigned int num_ss; unsigned int num_db; unsigned int num_si; + spinlock_t lock; + struct lock_class_key lock_key; struct list_head doorbells; struct tegra_hsp_mailbox *mailboxes; @@ -775,6 +777,18 @@ static int tegra_hsp_probe(struct platform_device *pdev) return err; } + lockdep_register_key(&hsp->lock_key); + lockdep_set_class(&hsp->lock, &hsp->lock_key); + + return 0; +} + +static int tegra_hsp_remove(struct platform_device *pdev) +{ + struct tegra_hsp *hsp = platform_get_drvdata(pdev); + + lockdep_unregister_key(&hsp->lock_key); + return 0; } @@ -834,6 +848,7 @@ static struct platform_driver tegra_hsp_driver = { .pm = &tegra_hsp_pm_ops, }, .probe = tegra_hsp_probe, + .remove = tegra_hsp_remove, }; static int __init tegra_hsp_init(void) diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index fce4cbf9529d..50f3e673729c 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -1526,6 +1526,10 @@ EXPORT_SYMBOL_GPL(dm_bufio_get_block_size); sector_t dm_bufio_get_device_size(struct dm_bufio_client *c) { sector_t s = i_size_read(c->bdev->bd_inode) >> SECTOR_SHIFT; + if (s >= c->start) + s -= c->start; + else + s = 0; if (likely(c->sectors_per_block_bits >= 0)) s >>= c->sectors_per_block_bits; else diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c index 4312007d2d34..2d3cda0acacb 100644 --- a/drivers/md/dm-io.c +++ b/drivers/md/dm-io.c @@ -341,8 +341,8 @@ static void do_region(int op, int op_flags, unsigned region, num_bvecs = 1; break; default: - num_bvecs = min_t(int, BIO_MAX_PAGES, - dm_sector_div_up(remaining, (PAGE_SIZE >> SECTOR_SHIFT))); + num_bvecs = bio_max_segs(dm_sector_div_up(remaining, + (PAGE_SIZE >> SECTOR_SHIFT))); } bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, &io->client->bios); diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c index e3d35c6c9f71..57882654ffee 100644 --- a/drivers/md/dm-log-writes.c +++ b/drivers/md/dm-log-writes.c @@ -264,15 +264,14 @@ static int write_inline_data(struct log_writes_c *lc, void *entry, size_t entrylen, void *data, size_t datalen, sector_t sector) { - int num_pages, bio_pages, pg_datalen, pg_sectorlen, i; + int bio_pages, pg_datalen, pg_sectorlen, i; struct page *page; struct bio *bio; size_t ret; void *ptr; while (datalen) { - num_pages = ALIGN(datalen, PAGE_SIZE) >> PAGE_SHIFT; - bio_pages = min(num_pages, BIO_MAX_PAGES); + bio_pages = bio_max_segs(DIV_ROUND_UP(datalen, PAGE_SIZE)); atomic_inc(&lc->io_blocks); @@ -364,7 +363,7 @@ static int log_one_block(struct log_writes_c *lc, goto out; atomic_inc(&lc->io_blocks); - bio = bio_alloc(GFP_KERNEL, min(block->vec_cnt, BIO_MAX_PAGES)); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(block->vec_cnt)); if (!bio) { DMERR("Couldn't alloc log bio"); goto error; @@ -386,7 +385,8 @@ static int log_one_block(struct log_writes_c *lc, if (ret != block->vecs[i].bv_len) { atomic_inc(&lc->io_blocks); submit_bio(bio); - bio = bio_alloc(GFP_KERNEL, min(block->vec_cnt - i, BIO_MAX_PAGES)); + bio = bio_alloc(GFP_KERNEL, + bio_max_segs(block->vec_cnt - i)); if (!bio) { DMERR("Couldn't alloc log bio"); goto error; diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index fb41b4f23c48..66f4c6398f67 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -61,19 +61,18 @@ static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, unsigned *offset, struct dm_buffer **buf) { - u64 position, block; + u64 position, block, rem; u8 *res; position = (index + rsb) * v->fec->roots; - block = position >> v->data_dev_block_bits; - *offset = (unsigned)(position - (block << v->data_dev_block_bits)); + block = div64_u64_rem(position, v->fec->roots << SECTOR_SHIFT, &rem); + *offset = (unsigned)rem; - res = dm_bufio_read(v->fec->bufio, v->fec->start + block, buf); + res = dm_bufio_read(v->fec->bufio, block, buf); if (IS_ERR(res)) { DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", v->data_dev->name, (unsigned long long)rsb, - (unsigned long long)(v->fec->start + block), - PTR_ERR(res)); + (unsigned long long)block, PTR_ERR(res)); *buf = NULL; } @@ -155,7 +154,7 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, /* read the next block when we run out of parity bytes */ offset += v->fec->roots; - if (offset >= 1 << v->data_dev_block_bits) { + if (offset >= v->fec->roots << SECTOR_SHIFT) { dm_bufio_release(buf); par = fec_read_parity(v, rsb, block_offset, &offset, &buf); @@ -674,7 +673,7 @@ int verity_fec_ctr(struct dm_verity *v) { struct dm_verity_fec *f = v->fec; struct dm_target *ti = v->ti; - u64 hash_blocks; + u64 hash_blocks, fec_blocks; int ret; if (!verity_fec_is_enabled(v)) { @@ -744,15 +743,17 @@ int verity_fec_ctr(struct dm_verity *v) } f->bufio = dm_bufio_client_create(f->dev->bdev, - 1 << v->data_dev_block_bits, + f->roots << SECTOR_SHIFT, 1, 0, NULL, NULL); if (IS_ERR(f->bufio)) { ti->error = "Cannot initialize FEC bufio client"; return PTR_ERR(f->bufio); } - if (dm_bufio_get_device_size(f->bufio) < - ((f->start + f->rounds * f->roots) >> v->data_dev_block_bits)) { + dm_bufio_set_sector_offset(f->bufio, f->start << (v->data_dev_block_bits - SECTOR_SHIFT)); + + fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT); + if (dm_bufio_get_device_size(f->bufio) < fec_blocks) { ti->error = "FEC device is too small"; return -E2BIG; } diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig index 7d9d33d8ebf6..72c0df129d5c 100644 --- a/drivers/memory/Kconfig +++ b/drivers/memory/Kconfig @@ -137,6 +137,15 @@ config TI_EMIF_SRAM sequence so this driver provides several relocatable PM functions for the SoC PM code to use. +config FPGA_DFL_EMIF + tristate "FPGA DFL EMIF Driver" + depends on FPGA_DFL && HAS_IOMEM + help + This driver is for the EMIF private feature implemented under + FPGA Device Feature List (DFL) framework. It is used to expose + memory interface status information as well as memory clearing + control. + config MVEBU_DEVBUS bool "Marvell EBU Device Bus Controller" default y if PLAT_ORION diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile index e71cf7b99641..bc7663ed1c25 100644 --- a/drivers/memory/Makefile +++ b/drivers/memory/Makefile @@ -28,6 +28,8 @@ obj-$(CONFIG_STM32_FMC2_EBI) += stm32-fmc2-ebi.o obj-$(CONFIG_SAMSUNG_MC) += samsung/ obj-$(CONFIG_TEGRA_MC) += tegra/ obj-$(CONFIG_TI_EMIF_SRAM) += ti-emif-sram.o +obj-$(CONFIG_FPGA_DFL_EMIF) += dfl-emif.o + ti-emif-sram-objs := ti-emif-pm.o ti-emif-sram-pm.o AFLAGS_ti-emif-sram-pm.o :=-Wa,-march=armv7-a diff --git a/drivers/memory/dfl-emif.c b/drivers/memory/dfl-emif.c new file mode 100644 index 000000000000..3f719816771d --- /dev/null +++ b/drivers/memory/dfl-emif.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DFL device driver for EMIF private feature + * + * Copyright (C) 2020 Intel Corporation, Inc. + * + */ +#include <linux/bitfield.h> +#include <linux/dfl.h> +#include <linux/errno.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/spinlock.h> +#include <linux/types.h> + +#define FME_FEATURE_ID_EMIF 0x9 + +#define EMIF_STAT 0x8 +#define EMIF_STAT_INIT_DONE_SFT 0 +#define EMIF_STAT_CALC_FAIL_SFT 8 +#define EMIF_STAT_CLEAR_BUSY_SFT 16 +#define EMIF_CTRL 0x10 +#define EMIF_CTRL_CLEAR_EN_SFT 0 +#define EMIF_CTRL_CLEAR_EN_MSK GENMASK_ULL(3, 0) + +#define EMIF_POLL_INVL 10000 /* us */ +#define EMIF_POLL_TIMEOUT 5000000 /* us */ + +struct dfl_emif { + struct device *dev; + void __iomem *base; + spinlock_t lock; /* Serialises access to EMIF_CTRL reg */ +}; + +struct emif_attr { + struct device_attribute attr; + u32 shift; + u32 index; +}; + +#define to_emif_attr(dev_attr) \ + container_of(dev_attr, struct emif_attr, attr) + +static ssize_t emif_state_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct emif_attr *eattr = to_emif_attr(attr); + struct dfl_emif *de = dev_get_drvdata(dev); + u64 val; + + val = readq(de->base + EMIF_STAT); + + return sysfs_emit(buf, "%u\n", + !!(val & BIT_ULL(eattr->shift + eattr->index))); +} + +static ssize_t emif_clear_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct emif_attr *eattr = to_emif_attr(attr); + struct dfl_emif *de = dev_get_drvdata(dev); + u64 clear_busy_msk, clear_en_msk, val; + void __iomem *base = de->base; + + if (!sysfs_streq(buf, "1")) + return -EINVAL; + + clear_busy_msk = BIT_ULL(EMIF_STAT_CLEAR_BUSY_SFT + eattr->index); + clear_en_msk = BIT_ULL(EMIF_CTRL_CLEAR_EN_SFT + eattr->index); + + spin_lock(&de->lock); + /* The CLEAR_EN field is WO, but other fields are RW */ + val = readq(base + EMIF_CTRL); + val &= ~EMIF_CTRL_CLEAR_EN_MSK; + val |= clear_en_msk; + writeq(val, base + EMIF_CTRL); + spin_unlock(&de->lock); + + if (readq_poll_timeout(base + EMIF_STAT, val, + !(val & clear_busy_msk), + EMIF_POLL_INVL, EMIF_POLL_TIMEOUT)) { + dev_err(de->dev, "timeout, fail to clear\n"); + return -ETIMEDOUT; + } + + return count; +} + +#define emif_state_attr(_name, _shift, _index) \ + static struct emif_attr emif_attr_##inf##_index##_##_name = \ + { .attr = __ATTR(inf##_index##_##_name, 0444, \ + emif_state_show, NULL), \ + .shift = (_shift), .index = (_index) } + +#define emif_clear_attr(_index) \ + static struct emif_attr emif_attr_##inf##_index##_clear = \ + { .attr = __ATTR(inf##_index##_clear, 0200, \ + NULL, emif_clear_store), \ + .index = (_index) } + +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 0); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 1); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 2); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 3); + +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 0); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 1); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 2); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 3); + +emif_clear_attr(0); +emif_clear_attr(1); +emif_clear_attr(2); +emif_clear_attr(3); + +static struct attribute *dfl_emif_attrs[] = { + &emif_attr_inf0_init_done.attr.attr, + &emif_attr_inf0_cal_fail.attr.attr, + &emif_attr_inf0_clear.attr.attr, + + &emif_attr_inf1_init_done.attr.attr, + &emif_attr_inf1_cal_fail.attr.attr, + &emif_attr_inf1_clear.attr.attr, + + &emif_attr_inf2_init_done.attr.attr, + &emif_attr_inf2_cal_fail.attr.attr, + &emif_attr_inf2_clear.attr.attr, + + &emif_attr_inf3_init_done.attr.attr, + &emif_attr_inf3_cal_fail.attr.attr, + &emif_attr_inf3_clear.attr.attr, + + NULL, +}; + +static umode_t dfl_emif_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct dfl_emif *de = dev_get_drvdata(kobj_to_dev(kobj)); + struct emif_attr *eattr = container_of(attr, struct emif_attr, + attr.attr); + u64 val; + + /* + * This device supports upto 4 memory interfaces, but not all + * interfaces are used on different platforms. The read out value of + * CLEAN_EN field (which is a bitmap) could tell how many interfaces + * are available. + */ + val = FIELD_GET(EMIF_CTRL_CLEAR_EN_MSK, readq(de->base + EMIF_CTRL)); + + return (val & BIT_ULL(eattr->index)) ? attr->mode : 0; +} + +static const struct attribute_group dfl_emif_group = { + .is_visible = dfl_emif_visible, + .attrs = dfl_emif_attrs, +}; + +static const struct attribute_group *dfl_emif_groups[] = { + &dfl_emif_group, + NULL, +}; + +static int dfl_emif_probe(struct dfl_device *ddev) +{ + struct device *dev = &ddev->dev; + struct dfl_emif *de; + + de = devm_kzalloc(dev, sizeof(*de), GFP_KERNEL); + if (!de) + return -ENOMEM; + + de->base = devm_ioremap_resource(dev, &ddev->mmio_res); + if (IS_ERR(de->base)) + return PTR_ERR(de->base); + + de->dev = dev; + spin_lock_init(&de->lock); + dev_set_drvdata(dev, de); + + return 0; +} + +static const struct dfl_device_id dfl_emif_ids[] = { + { FME_ID, FME_FEATURE_ID_EMIF }, + { } +}; +MODULE_DEVICE_TABLE(dfl, dfl_emif_ids); + +static struct dfl_driver dfl_emif_driver = { + .drv = { + .name = "dfl-emif", + .dev_groups = dfl_emif_groups, + }, + .id_table = dfl_emif_ids, + .probe = dfl_emif_probe, +}; +module_dfl_driver(dfl_emif_driver); + +MODULE_DESCRIPTION("DFL EMIF driver"); +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 40c723e9a852..b74efa469e90 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -659,15 +659,6 @@ config MFD_INTEL_LPSS_PCI I2C, SPI and HS-UART starting from Intel Sunrisepoint (Intel Skylake PCH) in PCI mode. -config MFD_INTEL_MSIC - bool "Intel MSIC" - depends on INTEL_SCU - select MFD_CORE - help - Select this option to enable access to Intel MSIC (Avatele - Passage) chip. This chip embeds audio, battery, GPIO, etc. - devices used in Intel Medfield platforms. - config MFD_INTEL_PMC_BXT tristate "Intel PMC Driver for Broxton" depends on X86 diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile index 025543418835..834f5463af28 100644 --- a/drivers/mfd/Makefile +++ b/drivers/mfd/Makefile @@ -214,7 +214,6 @@ obj-$(CONFIG_MFD_ATMEL_SMC) += atmel-smc.o obj-$(CONFIG_MFD_INTEL_LPSS) += intel-lpss.o obj-$(CONFIG_MFD_INTEL_LPSS_PCI) += intel-lpss-pci.o obj-$(CONFIG_MFD_INTEL_LPSS_ACPI) += intel-lpss-acpi.o -obj-$(CONFIG_MFD_INTEL_MSIC) += intel_msic.o obj-$(CONFIG_MFD_INTEL_PMC_BXT) += intel_pmc_bxt.o obj-$(CONFIG_MFD_INTEL_PMT) += intel_pmt.o obj-$(CONFIG_MFD_PALMAS) += palmas.o diff --git a/drivers/mfd/intel_msic.c b/drivers/mfd/intel_msic.c deleted file mode 100644 index daa772f8146b..000000000000 --- a/drivers/mfd/intel_msic.c +++ /dev/null @@ -1,425 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Driver for Intel MSIC - * - * Copyright (C) 2011, Intel Corporation - * Author: Mika Westerberg <mika.westerberg@linux.intel.com> - */ - -#include <linux/err.h> -#include <linux/gpio.h> -#include <linux/io.h> -#include <linux/init.h> -#include <linux/mfd/core.h> -#include <linux/mfd/intel_msic.h> -#include <linux/platform_device.h> -#include <linux/slab.h> - -#include <asm/intel_scu_ipc.h> - -#define MSIC_VENDOR(id) ((id >> 6) & 3) -#define MSIC_VERSION(id) (id & 0x3f) -#define MSIC_MAJOR(id) ('A' + ((id >> 3) & 7)) -#define MSIC_MINOR(id) (id & 7) - -/* - * MSIC interrupt tree is readable from SRAM at INTEL_MSIC_IRQ_PHYS_BASE. - * Since IRQ block starts from address 0x002 we need to subtract that from - * the actual IRQ status register address. - */ -#define MSIC_IRQ_STATUS(x) (INTEL_MSIC_IRQ_PHYS_BASE + ((x) - 2)) -#define MSIC_IRQ_STATUS_ACCDET MSIC_IRQ_STATUS(INTEL_MSIC_ACCDET) - -/* - * The SCU hardware has limitation of 16 bytes per read/write buffer on - * Medfield. - */ -#define SCU_IPC_RWBUF_LIMIT 16 - -/** - * struct intel_msic - an MSIC MFD instance - * @pdev: pointer to the platform device - * @vendor: vendor ID - * @version: chip version - * @irq_base: base address of the mapped MSIC SRAM interrupt tree - */ -struct intel_msic { - struct platform_device *pdev; - unsigned vendor; - unsigned version; - void __iomem *irq_base; -}; - -static const struct resource msic_touch_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_adc_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_battery_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_gpio_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_audio_resources[] = { - DEFINE_RES_IRQ_NAMED(0, "IRQ"), - /* - * We will pass IRQ_BASE to the driver now but this can be removed - * when/if the driver starts to use intel_msic_irq_read(). - */ - DEFINE_RES_MEM_NAMED(MSIC_IRQ_STATUS_ACCDET, 1, "IRQ_BASE"), -}; - -static const struct resource msic_hdmi_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_thermal_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_power_btn_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_ocd_resources[] = { - DEFINE_RES_IRQ(0), -}; - -/* - * Devices that are part of the MSIC and are available via firmware - * populated SFI DEVS table. - */ -static struct mfd_cell msic_devs[] = { - [INTEL_MSIC_BLOCK_TOUCH] = { - .name = "msic_touch", - .num_resources = ARRAY_SIZE(msic_touch_resources), - .resources = msic_touch_resources, - }, - [INTEL_MSIC_BLOCK_ADC] = { - .name = "msic_adc", - .num_resources = ARRAY_SIZE(msic_adc_resources), - .resources = msic_adc_resources, - }, - [INTEL_MSIC_BLOCK_BATTERY] = { - .name = "msic_battery", - .num_resources = ARRAY_SIZE(msic_battery_resources), - .resources = msic_battery_resources, - }, - [INTEL_MSIC_BLOCK_GPIO] = { - .name = "msic_gpio", - .num_resources = ARRAY_SIZE(msic_gpio_resources), - .resources = msic_gpio_resources, - }, - [INTEL_MSIC_BLOCK_AUDIO] = { - .name = "msic_audio", - .num_resources = ARRAY_SIZE(msic_audio_resources), - .resources = msic_audio_resources, - }, - [INTEL_MSIC_BLOCK_HDMI] = { - .name = "msic_hdmi", - .num_resources = ARRAY_SIZE(msic_hdmi_resources), - .resources = msic_hdmi_resources, - }, - [INTEL_MSIC_BLOCK_THERMAL] = { - .name = "msic_thermal", - .num_resources = ARRAY_SIZE(msic_thermal_resources), - .resources = msic_thermal_resources, - }, - [INTEL_MSIC_BLOCK_POWER_BTN] = { - .name = "msic_power_btn", - .num_resources = ARRAY_SIZE(msic_power_btn_resources), - .resources = msic_power_btn_resources, - }, - [INTEL_MSIC_BLOCK_OCD] = { - .name = "msic_ocd", - .num_resources = ARRAY_SIZE(msic_ocd_resources), - .resources = msic_ocd_resources, - }, -}; - -/* - * Other MSIC related devices which are not directly available via SFI DEVS - * table. These can be pseudo devices, regulators etc. which are needed for - * different purposes. - * - * These devices appear only after the MSIC driver itself is initialized so - * we can guarantee that the SCU IPC interface is ready. - */ -static const struct mfd_cell msic_other_devs[] = { - /* Audio codec in the MSIC */ - { - .id = -1, - .name = "sn95031", - }, -}; - -/** - * intel_msic_reg_read - read a single MSIC register - * @reg: register to read - * @val: register value is placed here - * - * Read a single register from MSIC. Returns %0 on success and negative - * errno in case of failure. - * - * Function may sleep. - */ -int intel_msic_reg_read(unsigned short reg, u8 *val) -{ - return intel_scu_ipc_ioread8(reg, val); -} -EXPORT_SYMBOL_GPL(intel_msic_reg_read); - -/** - * intel_msic_reg_write - write a single MSIC register - * @reg: register to write - * @val: value to write to that register - * - * Write a single MSIC register. Returns 0 on success and negative - * errno in case of failure. - * - * Function may sleep. - */ -int intel_msic_reg_write(unsigned short reg, u8 val) -{ - return intel_scu_ipc_iowrite8(reg, val); -} -EXPORT_SYMBOL_GPL(intel_msic_reg_write); - -/** - * intel_msic_reg_update - update a single MSIC register - * @reg: register to update - * @val: value to write to the register - * @mask: specifies which of the bits are updated (%0 = don't update, - * %1 = update) - * - * Perform an update to a register @reg. @mask is used to specify which - * bits are updated. Returns %0 in case of success and negative errno in - * case of failure. - * - * Function may sleep. - */ -int intel_msic_reg_update(unsigned short reg, u8 val, u8 mask) -{ - return intel_scu_ipc_update_register(reg, val, mask); -} -EXPORT_SYMBOL_GPL(intel_msic_reg_update); - -/** - * intel_msic_bulk_read - read an array of registers - * @reg: array of register addresses to read - * @buf: array where the read values are placed - * @count: number of registers to read - * - * Function reads @count registers from the MSIC using addresses passed in - * @reg. Read values are placed in @buf. Reads are performed atomically - * wrt. MSIC. - * - * Returns %0 in case of success and negative errno in case of failure. - * - * Function may sleep. - */ -int intel_msic_bulk_read(unsigned short *reg, u8 *buf, size_t count) -{ - if (WARN_ON(count > SCU_IPC_RWBUF_LIMIT)) - return -EINVAL; - - return intel_scu_ipc_readv(reg, buf, count); -} -EXPORT_SYMBOL_GPL(intel_msic_bulk_read); - -/** - * intel_msic_bulk_write - write an array of values to the MSIC registers - * @reg: array of registers to write - * @buf: values to write to each register - * @count: number of registers to write - * - * Function writes @count registers in @buf to MSIC. Writes are performed - * atomically wrt MSIC. Returns %0 in case of success and negative errno in - * case of failure. - * - * Function may sleep. - */ -int intel_msic_bulk_write(unsigned short *reg, u8 *buf, size_t count) -{ - if (WARN_ON(count > SCU_IPC_RWBUF_LIMIT)) - return -EINVAL; - - return intel_scu_ipc_writev(reg, buf, count); -} -EXPORT_SYMBOL_GPL(intel_msic_bulk_write); - -/** - * intel_msic_irq_read - read a register from an MSIC interrupt tree - * @msic: MSIC instance - * @reg: interrupt register (between %INTEL_MSIC_IRQLVL1 and - * %INTEL_MSIC_RESETIRQ2) - * @val: value of the register is placed here - * - * This function can be used by an MSIC subdevice interrupt handler to read - * a register value from the MSIC interrupt tree. In this way subdevice - * drivers don't have to map in the interrupt tree themselves but can just - * call this function instead. - * - * Function doesn't sleep and is callable from interrupt context. - * - * Returns %-EINVAL if @reg is outside of the allowed register region. - */ -int intel_msic_irq_read(struct intel_msic *msic, unsigned short reg, u8 *val) -{ - if (WARN_ON(reg < INTEL_MSIC_IRQLVL1 || reg > INTEL_MSIC_RESETIRQ2)) - return -EINVAL; - - *val = readb(msic->irq_base + (reg - INTEL_MSIC_IRQLVL1)); - return 0; -} -EXPORT_SYMBOL_GPL(intel_msic_irq_read); - -static int intel_msic_init_devices(struct intel_msic *msic) -{ - struct platform_device *pdev = msic->pdev; - struct intel_msic_platform_data *pdata = dev_get_platdata(&pdev->dev); - int ret, i; - - if (pdata->gpio) { - struct mfd_cell *cell = &msic_devs[INTEL_MSIC_BLOCK_GPIO]; - - cell->platform_data = pdata->gpio; - cell->pdata_size = sizeof(*pdata->gpio); - } - - if (pdata->ocd) { - unsigned gpio = pdata->ocd->gpio; - - ret = devm_gpio_request_one(&pdev->dev, gpio, - GPIOF_IN, "ocd_gpio"); - if (ret) { - dev_err(&pdev->dev, "failed to register OCD GPIO\n"); - return ret; - } - - ret = gpio_to_irq(gpio); - if (ret < 0) { - dev_err(&pdev->dev, "no IRQ number for OCD GPIO\n"); - return ret; - } - - /* Update the IRQ number for the OCD */ - pdata->irq[INTEL_MSIC_BLOCK_OCD] = ret; - } - - for (i = 0; i < ARRAY_SIZE(msic_devs); i++) { - if (!pdata->irq[i]) - continue; - - ret = mfd_add_devices(&pdev->dev, -1, &msic_devs[i], 1, NULL, - pdata->irq[i], NULL); - if (ret) - goto fail; - } - - ret = mfd_add_devices(&pdev->dev, 0, msic_other_devs, - ARRAY_SIZE(msic_other_devs), NULL, 0, NULL); - if (ret) - goto fail; - - return 0; - -fail: - mfd_remove_devices(&pdev->dev); - - return ret; -} - -static void intel_msic_remove_devices(struct intel_msic *msic) -{ - struct platform_device *pdev = msic->pdev; - - mfd_remove_devices(&pdev->dev); -} - -static int intel_msic_probe(struct platform_device *pdev) -{ - struct intel_msic_platform_data *pdata = dev_get_platdata(&pdev->dev); - struct intel_msic *msic; - struct resource *res; - u8 id0, id1; - int ret; - - if (!pdata) { - dev_err(&pdev->dev, "no platform data passed\n"); - return -EINVAL; - } - - /* First validate that we have an MSIC in place */ - ret = intel_scu_ipc_ioread8(INTEL_MSIC_ID0, &id0); - if (ret) { - dev_err(&pdev->dev, "failed to identify the MSIC chip (ID0)\n"); - return -ENXIO; - } - - ret = intel_scu_ipc_ioread8(INTEL_MSIC_ID1, &id1); - if (ret) { - dev_err(&pdev->dev, "failed to identify the MSIC chip (ID1)\n"); - return -ENXIO; - } - - if (MSIC_VENDOR(id0) != MSIC_VENDOR(id1)) { - dev_err(&pdev->dev, "invalid vendor ID: %x, %x\n", id0, id1); - return -ENXIO; - } - - msic = devm_kzalloc(&pdev->dev, sizeof(*msic), GFP_KERNEL); - if (!msic) - return -ENOMEM; - - msic->vendor = MSIC_VENDOR(id0); - msic->version = MSIC_VERSION(id0); - msic->pdev = pdev; - - /* - * Map in the MSIC interrupt tree area in SRAM. This is exposed to - * the clients via intel_msic_irq_read(). - */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - msic->irq_base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(msic->irq_base)) - return PTR_ERR(msic->irq_base); - - platform_set_drvdata(pdev, msic); - - ret = intel_msic_init_devices(msic); - if (ret) { - dev_err(&pdev->dev, "failed to initialize MSIC devices\n"); - return ret; - } - - dev_info(&pdev->dev, "Intel MSIC version %c%d (vendor %#x)\n", - MSIC_MAJOR(msic->version), MSIC_MINOR(msic->version), - msic->vendor); - - return 0; -} - -static int intel_msic_remove(struct platform_device *pdev) -{ - struct intel_msic *msic = platform_get_drvdata(pdev); - - intel_msic_remove_devices(msic); - - return 0; -} - -static struct platform_driver intel_msic_driver = { - .probe = intel_msic_probe, - .remove = intel_msic_remove, - .driver = { - .name = "intel_msic", - }, -}; -builtin_platform_driver(intel_msic_driver); diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index fafa8b0d8099..f532c59bb59b 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -50,14 +50,6 @@ config AD525X_DPOT_SPI To compile this driver as a module, choose M here: the module will be called ad525x_dpot-spi. -config ATMEL_TCLIB - bool "Atmel AT32/AT91 Timer/Counter Library" - depends on ARCH_AT91 - help - Select this if you want a library to allocate the Timer/Counter - blocks found on many Atmel processors. This facilitates using - these blocks by different drivers despite processor differences. - config DUMMY_IRQ tristate "Dummy IRQ handler" help @@ -112,19 +104,6 @@ config PHANTOM If you choose to build module, its name will be phantom. If unsure, say N here. -config INTEL_MID_PTI - tristate "Parallel Trace Interface for MIPI P1149.7 cJTAG standard" - depends on PCI && TTY && (X86_INTEL_MID || COMPILE_TEST) - help - The PTI (Parallel Trace Interface) driver directs - trace data routed from various parts in the system out - through an Intel Penwell PTI port and out of the mobile - device for analysis with a debugging tool (Lauterbach or Fido). - - You should select this driver if the target kernel is meant for - an Intel Atom (non-netbook) mobile device containing a MIPI - P1149.7 standard implementation. - config TIFM_CORE tristate "TI Flash Media interface support" depends on PCI @@ -478,6 +457,7 @@ source "drivers/misc/genwqe/Kconfig" source "drivers/misc/echo/Kconfig" source "drivers/misc/cxl/Kconfig" source "drivers/misc/ocxl/Kconfig" +source "drivers/misc/bcm-vk/Kconfig" source "drivers/misc/cardreader/Kconfig" source "drivers/misc/habanalabs/Kconfig" source "drivers/misc/uacce/Kconfig" diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index d23231e73330..99b6f15a3c70 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -8,9 +8,7 @@ obj-$(CONFIG_IBMVMC) += ibmvmc.o obj-$(CONFIG_AD525X_DPOT) += ad525x_dpot.o obj-$(CONFIG_AD525X_DPOT_I2C) += ad525x_dpot-i2c.o obj-$(CONFIG_AD525X_DPOT_SPI) += ad525x_dpot-spi.o -obj-$(CONFIG_INTEL_MID_PTI) += pti.o obj-$(CONFIG_ATMEL_SSC) += atmel-ssc.o -obj-$(CONFIG_ATMEL_TCLIB) += atmel_tclib.o obj-$(CONFIG_DUMMY_IRQ) += dummy-irq.o obj-$(CONFIG_ICS932S401) += ics932s401.o obj-$(CONFIG_LKDTM) += lkdtm/ @@ -51,6 +49,7 @@ obj-$(CONFIG_ECHO) += echo/ obj-$(CONFIG_CXL_BASE) += cxl/ obj-$(CONFIG_PCI_ENDPOINT_TEST) += pci_endpoint_test.o obj-$(CONFIG_OCXL) += ocxl/ +obj-$(CONFIG_BCM_VK) += bcm-vk/ obj-y += cardreader/ obj-$(CONFIG_PVPANIC) += pvpanic.o obj-$(CONFIG_HABANA_AI) += habanalabs/ diff --git a/drivers/misc/atmel_tclib.c b/drivers/misc/atmel_tclib.c deleted file mode 100644 index 7de7840f613c..000000000000 --- a/drivers/misc/atmel_tclib.c +++ /dev/null @@ -1,200 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#include <linux/clk.h> -#include <linux/err.h> -#include <linux/init.h> -#include <linux/io.h> -#include <linux/ioport.h> -#include <linux/kernel.h> -#include <linux/platform_device.h> -#include <linux/module.h> -#include <linux/slab.h> -#include <linux/export.h> -#include <linux/of.h> -#include <soc/at91/atmel_tcb.h> - -/* - * This is a thin library to solve the problem of how to portably allocate - * one of the TC blocks. For simplicity, it doesn't currently expect to - * share individual timers between different drivers. - */ - -#if defined(CONFIG_AVR32) -/* AVR32 has these divide PBB */ -const u8 atmel_tc_divisors[5] = { 0, 4, 8, 16, 32, }; -EXPORT_SYMBOL(atmel_tc_divisors); - -#elif defined(CONFIG_ARCH_AT91) -/* AT91 has these divide MCK */ -const u8 atmel_tc_divisors[5] = { 2, 8, 32, 128, 0, }; -EXPORT_SYMBOL(atmel_tc_divisors); - -#endif - -static DEFINE_SPINLOCK(tc_list_lock); -static LIST_HEAD(tc_list); - -/** - * atmel_tc_alloc - allocate a specified TC block - * @block: which block to allocate - * - * Caller allocates a block. If it is available, a pointer to a - * pre-initialized struct atmel_tc is returned. The caller can access - * the registers directly through the "regs" field. - */ -struct atmel_tc *atmel_tc_alloc(unsigned block) -{ - struct atmel_tc *tc; - struct platform_device *pdev = NULL; - - spin_lock(&tc_list_lock); - list_for_each_entry(tc, &tc_list, node) { - if (tc->allocated) - continue; - - if ((tc->pdev->dev.of_node && tc->id == block) || - (tc->pdev->id == block)) { - pdev = tc->pdev; - tc->allocated = true; - break; - } - } - spin_unlock(&tc_list_lock); - - return pdev ? tc : NULL; -} -EXPORT_SYMBOL_GPL(atmel_tc_alloc); - -/** - * atmel_tc_free - release a specified TC block - * @tc: Timer/counter block that was returned by atmel_tc_alloc() - * - * This reverses the effect of atmel_tc_alloc(), invalidating the resource - * returned by that routine and making the TC available to other drivers. - */ -void atmel_tc_free(struct atmel_tc *tc) -{ - spin_lock(&tc_list_lock); - if (tc->allocated) - tc->allocated = false; - spin_unlock(&tc_list_lock); -} -EXPORT_SYMBOL_GPL(atmel_tc_free); - -#if defined(CONFIG_OF) -static struct atmel_tcb_config tcb_rm9200_config = { - .counter_width = 16, -}; - -static struct atmel_tcb_config tcb_sam9x5_config = { - .counter_width = 32, -}; - -static const struct of_device_id atmel_tcb_dt_ids[] = { - { - .compatible = "atmel,at91rm9200-tcb", - .data = &tcb_rm9200_config, - }, { - .compatible = "atmel,at91sam9x5-tcb", - .data = &tcb_sam9x5_config, - }, { - /* sentinel */ - } -}; - -MODULE_DEVICE_TABLE(of, atmel_tcb_dt_ids); -#endif - -static int __init tc_probe(struct platform_device *pdev) -{ - struct atmel_tc *tc; - struct clk *clk; - int irq; - unsigned int i; - - if (of_get_child_count(pdev->dev.of_node)) - return -EBUSY; - - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return -EINVAL; - - tc = devm_kzalloc(&pdev->dev, sizeof(struct atmel_tc), GFP_KERNEL); - if (!tc) - return -ENOMEM; - - tc->pdev = pdev; - - clk = devm_clk_get(&pdev->dev, "t0_clk"); - if (IS_ERR(clk)) - return PTR_ERR(clk); - - tc->slow_clk = devm_clk_get(&pdev->dev, "slow_clk"); - if (IS_ERR(tc->slow_clk)) - return PTR_ERR(tc->slow_clk); - - tc->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(tc->regs)) - return PTR_ERR(tc->regs); - - /* Now take SoC information if available */ - if (pdev->dev.of_node) { - const struct of_device_id *match; - match = of_match_node(atmel_tcb_dt_ids, pdev->dev.of_node); - if (match) - tc->tcb_config = match->data; - - tc->id = of_alias_get_id(tc->pdev->dev.of_node, "tcb"); - } else { - tc->id = pdev->id; - } - - tc->clk[0] = clk; - tc->clk[1] = devm_clk_get(&pdev->dev, "t1_clk"); - if (IS_ERR(tc->clk[1])) - tc->clk[1] = clk; - tc->clk[2] = devm_clk_get(&pdev->dev, "t2_clk"); - if (IS_ERR(tc->clk[2])) - tc->clk[2] = clk; - - tc->irq[0] = irq; - tc->irq[1] = platform_get_irq(pdev, 1); - if (tc->irq[1] < 0) - tc->irq[1] = irq; - tc->irq[2] = platform_get_irq(pdev, 2); - if (tc->irq[2] < 0) - tc->irq[2] = irq; - - for (i = 0; i < 3; i++) - writel(ATMEL_TC_ALL_IRQ, tc->regs + ATMEL_TC_REG(i, IDR)); - - spin_lock(&tc_list_lock); - list_add_tail(&tc->node, &tc_list); - spin_unlock(&tc_list_lock); - - platform_set_drvdata(pdev, tc); - - return 0; -} - -static void tc_shutdown(struct platform_device *pdev) -{ - int i; - struct atmel_tc *tc = platform_get_drvdata(pdev); - - for (i = 0; i < 3; i++) - writel(ATMEL_TC_ALL_IRQ, tc->regs + ATMEL_TC_REG(i, IDR)); -} - -static struct platform_driver tc_driver = { - .driver = { - .name = "atmel_tcb", - .of_match_table = of_match_ptr(atmel_tcb_dt_ids), - }, - .shutdown = tc_shutdown, -}; - -static int __init tc_init(void) -{ - return platform_driver_probe(&tc_driver, tc_probe); -} -arch_initcall(tc_init); diff --git a/drivers/misc/bcm-vk/Kconfig b/drivers/misc/bcm-vk/Kconfig new file mode 100644 index 000000000000..68a972772b99 --- /dev/null +++ b/drivers/misc/bcm-vk/Kconfig @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Broadcom VK device +# +config BCM_VK + tristate "Support for Broadcom VK Accelerators" + depends on PCI_MSI + help + Select this option to enable support for Broadcom + VK Accelerators. VK is used for performing + multiple specific offload processing tasks in parallel. + Such offload tasks assist in such operations as video + transcoding, compression, and crypto tasks. + This driver enables userspace programs to access these + accelerators via /dev/bcm-vk.N devices. + + If unsure, say N. + +config BCM_VK_TTY + bool "Enable tty ports on a Broadcom VK Accelerator device" + depends on TTY + depends on BCM_VK + help + Select this option to enable tty support to allow console + access to Broadcom VK Accelerator cards from host. + + Device node will in the form /dev/bcm-vk.x_ttyVKy where: + x is the instance of the VK card + y is the tty device number on the VK card. diff --git a/drivers/misc/bcm-vk/Makefile b/drivers/misc/bcm-vk/Makefile new file mode 100644 index 000000000000..1df2ebe851ca --- /dev/null +++ b/drivers/misc/bcm-vk/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for Broadcom VK driver +# + +obj-$(CONFIG_BCM_VK) += bcm_vk.o +bcm_vk-objs := \ + bcm_vk_dev.o \ + bcm_vk_msg.o \ + bcm_vk_sg.o + +bcm_vk-$(CONFIG_BCM_VK_TTY) += bcm_vk_tty.o diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h new file mode 100644 index 000000000000..a1338f375589 --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk.h @@ -0,0 +1,549 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2018-2020 Broadcom. + */ + +#ifndef BCM_VK_H +#define BCM_VK_H + +#include <linux/atomic.h> +#include <linux/firmware.h> +#include <linux/irq.h> +#include <linux/kref.h> +#include <linux/miscdevice.h> +#include <linux/mutex.h> +#include <linux/pci.h> +#include <linux/poll.h> +#include <linux/sched/signal.h> +#include <linux/tty.h> +#include <linux/uaccess.h> +#include <uapi/linux/misc/bcm_vk.h> + +#include "bcm_vk_msg.h" + +#define DRV_MODULE_NAME "bcm-vk" + +/* + * Load Image is completed in two stages: + * + * 1) When the VK device boot-up, M7 CPU runs and executes the BootROM. + * The Secure Boot Loader (SBL) as part of the BootROM will run + * to open up ITCM for host to push BOOT1 image. + * SBL will authenticate the image before jumping to BOOT1 image. + * + * 2) Because BOOT1 image is a secured image, we also called it the + * Secure Boot Image (SBI). At second stage, SBI will initialize DDR + * and wait for host to push BOOT2 image to DDR. + * SBI will authenticate the image before jumping to BOOT2 image. + * + */ +/* Location of registers of interest in BAR0 */ + +/* Request register for Secure Boot Loader (SBL) download */ +#define BAR_CODEPUSH_SBL 0x400 +/* Start of ITCM */ +#define CODEPUSH_BOOT1_ENTRY 0x00400000 +#define CODEPUSH_MASK 0xfffff000 +#define CODEPUSH_BOOTSTART BIT(0) + +/* Boot Status register */ +#define BAR_BOOT_STATUS 0x404 + +#define SRAM_OPEN BIT(16) +#define DDR_OPEN BIT(17) + +/* Firmware loader progress status definitions */ +#define FW_LOADER_ACK_SEND_MORE_DATA BIT(18) +#define FW_LOADER_ACK_IN_PROGRESS BIT(19) +#define FW_LOADER_ACK_RCVD_ALL_DATA BIT(20) + +/* Boot1/2 is running in standalone mode */ +#define BOOT_STDALONE_RUNNING BIT(21) + +/* definitions for boot status register */ +#define BOOT_STATE_MASK (0xffffffff & \ + ~(FW_LOADER_ACK_SEND_MORE_DATA | \ + FW_LOADER_ACK_IN_PROGRESS | \ + BOOT_STDALONE_RUNNING)) + +#define BOOT_ERR_SHIFT 4 +#define BOOT_ERR_MASK (0xf << BOOT_ERR_SHIFT) +#define BOOT_PROG_MASK 0xf + +#define BROM_STATUS_NOT_RUN 0x2 +#define BROM_NOT_RUN (SRAM_OPEN | BROM_STATUS_NOT_RUN) +#define BROM_STATUS_COMPLETE 0x6 +#define BROM_RUNNING (SRAM_OPEN | BROM_STATUS_COMPLETE) +#define BOOT1_STATUS_COMPLETE 0x6 +#define BOOT1_RUNNING (DDR_OPEN | BOOT1_STATUS_COMPLETE) +#define BOOT2_STATUS_COMPLETE 0x6 +#define BOOT2_RUNNING (FW_LOADER_ACK_RCVD_ALL_DATA | \ + BOOT2_STATUS_COMPLETE) + +/* Boot request for Secure Boot Image (SBI) */ +#define BAR_CODEPUSH_SBI 0x408 +/* 64M mapped to BAR2 */ +#define CODEPUSH_BOOT2_ENTRY 0x60000000 + +#define BAR_CARD_STATUS 0x410 +/* CARD_STATUS definitions */ +#define CARD_STATUS_TTYVK0_READY BIT(0) +#define CARD_STATUS_TTYVK1_READY BIT(1) + +#define BAR_BOOT1_STDALONE_PROGRESS 0x420 +#define BOOT1_STDALONE_SUCCESS (BIT(13) | BIT(14)) +#define BOOT1_STDALONE_PROGRESS_MASK BOOT1_STDALONE_SUCCESS + +#define BAR_METADATA_VERSION 0x440 +#define BAR_OS_UPTIME 0x444 +#define BAR_CHIP_ID 0x448 +#define MAJOR_SOC_REV(_chip_id) (((_chip_id) >> 20) & 0xf) + +#define BAR_CARD_TEMPERATURE 0x45c +/* defines for all temperature sensor */ +#define BCM_VK_TEMP_FIELD_MASK 0xff +#define BCM_VK_CPU_TEMP_SHIFT 0 +#define BCM_VK_DDR0_TEMP_SHIFT 8 +#define BCM_VK_DDR1_TEMP_SHIFT 16 + +#define BAR_CARD_VOLTAGE 0x460 +/* defines for voltage rail conversion */ +#define BCM_VK_VOLT_RAIL_MASK 0xffff +#define BCM_VK_3P3_VOLT_REG_SHIFT 16 + +#define BAR_CARD_ERR_LOG 0x464 +/* Error log register bit definition - register for error alerts */ +#define ERR_LOG_UECC BIT(0) +#define ERR_LOG_SSIM_BUSY BIT(1) +#define ERR_LOG_AFBC_BUSY BIT(2) +#define ERR_LOG_HIGH_TEMP_ERR BIT(3) +#define ERR_LOG_WDOG_TIMEOUT BIT(4) +#define ERR_LOG_SYS_FAULT BIT(5) +#define ERR_LOG_RAMDUMP BIT(6) +#define ERR_LOG_COP_WDOG_TIMEOUT BIT(7) +/* warnings */ +#define ERR_LOG_MEM_ALLOC_FAIL BIT(8) +#define ERR_LOG_LOW_TEMP_WARN BIT(9) +#define ERR_LOG_ECC BIT(10) +#define ERR_LOG_IPC_DWN BIT(11) + +/* Alert bit definitions detectd on host */ +#define ERR_LOG_HOST_INTF_V_FAIL BIT(13) +#define ERR_LOG_HOST_HB_FAIL BIT(14) +#define ERR_LOG_HOST_PCIE_DWN BIT(15) + +#define BAR_CARD_ERR_MEM 0x468 +/* defines for mem err, all fields have same width */ +#define BCM_VK_MEM_ERR_FIELD_MASK 0xff +#define BCM_VK_ECC_MEM_ERR_SHIFT 0 +#define BCM_VK_UECC_MEM_ERR_SHIFT 8 +/* threshold of event occurrence and logs start to come out */ +#define BCM_VK_ECC_THRESHOLD 10 +#define BCM_VK_UECC_THRESHOLD 1 + +#define BAR_CARD_PWR_AND_THRE 0x46c +/* defines for power and temp threshold, all fields have same width */ +#define BCM_VK_PWR_AND_THRE_FIELD_MASK 0xff +#define BCM_VK_LOW_TEMP_THRE_SHIFT 0 +#define BCM_VK_HIGH_TEMP_THRE_SHIFT 8 +#define BCM_VK_PWR_STATE_SHIFT 16 + +#define BAR_CARD_STATIC_INFO 0x470 + +#define BAR_INTF_VER 0x47c +#define BAR_INTF_VER_MAJOR_SHIFT 16 +#define BAR_INTF_VER_MASK 0xffff +/* + * major and minor semantic version numbers supported + * Please update as required on interface changes + */ +#define SEMANTIC_MAJOR 1 +#define SEMANTIC_MINOR 0 + +/* + * first door bell reg, ie for queue = 0. Only need the first one, as + * we will use the queue number to derive the others + */ +#define VK_BAR0_REGSEG_DB_BASE 0x484 +#define VK_BAR0_REGSEG_DB_REG_GAP 8 /* + * DB register gap, + * DB1 at 0x48c and DB2 at 0x494 + */ + +/* reset register and specific values */ +#define VK_BAR0_RESET_DB_NUM 3 +#define VK_BAR0_RESET_DB_SOFT 0xffffffff +#define VK_BAR0_RESET_DB_HARD 0xfffffffd +#define VK_BAR0_RESET_RAMPDUMP 0xa0000000 + +#define VK_BAR0_Q_DB_BASE(q_num) (VK_BAR0_REGSEG_DB_BASE + \ + ((q_num) * VK_BAR0_REGSEG_DB_REG_GAP)) +#define VK_BAR0_RESET_DB_BASE (VK_BAR0_REGSEG_DB_BASE + \ + (VK_BAR0_RESET_DB_NUM * VK_BAR0_REGSEG_DB_REG_GAP)) + +#define BAR_BOOTSRC_SELECT 0xc78 +/* BOOTSRC definitions */ +#define BOOTSRC_SOFT_ENABLE BIT(14) + +/* Card OS Firmware version size */ +#define BAR_FIRMWARE_TAG_SIZE 50 +#define FIRMWARE_STATUS_PRE_INIT_DONE 0x1f + +/* VK MSG_ID defines */ +#define VK_MSG_ID_BITMAP_SIZE 4096 +#define VK_MSG_ID_BITMAP_MASK (VK_MSG_ID_BITMAP_SIZE - 1) +#define VK_MSG_ID_OVERFLOW 0xffff + +/* + * BAR1 + */ + +/* BAR1 message q definition */ + +/* indicate if msgq ctrl in BAR1 is populated */ +#define VK_BAR1_MSGQ_DEF_RDY 0x60c0 +/* ready marker value for the above location, normal boot2 */ +#define VK_BAR1_MSGQ_RDY_MARKER 0xbeefcafe +/* ready marker value for the above location, normal boot2 */ +#define VK_BAR1_DIAG_RDY_MARKER 0xdeadcafe +/* number of msgqs in BAR1 */ +#define VK_BAR1_MSGQ_NR 0x60c4 +/* BAR1 queue control structure offset */ +#define VK_BAR1_MSGQ_CTRL_OFF 0x60c8 + +/* BAR1 ucode and boot1 version tag */ +#define VK_BAR1_UCODE_VER_TAG 0x6170 +#define VK_BAR1_BOOT1_VER_TAG 0x61b0 +#define VK_BAR1_VER_TAG_SIZE 64 + +/* Memory to hold the DMA buffer memory address allocated for boot2 download */ +#define VK_BAR1_DMA_BUF_OFF_HI 0x61e0 +#define VK_BAR1_DMA_BUF_OFF_LO (VK_BAR1_DMA_BUF_OFF_HI + 4) +#define VK_BAR1_DMA_BUF_SZ (VK_BAR1_DMA_BUF_OFF_HI + 8) + +/* Scratch memory allocated on host for VK */ +#define VK_BAR1_SCRATCH_OFF_HI 0x61f0 +#define VK_BAR1_SCRATCH_OFF_LO (VK_BAR1_SCRATCH_OFF_HI + 4) +#define VK_BAR1_SCRATCH_SZ_ADDR (VK_BAR1_SCRATCH_OFF_HI + 8) +#define VK_BAR1_SCRATCH_DEF_NR_PAGES 32 + +/* BAR1 DAUTH info */ +#define VK_BAR1_DAUTH_BASE_ADDR 0x6200 +#define VK_BAR1_DAUTH_STORE_SIZE 0x48 +#define VK_BAR1_DAUTH_VALID_SIZE 0x8 +#define VK_BAR1_DAUTH_MAX 4 +#define VK_BAR1_DAUTH_STORE_ADDR(x) \ + (VK_BAR1_DAUTH_BASE_ADDR + \ + (x) * (VK_BAR1_DAUTH_STORE_SIZE + VK_BAR1_DAUTH_VALID_SIZE)) +#define VK_BAR1_DAUTH_VALID_ADDR(x) \ + (VK_BAR1_DAUTH_STORE_ADDR(x) + VK_BAR1_DAUTH_STORE_SIZE) + +/* BAR1 SOTP AUTH and REVID info */ +#define VK_BAR1_SOTP_REVID_BASE_ADDR 0x6340 +#define VK_BAR1_SOTP_REVID_SIZE 0x10 +#define VK_BAR1_SOTP_REVID_MAX 2 +#define VK_BAR1_SOTP_REVID_ADDR(x) \ + (VK_BAR1_SOTP_REVID_BASE_ADDR + (x) * VK_BAR1_SOTP_REVID_SIZE) + +/* VK device supports a maximum of 3 bars */ +#define MAX_BAR 3 + +/* default number of msg blk for inband SGL */ +#define BCM_VK_DEF_IB_SGL_BLK_LEN 16 +#define BCM_VK_IB_SGL_BLK_MAX 24 + +enum pci_barno { + BAR_0 = 0, + BAR_1, + BAR_2 +}; + +#ifdef CONFIG_BCM_VK_TTY +#define BCM_VK_NUM_TTY 2 +#else +#define BCM_VK_NUM_TTY 0 +#endif + +struct bcm_vk_tty { + struct tty_port port; + u32 to_offset; /* bar offset to use */ + u32 to_size; /* to VK buffer size */ + u32 wr; /* write offset shadow */ + u32 from_offset; /* bar offset to use */ + u32 from_size; /* from VK buffer size */ + u32 rd; /* read offset shadow */ + pid_t pid; + bool irq_enabled; + bool is_opened; /* tracks tty open/close */ +}; + +/* VK device max power state, supports 3, full, reduced and low */ +#define MAX_OPP 3 +#define MAX_CARD_INFO_TAG_SIZE 64 + +struct bcm_vk_card_info { + u32 version; + char os_tag[MAX_CARD_INFO_TAG_SIZE]; + char cmpt_tag[MAX_CARD_INFO_TAG_SIZE]; + u32 cpu_freq_mhz; + u32 cpu_scale[MAX_OPP]; + u32 ddr_freq_mhz; + u32 ddr_size_MB; + u32 video_core_freq_mhz; +}; + +/* DAUTH related info */ +struct bcm_vk_dauth_key { + char store[VK_BAR1_DAUTH_STORE_SIZE]; + char valid[VK_BAR1_DAUTH_VALID_SIZE]; +}; + +struct bcm_vk_dauth_info { + struct bcm_vk_dauth_key keys[VK_BAR1_DAUTH_MAX]; +}; + +/* + * Control structure of logging messages from the card. This + * buffer is for logmsg that comes from vk + */ +struct bcm_vk_peer_log { + u32 rd_idx; + u32 wr_idx; + u32 buf_size; + u32 mask; + char data[0]; +}; + +/* max buf size allowed */ +#define BCM_VK_PEER_LOG_BUF_MAX SZ_16K +/* max size per line of peer log */ +#define BCM_VK_PEER_LOG_LINE_MAX 256 + +/* + * single entry for processing type + utilization + */ +#define BCM_VK_PROC_TYPE_TAG_LEN 8 +struct bcm_vk_proc_mon_entry_t { + char tag[BCM_VK_PROC_TYPE_TAG_LEN]; + u32 used; + u32 max; /**< max capacity */ +}; + +/** + * Structure for run time utilization + */ +#define BCM_VK_PROC_MON_MAX 8 /* max entries supported */ +struct bcm_vk_proc_mon_info { + u32 num; /**< no of entries */ + u32 entry_size; /**< per entry size */ + struct bcm_vk_proc_mon_entry_t entries[BCM_VK_PROC_MON_MAX]; +}; + +struct bcm_vk_hb_ctrl { + struct timer_list timer; + u32 last_uptime; + u32 lost_cnt; +}; + +struct bcm_vk_alert { + u16 flags; + u16 notfs; +}; + +/* some alert counters that the driver will keep track */ +struct bcm_vk_alert_cnts { + u16 ecc; + u16 uecc; +}; + +struct bcm_vk { + struct pci_dev *pdev; + void __iomem *bar[MAX_BAR]; + int num_irqs; + + struct bcm_vk_card_info card_info; + struct bcm_vk_proc_mon_info proc_mon_info; + struct bcm_vk_dauth_info dauth_info; + + /* mutex to protect the ioctls */ + struct mutex mutex; + struct miscdevice miscdev; + int devid; /* dev id allocated */ + +#ifdef CONFIG_BCM_VK_TTY + struct tty_driver *tty_drv; + struct timer_list serial_timer; + struct bcm_vk_tty tty[BCM_VK_NUM_TTY]; + struct workqueue_struct *tty_wq_thread; + struct work_struct tty_wq_work; +#endif + + /* Reference-counting to handle file operations */ + struct kref kref; + + spinlock_t msg_id_lock; /* Spinlock for msg_id */ + u16 msg_id; + DECLARE_BITMAP(bmap, VK_MSG_ID_BITMAP_SIZE); + spinlock_t ctx_lock; /* Spinlock for component context */ + struct bcm_vk_ctx ctx[VK_CMPT_CTX_MAX]; + struct bcm_vk_ht_entry pid_ht[VK_PID_HT_SZ]; + pid_t reset_pid; /* process that issue reset */ + + atomic_t msgq_inited; /* indicate if info has been synced with vk */ + struct bcm_vk_msg_chan to_v_msg_chan; + struct bcm_vk_msg_chan to_h_msg_chan; + + struct workqueue_struct *wq_thread; + struct work_struct wq_work; /* work queue for deferred job */ + unsigned long wq_offload[1]; /* various flags on wq requested */ + void *tdma_vaddr; /* test dma segment virtual addr */ + dma_addr_t tdma_addr; /* test dma segment bus addr */ + + struct notifier_block panic_nb; + u32 ib_sgl_size; /* size allocated for inband sgl insertion */ + + /* heart beat mechanism control structure */ + struct bcm_vk_hb_ctrl hb_ctrl; + /* house-keeping variable of error logs */ + spinlock_t host_alert_lock; /* protection to access host_alert struct */ + struct bcm_vk_alert host_alert; + struct bcm_vk_alert peer_alert; /* bits set by the card */ + struct bcm_vk_alert_cnts alert_cnts; + + /* offset of the peer log control in BAR2 */ + u32 peerlog_off; + struct bcm_vk_peer_log peerlog_info; /* record of peer log info */ + /* offset of processing monitoring info in BAR2 */ + u32 proc_mon_off; +}; + +/* wq offload work items bits definitions */ +enum bcm_vk_wq_offload_flags { + BCM_VK_WQ_DWNLD_PEND = 0, + BCM_VK_WQ_DWNLD_AUTO = 1, + BCM_VK_WQ_NOTF_PEND = 2, +}; + +/* a macro to get an individual field with mask and shift */ +#define BCM_VK_EXTRACT_FIELD(_field, _reg, _mask, _shift) \ + (_field = (((_reg) >> (_shift)) & (_mask))) + +struct bcm_vk_entry { + const u32 mask; + const u32 exp_val; + const char *str; +}; + +/* alerts that could be generated from peer */ +#define BCM_VK_PEER_ERR_NUM 12 +extern struct bcm_vk_entry const bcm_vk_peer_err[BCM_VK_PEER_ERR_NUM]; +/* alerts detected by the host */ +#define BCM_VK_HOST_ERR_NUM 3 +extern struct bcm_vk_entry const bcm_vk_host_err[BCM_VK_HOST_ERR_NUM]; + +/* + * check if PCIe interface is down on read. Use it when it is + * certain that _val should never be all ones. + */ +#define BCM_VK_INTF_IS_DOWN(val) ((val) == 0xffffffff) + +static inline u32 vkread32(struct bcm_vk *vk, enum pci_barno bar, u64 offset) +{ + return readl(vk->bar[bar] + offset); +} + +static inline void vkwrite32(struct bcm_vk *vk, + u32 value, + enum pci_barno bar, + u64 offset) +{ + writel(value, vk->bar[bar] + offset); +} + +static inline u8 vkread8(struct bcm_vk *vk, enum pci_barno bar, u64 offset) +{ + return readb(vk->bar[bar] + offset); +} + +static inline void vkwrite8(struct bcm_vk *vk, + u8 value, + enum pci_barno bar, + u64 offset) +{ + writeb(value, vk->bar[bar] + offset); +} + +static inline bool bcm_vk_msgq_marker_valid(struct bcm_vk *vk) +{ + u32 rdy_marker = 0; + u32 fw_status; + + fw_status = vkread32(vk, BAR_0, VK_BAR_FWSTS); + + if ((fw_status & VK_FWSTS_READY) == VK_FWSTS_READY) + rdy_marker = vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY); + + return (rdy_marker == VK_BAR1_MSGQ_RDY_MARKER); +} + +int bcm_vk_open(struct inode *inode, struct file *p_file); +ssize_t bcm_vk_read(struct file *p_file, char __user *buf, size_t count, + loff_t *f_pos); +ssize_t bcm_vk_write(struct file *p_file, const char __user *buf, + size_t count, loff_t *f_pos); +__poll_t bcm_vk_poll(struct file *p_file, struct poll_table_struct *wait); +int bcm_vk_release(struct inode *inode, struct file *p_file); +void bcm_vk_release_data(struct kref *kref); +irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id); +irqreturn_t bcm_vk_notf_irqhandler(int irq, void *dev_id); +irqreturn_t bcm_vk_tty_irqhandler(int irq, void *dev_id); +int bcm_vk_msg_init(struct bcm_vk *vk); +void bcm_vk_msg_remove(struct bcm_vk *vk); +void bcm_vk_drain_msg_on_reset(struct bcm_vk *vk); +int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync); +void bcm_vk_blk_drv_access(struct bcm_vk *vk); +s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk); +int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type, + const pid_t pid, const u32 q_num); +void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val); +int bcm_vk_auto_load_all_images(struct bcm_vk *vk); +void bcm_vk_hb_init(struct bcm_vk *vk); +void bcm_vk_hb_deinit(struct bcm_vk *vk); +void bcm_vk_handle_notf(struct bcm_vk *vk); +bool bcm_vk_drv_access_ok(struct bcm_vk *vk); +void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask); + +#ifdef CONFIG_BCM_VK_TTY +int bcm_vk_tty_init(struct bcm_vk *vk, char *name); +void bcm_vk_tty_exit(struct bcm_vk *vk); +void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk); +void bcm_vk_tty_wq_exit(struct bcm_vk *vk); + +static inline void bcm_vk_tty_set_irq_enabled(struct bcm_vk *vk, int index) +{ + vk->tty[index].irq_enabled = true; +} +#else +static inline int bcm_vk_tty_init(struct bcm_vk *vk, char *name) +{ + return 0; +} + +static inline void bcm_vk_tty_exit(struct bcm_vk *vk) +{ +} + +static inline void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk) +{ +} + +static inline void bcm_vk_tty_wq_exit(struct bcm_vk *vk) +{ +} + +static inline void bcm_vk_tty_set_irq_enabled(struct bcm_vk *vk, int index) +{ +} +#endif /* CONFIG_BCM_VK_TTY */ + +#endif diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c new file mode 100644 index 000000000000..6bfea3210389 --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -0,0 +1,1652 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2018-2020 Broadcom. + */ + +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/firmware.h> +#include <linux/fs.h> +#include <linux/idr.h> +#include <linux/interrupt.h> +#include <linux/kref.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/pci.h> +#include <linux/pci_regs.h> +#include <uapi/linux/misc/bcm_vk.h> + +#include "bcm_vk.h" + +#define PCI_DEVICE_ID_VALKYRIE 0x5e87 +#define PCI_DEVICE_ID_VIPER 0x5e88 + +static DEFINE_IDA(bcm_vk_ida); + +enum soc_idx { + VALKYRIE_A0 = 0, + VALKYRIE_B0, + VIPER, + VK_IDX_INVALID +}; + +enum img_idx { + IMG_PRI = 0, + IMG_SEC, + IMG_PER_TYPE_MAX +}; + +struct load_image_entry { + const u32 image_type; + const char *image_name[IMG_PER_TYPE_MAX]; +}; + +#define NUM_BOOT_STAGES 2 +/* default firmware images names */ +static const struct load_image_entry image_tab[][NUM_BOOT_STAGES] = { + [VALKYRIE_A0] = { + {VK_IMAGE_TYPE_BOOT1, {"vk_a0-boot1.bin", "vk-boot1.bin"}}, + {VK_IMAGE_TYPE_BOOT2, {"vk_a0-boot2.bin", "vk-boot2.bin"}} + }, + [VALKYRIE_B0] = { + {VK_IMAGE_TYPE_BOOT1, {"vk_b0-boot1.bin", "vk-boot1.bin"}}, + {VK_IMAGE_TYPE_BOOT2, {"vk_b0-boot2.bin", "vk-boot2.bin"}} + }, + + [VIPER] = { + {VK_IMAGE_TYPE_BOOT1, {"vp-boot1.bin", ""}}, + {VK_IMAGE_TYPE_BOOT2, {"vp-boot2.bin", ""}} + }, +}; + +/* Location of memory base addresses of interest in BAR1 */ +/* Load Boot1 to start of ITCM */ +#define BAR1_CODEPUSH_BASE_BOOT1 0x100000 + +/* Allow minimum 1s for Load Image timeout responses */ +#define LOAD_IMAGE_TIMEOUT_MS (1 * MSEC_PER_SEC) + +/* Image startup timeouts */ +#define BOOT1_STARTUP_TIMEOUT_MS (5 * MSEC_PER_SEC) +#define BOOT2_STARTUP_TIMEOUT_MS (10 * MSEC_PER_SEC) + +/* 1ms wait for checking the transfer complete status */ +#define TXFR_COMPLETE_TIMEOUT_MS 1 + +/* MSIX usages */ +#define VK_MSIX_MSGQ_MAX 3 +#define VK_MSIX_NOTF_MAX 1 +#define VK_MSIX_TTY_MAX BCM_VK_NUM_TTY +#define VK_MSIX_IRQ_MAX (VK_MSIX_MSGQ_MAX + VK_MSIX_NOTF_MAX + \ + VK_MSIX_TTY_MAX) +#define VK_MSIX_IRQ_MIN_REQ (VK_MSIX_MSGQ_MAX + VK_MSIX_NOTF_MAX) + +/* Number of bits set in DMA mask*/ +#define BCM_VK_DMA_BITS 64 + +/* Ucode boot wait time */ +#define BCM_VK_UCODE_BOOT_US (100 * USEC_PER_MSEC) +/* 50% margin */ +#define BCM_VK_UCODE_BOOT_MAX_US ((BCM_VK_UCODE_BOOT_US * 3) >> 1) + +/* deinit time for the card os after receiving doorbell */ +#define BCM_VK_DEINIT_TIME_MS (2 * MSEC_PER_SEC) + +/* + * module parameters + */ +static bool auto_load = true; +module_param(auto_load, bool, 0444); +MODULE_PARM_DESC(auto_load, + "Load images automatically at PCIe probe time.\n"); +static uint nr_scratch_pages = VK_BAR1_SCRATCH_DEF_NR_PAGES; +module_param(nr_scratch_pages, uint, 0444); +MODULE_PARM_DESC(nr_scratch_pages, + "Number of pre allocated DMAable coherent pages.\n"); +static uint nr_ib_sgl_blk = BCM_VK_DEF_IB_SGL_BLK_LEN; +module_param(nr_ib_sgl_blk, uint, 0444); +MODULE_PARM_DESC(nr_ib_sgl_blk, + "Number of in-band msg blks for short SGL.\n"); + +/* + * alerts that could be generated from peer + */ +const struct bcm_vk_entry bcm_vk_peer_err[BCM_VK_PEER_ERR_NUM] = { + {ERR_LOG_UECC, ERR_LOG_UECC, "uecc"}, + {ERR_LOG_SSIM_BUSY, ERR_LOG_SSIM_BUSY, "ssim_busy"}, + {ERR_LOG_AFBC_BUSY, ERR_LOG_AFBC_BUSY, "afbc_busy"}, + {ERR_LOG_HIGH_TEMP_ERR, ERR_LOG_HIGH_TEMP_ERR, "high_temp"}, + {ERR_LOG_WDOG_TIMEOUT, ERR_LOG_WDOG_TIMEOUT, "wdog_timeout"}, + {ERR_LOG_SYS_FAULT, ERR_LOG_SYS_FAULT, "sys_fault"}, + {ERR_LOG_RAMDUMP, ERR_LOG_RAMDUMP, "ramdump"}, + {ERR_LOG_COP_WDOG_TIMEOUT, ERR_LOG_COP_WDOG_TIMEOUT, + "cop_wdog_timeout"}, + {ERR_LOG_MEM_ALLOC_FAIL, ERR_LOG_MEM_ALLOC_FAIL, "malloc_fail warn"}, + {ERR_LOG_LOW_TEMP_WARN, ERR_LOG_LOW_TEMP_WARN, "low_temp warn"}, + {ERR_LOG_ECC, ERR_LOG_ECC, "ecc"}, + {ERR_LOG_IPC_DWN, ERR_LOG_IPC_DWN, "ipc_down"}, +}; + +/* alerts detected by the host */ +const struct bcm_vk_entry bcm_vk_host_err[BCM_VK_HOST_ERR_NUM] = { + {ERR_LOG_HOST_PCIE_DWN, ERR_LOG_HOST_PCIE_DWN, "PCIe_down"}, + {ERR_LOG_HOST_HB_FAIL, ERR_LOG_HOST_HB_FAIL, "hb_fail"}, + {ERR_LOG_HOST_INTF_V_FAIL, ERR_LOG_HOST_INTF_V_FAIL, "intf_ver_fail"}, +}; + +irqreturn_t bcm_vk_notf_irqhandler(int irq, void *dev_id) +{ + struct bcm_vk *vk = dev_id; + + if (!bcm_vk_drv_access_ok(vk)) { + dev_err(&vk->pdev->dev, + "Interrupt %d received when msgq not inited\n", irq); + goto skip_schedule_work; + } + + /* if notification is not pending, set bit and schedule work */ + if (test_and_set_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload) == 0) + queue_work(vk->wq_thread, &vk->wq_work); + +skip_schedule_work: + return IRQ_HANDLED; +} + +static int bcm_vk_intf_ver_chk(struct bcm_vk *vk) +{ + struct device *dev = &vk->pdev->dev; + u32 reg; + u16 major, minor; + int ret = 0; + + /* read interface register */ + reg = vkread32(vk, BAR_0, BAR_INTF_VER); + major = (reg >> BAR_INTF_VER_MAJOR_SHIFT) & BAR_INTF_VER_MASK; + minor = reg & BAR_INTF_VER_MASK; + + /* + * if major number is 0, it is pre-release and it would be allowed + * to continue, else, check versions accordingly + */ + if (!major) { + dev_warn(dev, "Pre-release major.minor=%d.%d - drv %d.%d\n", + major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR); + } else if (major != SEMANTIC_MAJOR) { + dev_err(dev, + "Intf major.minor=%d.%d rejected - drv %d.%d\n", + major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR); + bcm_vk_set_host_alert(vk, ERR_LOG_HOST_INTF_V_FAIL); + ret = -EPFNOSUPPORT; + } else { + dev_dbg(dev, + "Intf major.minor=%d.%d passed - drv %d.%d\n", + major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR); + } + return ret; +} + +static void bcm_vk_log_notf(struct bcm_vk *vk, + struct bcm_vk_alert *alert, + struct bcm_vk_entry const *entry_tab, + const u32 table_size) +{ + u32 i; + u32 masked_val, latched_val; + struct bcm_vk_entry const *entry; + u32 reg; + u16 ecc_mem_err, uecc_mem_err; + struct device *dev = &vk->pdev->dev; + + for (i = 0; i < table_size; i++) { + entry = &entry_tab[i]; + masked_val = entry->mask & alert->notfs; + latched_val = entry->mask & alert->flags; + + if (masked_val == ERR_LOG_UECC) { + /* + * if there is difference between stored cnt and it + * is greater than threshold, log it. + */ + reg = vkread32(vk, BAR_0, BAR_CARD_ERR_MEM); + BCM_VK_EXTRACT_FIELD(uecc_mem_err, reg, + BCM_VK_MEM_ERR_FIELD_MASK, + BCM_VK_UECC_MEM_ERR_SHIFT); + if ((uecc_mem_err != vk->alert_cnts.uecc) && + (uecc_mem_err >= BCM_VK_UECC_THRESHOLD)) + dev_info(dev, + "ALERT! %s.%d uecc RAISED - ErrCnt %d\n", + DRV_MODULE_NAME, vk->devid, + uecc_mem_err); + vk->alert_cnts.uecc = uecc_mem_err; + } else if (masked_val == ERR_LOG_ECC) { + reg = vkread32(vk, BAR_0, BAR_CARD_ERR_MEM); + BCM_VK_EXTRACT_FIELD(ecc_mem_err, reg, + BCM_VK_MEM_ERR_FIELD_MASK, + BCM_VK_ECC_MEM_ERR_SHIFT); + if ((ecc_mem_err != vk->alert_cnts.ecc) && + (ecc_mem_err >= BCM_VK_ECC_THRESHOLD)) + dev_info(dev, "ALERT! %s.%d ecc RAISED - ErrCnt %d\n", + DRV_MODULE_NAME, vk->devid, + ecc_mem_err); + vk->alert_cnts.ecc = ecc_mem_err; + } else if (masked_val != latched_val) { + /* print a log as info */ + dev_info(dev, "ALERT! %s.%d %s %s\n", + DRV_MODULE_NAME, vk->devid, entry->str, + masked_val ? "RAISED" : "CLEARED"); + } + } +} + +static void bcm_vk_dump_peer_log(struct bcm_vk *vk) +{ + struct bcm_vk_peer_log log; + struct bcm_vk_peer_log *log_info = &vk->peerlog_info; + char loc_buf[BCM_VK_PEER_LOG_LINE_MAX]; + int cnt; + struct device *dev = &vk->pdev->dev; + unsigned int data_offset; + + memcpy_fromio(&log, vk->bar[BAR_2] + vk->peerlog_off, sizeof(log)); + + dev_dbg(dev, "Peer PANIC: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n", + log.buf_size, log.mask, log.rd_idx, log.wr_idx); + + if (!log_info->buf_size) { + dev_err(dev, "Peer log dump disabled - skipped!\n"); + return; + } + + /* perform range checking for rd/wr idx */ + if ((log.rd_idx > log_info->mask) || + (log.wr_idx > log_info->mask) || + (log.buf_size != log_info->buf_size) || + (log.mask != log_info->mask)) { + dev_err(dev, + "Corrupted Ptrs: Size 0x%x(0x%x) Mask 0x%x(0x%x) [Rd Wr] = [%d %d], skip log dump.\n", + log_info->buf_size, log.buf_size, + log_info->mask, log.mask, + log.rd_idx, log.wr_idx); + return; + } + + cnt = 0; + data_offset = vk->peerlog_off + sizeof(struct bcm_vk_peer_log); + loc_buf[BCM_VK_PEER_LOG_LINE_MAX - 1] = '\0'; + while (log.rd_idx != log.wr_idx) { + loc_buf[cnt] = vkread8(vk, BAR_2, data_offset + log.rd_idx); + + if ((loc_buf[cnt] == '\0') || + (cnt == (BCM_VK_PEER_LOG_LINE_MAX - 1))) { + dev_err(dev, "%s", loc_buf); + cnt = 0; + } else { + cnt++; + } + log.rd_idx = (log.rd_idx + 1) & log.mask; + } + /* update rd idx at the end */ + vkwrite32(vk, log.rd_idx, BAR_2, + vk->peerlog_off + offsetof(struct bcm_vk_peer_log, rd_idx)); +} + +void bcm_vk_handle_notf(struct bcm_vk *vk) +{ + u32 reg; + struct bcm_vk_alert alert; + bool intf_down; + unsigned long flags; + + /* handle peer alerts and then locally detected ones */ + reg = vkread32(vk, BAR_0, BAR_CARD_ERR_LOG); + intf_down = BCM_VK_INTF_IS_DOWN(reg); + if (!intf_down) { + vk->peer_alert.notfs = reg; + bcm_vk_log_notf(vk, &vk->peer_alert, bcm_vk_peer_err, + ARRAY_SIZE(bcm_vk_peer_err)); + vk->peer_alert.flags = vk->peer_alert.notfs; + } else { + /* turn off access */ + bcm_vk_blk_drv_access(vk); + } + + /* check and make copy of alert with lock and then free lock */ + spin_lock_irqsave(&vk->host_alert_lock, flags); + if (intf_down) + vk->host_alert.notfs |= ERR_LOG_HOST_PCIE_DWN; + + alert = vk->host_alert; + vk->host_alert.flags = vk->host_alert.notfs; + spin_unlock_irqrestore(&vk->host_alert_lock, flags); + + /* call display with copy */ + bcm_vk_log_notf(vk, &alert, bcm_vk_host_err, + ARRAY_SIZE(bcm_vk_host_err)); + + /* + * If it is a sys fault or heartbeat timeout, we would like extract + * log msg from the card so that we would know what is the last fault + */ + if (!intf_down && + ((vk->host_alert.flags & ERR_LOG_HOST_HB_FAIL) || + (vk->peer_alert.flags & ERR_LOG_SYS_FAULT))) + bcm_vk_dump_peer_log(vk); +} + +static inline int bcm_vk_wait(struct bcm_vk *vk, enum pci_barno bar, + u64 offset, u32 mask, u32 value, + unsigned long timeout_ms) +{ + struct device *dev = &vk->pdev->dev; + unsigned long start_time; + unsigned long timeout; + u32 rd_val, boot_status; + + start_time = jiffies; + timeout = start_time + msecs_to_jiffies(timeout_ms); + + do { + rd_val = vkread32(vk, bar, offset); + dev_dbg(dev, "BAR%d Offset=0x%llx: 0x%x\n", + bar, offset, rd_val); + + /* check for any boot err condition */ + boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); + if (boot_status & BOOT_ERR_MASK) { + dev_err(dev, "Boot Err 0x%x, progress 0x%x after %d ms\n", + (boot_status & BOOT_ERR_MASK) >> BOOT_ERR_SHIFT, + boot_status & BOOT_PROG_MASK, + jiffies_to_msecs(jiffies - start_time)); + return -EFAULT; + } + + if (time_after(jiffies, timeout)) + return -ETIMEDOUT; + + cpu_relax(); + cond_resched(); + } while ((rd_val & mask) != value); + + return 0; +} + +static void bcm_vk_get_card_info(struct bcm_vk *vk) +{ + struct device *dev = &vk->pdev->dev; + u32 offset; + int i; + u8 *dst; + struct bcm_vk_card_info *info = &vk->card_info; + + /* first read the offset from spare register */ + offset = vkread32(vk, BAR_0, BAR_CARD_STATIC_INFO); + offset &= (pci_resource_len(vk->pdev, BAR_2 * 2) - 1); + + /* based on the offset, read info to internal card info structure */ + dst = (u8 *)info; + for (i = 0; i < sizeof(*info); i++) + *dst++ = vkread8(vk, BAR_2, offset++); + +#define CARD_INFO_LOG_FMT "version : %x\n" \ + "os_tag : %s\n" \ + "cmpt_tag : %s\n" \ + "cpu_freq : %d MHz\n" \ + "cpu_scale : %d full, %d lowest\n" \ + "ddr_freq : %d MHz\n" \ + "ddr_size : %d MB\n" \ + "video_freq: %d MHz\n" + dev_dbg(dev, CARD_INFO_LOG_FMT, info->version, info->os_tag, + info->cmpt_tag, info->cpu_freq_mhz, info->cpu_scale[0], + info->cpu_scale[MAX_OPP - 1], info->ddr_freq_mhz, + info->ddr_size_MB, info->video_core_freq_mhz); + + /* + * get the peer log pointer, only need the offset, and get record + * of the log buffer information which would be used for checking + * before dump, in case the BAR2 memory has been corrupted. + */ + vk->peerlog_off = offset; + memcpy_fromio(&vk->peerlog_info, vk->bar[BAR_2] + vk->peerlog_off, + sizeof(vk->peerlog_info)); + + /* + * Do a range checking and if out of bound, the record will be zeroed + * which guarantees that nothing would be dumped. In other words, + * peer dump is disabled. + */ + if ((vk->peerlog_info.buf_size > BCM_VK_PEER_LOG_BUF_MAX) || + (vk->peerlog_info.mask != (vk->peerlog_info.buf_size - 1)) || + (vk->peerlog_info.rd_idx > vk->peerlog_info.mask) || + (vk->peerlog_info.wr_idx > vk->peerlog_info.mask)) { + dev_err(dev, "Peer log disabled - range error: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n", + vk->peerlog_info.buf_size, + vk->peerlog_info.mask, + vk->peerlog_info.rd_idx, + vk->peerlog_info.wr_idx); + memset(&vk->peerlog_info, 0, sizeof(vk->peerlog_info)); + } else { + dev_dbg(dev, "Peer log: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n", + vk->peerlog_info.buf_size, + vk->peerlog_info.mask, + vk->peerlog_info.rd_idx, + vk->peerlog_info.wr_idx); + } +} + +static void bcm_vk_get_proc_mon_info(struct bcm_vk *vk) +{ + struct device *dev = &vk->pdev->dev; + struct bcm_vk_proc_mon_info *mon = &vk->proc_mon_info; + u32 num, entry_size, offset, buf_size; + u8 *dst; + + /* calculate offset which is based on peerlog offset */ + buf_size = vkread32(vk, BAR_2, + vk->peerlog_off + + offsetof(struct bcm_vk_peer_log, buf_size)); + offset = vk->peerlog_off + sizeof(struct bcm_vk_peer_log) + + buf_size; + + /* first read the num and entry size */ + num = vkread32(vk, BAR_2, offset); + entry_size = vkread32(vk, BAR_2, offset + sizeof(num)); + + /* check for max allowed */ + if (num > BCM_VK_PROC_MON_MAX) { + dev_err(dev, "Processing monitoring entry %d exceeds max %d\n", + num, BCM_VK_PROC_MON_MAX); + return; + } + mon->num = num; + mon->entry_size = entry_size; + + vk->proc_mon_off = offset; + + /* read it once that will capture those static info */ + dst = (u8 *)&mon->entries[0]; + offset += sizeof(num) + sizeof(entry_size); + memcpy_fromio(dst, vk->bar[BAR_2] + offset, num * entry_size); +} + +static int bcm_vk_sync_card_info(struct bcm_vk *vk) +{ + u32 rdy_marker = vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY); + + /* check for marker, but allow diags mode to skip sync */ + if (!bcm_vk_msgq_marker_valid(vk)) + return (rdy_marker == VK_BAR1_DIAG_RDY_MARKER ? 0 : -EINVAL); + + /* + * Write down scratch addr which is used for DMA. For + * signed part, BAR1 is accessible only after boot2 has come + * up + */ + if (vk->tdma_addr) { + vkwrite32(vk, (u64)vk->tdma_addr >> 32, BAR_1, + VK_BAR1_SCRATCH_OFF_HI); + vkwrite32(vk, (u32)vk->tdma_addr, BAR_1, + VK_BAR1_SCRATCH_OFF_LO); + vkwrite32(vk, nr_scratch_pages * PAGE_SIZE, BAR_1, + VK_BAR1_SCRATCH_SZ_ADDR); + } + + /* get static card info, only need to read once */ + bcm_vk_get_card_info(vk); + + /* get the proc mon info once */ + bcm_vk_get_proc_mon_info(vk); + + return 0; +} + +void bcm_vk_blk_drv_access(struct bcm_vk *vk) +{ + int i; + + /* + * kill all the apps except for the process that is resetting. + * If not called during reset, reset_pid will be 0, and all will be + * killed. + */ + spin_lock(&vk->ctx_lock); + + /* set msgq_inited to 0 so that all rd/wr will be blocked */ + atomic_set(&vk->msgq_inited, 0); + + for (i = 0; i < VK_PID_HT_SZ; i++) { + struct bcm_vk_ctx *ctx; + + list_for_each_entry(ctx, &vk->pid_ht[i].head, node) { + if (ctx->pid != vk->reset_pid) { + dev_dbg(&vk->pdev->dev, + "Send kill signal to pid %d\n", + ctx->pid); + kill_pid(find_vpid(ctx->pid), SIGKILL, 1); + } + } + } + bcm_vk_tty_terminate_tty_user(vk); + spin_unlock(&vk->ctx_lock); +} + +static void bcm_vk_buf_notify(struct bcm_vk *vk, void *bufp, + dma_addr_t host_buf_addr, u32 buf_size) +{ + /* update the dma address to the card */ + vkwrite32(vk, (u64)host_buf_addr >> 32, BAR_1, + VK_BAR1_DMA_BUF_OFF_HI); + vkwrite32(vk, (u32)host_buf_addr, BAR_1, + VK_BAR1_DMA_BUF_OFF_LO); + vkwrite32(vk, buf_size, BAR_1, VK_BAR1_DMA_BUF_SZ); +} + +static int bcm_vk_load_image_by_type(struct bcm_vk *vk, u32 load_type, + const char *filename) +{ + struct device *dev = &vk->pdev->dev; + const struct firmware *fw = NULL; + void *bufp = NULL; + size_t max_buf, offset; + int ret; + u64 offset_codepush; + u32 codepush; + u32 value; + dma_addr_t boot_dma_addr; + bool is_stdalone; + + if (load_type == VK_IMAGE_TYPE_BOOT1) { + /* + * After POR, enable VK soft BOOTSRC so bootrom do not clear + * the pushed image (the TCM memories). + */ + value = vkread32(vk, BAR_0, BAR_BOOTSRC_SELECT); + value |= BOOTSRC_SOFT_ENABLE; + vkwrite32(vk, value, BAR_0, BAR_BOOTSRC_SELECT); + + codepush = CODEPUSH_BOOTSTART + CODEPUSH_BOOT1_ENTRY; + offset_codepush = BAR_CODEPUSH_SBL; + + /* Write a 1 to request SRAM open bit */ + vkwrite32(vk, CODEPUSH_BOOTSTART, BAR_0, offset_codepush); + + /* Wait for VK to respond */ + ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, SRAM_OPEN, + SRAM_OPEN, LOAD_IMAGE_TIMEOUT_MS); + if (ret < 0) { + dev_err(dev, "boot1 wait SRAM err - ret(%d)\n", ret); + goto err_buf_out; + } + + max_buf = SZ_256K; + bufp = dma_alloc_coherent(dev, + max_buf, + &boot_dma_addr, GFP_KERNEL); + if (!bufp) { + dev_err(dev, "Error allocating 0x%zx\n", max_buf); + ret = -ENOMEM; + goto err_buf_out; + } + } else if (load_type == VK_IMAGE_TYPE_BOOT2) { + codepush = CODEPUSH_BOOT2_ENTRY; + offset_codepush = BAR_CODEPUSH_SBI; + + /* Wait for VK to respond */ + ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, DDR_OPEN, + DDR_OPEN, LOAD_IMAGE_TIMEOUT_MS); + if (ret < 0) { + dev_err(dev, "boot2 wait DDR open error - ret(%d)\n", + ret); + goto err_buf_out; + } + + max_buf = SZ_4M; + bufp = dma_alloc_coherent(dev, + max_buf, + &boot_dma_addr, GFP_KERNEL); + if (!bufp) { + dev_err(dev, "Error allocating 0x%zx\n", max_buf); + ret = -ENOMEM; + goto err_buf_out; + } + + bcm_vk_buf_notify(vk, bufp, boot_dma_addr, max_buf); + } else { + dev_err(dev, "Error invalid image type 0x%x\n", load_type); + ret = -EINVAL; + goto err_buf_out; + } + + offset = 0; + ret = request_partial_firmware_into_buf(&fw, filename, dev, + bufp, max_buf, offset); + if (ret) { + dev_err(dev, "Error %d requesting firmware file: %s\n", + ret, filename); + goto err_firmware_out; + } + dev_dbg(dev, "size=0x%zx\n", fw->size); + if (load_type == VK_IMAGE_TYPE_BOOT1) + memcpy_toio(vk->bar[BAR_1] + BAR1_CODEPUSH_BASE_BOOT1, + bufp, + fw->size); + + dev_dbg(dev, "Signaling 0x%x to 0x%llx\n", codepush, offset_codepush); + vkwrite32(vk, codepush, BAR_0, offset_codepush); + + if (load_type == VK_IMAGE_TYPE_BOOT1) { + u32 boot_status; + + /* wait until done */ + ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, + BOOT1_RUNNING, + BOOT1_RUNNING, + BOOT1_STARTUP_TIMEOUT_MS); + + boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); + is_stdalone = !BCM_VK_INTF_IS_DOWN(boot_status) && + (boot_status & BOOT_STDALONE_RUNNING); + if (ret && !is_stdalone) { + dev_err(dev, + "Timeout %ld ms waiting for boot1 to come up - ret(%d)\n", + BOOT1_STARTUP_TIMEOUT_MS, ret); + goto err_firmware_out; + } else if (is_stdalone) { + u32 reg; + + reg = vkread32(vk, BAR_0, BAR_BOOT1_STDALONE_PROGRESS); + if ((reg & BOOT1_STDALONE_PROGRESS_MASK) == + BOOT1_STDALONE_SUCCESS) { + dev_info(dev, "Boot1 standalone success\n"); + ret = 0; + } else { + dev_err(dev, "Timeout %ld ms - Boot1 standalone failure\n", + BOOT1_STARTUP_TIMEOUT_MS); + ret = -EINVAL; + goto err_firmware_out; + } + } + } else if (load_type == VK_IMAGE_TYPE_BOOT2) { + unsigned long timeout; + + timeout = jiffies + msecs_to_jiffies(LOAD_IMAGE_TIMEOUT_MS); + + /* To send more data to VK than max_buf allowed at a time */ + do { + /* + * Check for ack from card. when Ack is received, + * it means all the data is received by card. + * Exit the loop after ack is received. + */ + ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, + FW_LOADER_ACK_RCVD_ALL_DATA, + FW_LOADER_ACK_RCVD_ALL_DATA, + TXFR_COMPLETE_TIMEOUT_MS); + if (ret == 0) { + dev_dbg(dev, "Exit boot2 download\n"); + break; + } else if (ret == -EFAULT) { + dev_err(dev, "Error detected during ACK waiting"); + goto err_firmware_out; + } + + /* exit the loop, if there is no response from card */ + if (time_after(jiffies, timeout)) { + dev_err(dev, "Error. No reply from card\n"); + ret = -ETIMEDOUT; + goto err_firmware_out; + } + + /* Wait for VK to open BAR space to copy new data */ + ret = bcm_vk_wait(vk, BAR_0, offset_codepush, + codepush, 0, + TXFR_COMPLETE_TIMEOUT_MS); + if (ret == 0) { + offset += max_buf; + ret = request_partial_firmware_into_buf + (&fw, + filename, + dev, bufp, + max_buf, + offset); + if (ret) { + dev_err(dev, + "Error %d requesting firmware file: %s offset: 0x%zx\n", + ret, filename, offset); + goto err_firmware_out; + } + dev_dbg(dev, "size=0x%zx\n", fw->size); + dev_dbg(dev, "Signaling 0x%x to 0x%llx\n", + codepush, offset_codepush); + vkwrite32(vk, codepush, BAR_0, offset_codepush); + /* reload timeout after every codepush */ + timeout = jiffies + + msecs_to_jiffies(LOAD_IMAGE_TIMEOUT_MS); + } else if (ret == -EFAULT) { + dev_err(dev, "Error detected waiting for transfer\n"); + goto err_firmware_out; + } + } while (1); + + /* wait for fw status bits to indicate app ready */ + ret = bcm_vk_wait(vk, BAR_0, VK_BAR_FWSTS, + VK_FWSTS_READY, + VK_FWSTS_READY, + BOOT2_STARTUP_TIMEOUT_MS); + if (ret < 0) { + dev_err(dev, "Boot2 not ready - ret(%d)\n", ret); + goto err_firmware_out; + } + + is_stdalone = vkread32(vk, BAR_0, BAR_BOOT_STATUS) & + BOOT_STDALONE_RUNNING; + if (!is_stdalone) { + ret = bcm_vk_intf_ver_chk(vk); + if (ret) { + dev_err(dev, "failure in intf version check\n"); + goto err_firmware_out; + } + + /* + * Next, initialize Message Q if we are loading boot2. + * Do a force sync + */ + ret = bcm_vk_sync_msgq(vk, true); + if (ret) { + dev_err(dev, "Boot2 Error reading comm msg Q info\n"); + ret = -EIO; + goto err_firmware_out; + } + + /* sync & channel other info */ + ret = bcm_vk_sync_card_info(vk); + if (ret) { + dev_err(dev, "Syncing Card Info failure\n"); + goto err_firmware_out; + } + } + } + +err_firmware_out: + release_firmware(fw); + +err_buf_out: + if (bufp) + dma_free_coherent(dev, max_buf, bufp, boot_dma_addr); + + return ret; +} + +static u32 bcm_vk_next_boot_image(struct bcm_vk *vk) +{ + u32 boot_status; + u32 fw_status; + u32 load_type = 0; /* default for unknown */ + + boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); + fw_status = vkread32(vk, BAR_0, VK_BAR_FWSTS); + + if (!BCM_VK_INTF_IS_DOWN(boot_status) && (boot_status & SRAM_OPEN)) + load_type = VK_IMAGE_TYPE_BOOT1; + else if (boot_status == BOOT1_RUNNING) + load_type = VK_IMAGE_TYPE_BOOT2; + + /* Log status so that we know different stages */ + dev_info(&vk->pdev->dev, + "boot-status value for next image: 0x%x : fw-status 0x%x\n", + boot_status, fw_status); + + return load_type; +} + +static enum soc_idx get_soc_idx(struct bcm_vk *vk) +{ + struct pci_dev *pdev = vk->pdev; + enum soc_idx idx = VK_IDX_INVALID; + u32 rev; + static enum soc_idx const vk_soc_tab[] = { VALKYRIE_A0, VALKYRIE_B0 }; + + switch (pdev->device) { + case PCI_DEVICE_ID_VALKYRIE: + /* get the chip id to decide sub-class */ + rev = MAJOR_SOC_REV(vkread32(vk, BAR_0, BAR_CHIP_ID)); + if (rev < ARRAY_SIZE(vk_soc_tab)) { + idx = vk_soc_tab[rev]; + } else { + /* Default to A0 firmware for all other chip revs */ + idx = VALKYRIE_A0; + dev_warn(&pdev->dev, + "Rev %d not in image lookup table, default to idx=%d\n", + rev, idx); + } + break; + + case PCI_DEVICE_ID_VIPER: + idx = VIPER; + break; + + default: + dev_err(&pdev->dev, "no images for 0x%x\n", pdev->device); + } + return idx; +} + +static const char *get_load_fw_name(struct bcm_vk *vk, + const struct load_image_entry *entry) +{ + const struct firmware *fw; + struct device *dev = &vk->pdev->dev; + int ret; + unsigned long dummy; + int i; + + for (i = 0; i < IMG_PER_TYPE_MAX; i++) { + fw = NULL; + ret = request_partial_firmware_into_buf(&fw, + entry->image_name[i], + dev, &dummy, + sizeof(dummy), + 0); + release_firmware(fw); + if (!ret) + return entry->image_name[i]; + } + return NULL; +} + +int bcm_vk_auto_load_all_images(struct bcm_vk *vk) +{ + int i, ret = -1; + enum soc_idx idx; + struct device *dev = &vk->pdev->dev; + u32 curr_type; + const char *curr_name; + + idx = get_soc_idx(vk); + if (idx == VK_IDX_INVALID) + goto auto_load_all_exit; + + /* log a message to know the relative loading order */ + dev_dbg(dev, "Load All for device %d\n", vk->devid); + + for (i = 0; i < NUM_BOOT_STAGES; i++) { + curr_type = image_tab[idx][i].image_type; + if (bcm_vk_next_boot_image(vk) == curr_type) { + curr_name = get_load_fw_name(vk, &image_tab[idx][i]); + if (!curr_name) { + dev_err(dev, "No suitable firmware exists for type %d", + curr_type); + ret = -ENOENT; + goto auto_load_all_exit; + } + ret = bcm_vk_load_image_by_type(vk, curr_type, + curr_name); + dev_info(dev, "Auto load %s, ret %d\n", + curr_name, ret); + + if (ret) { + dev_err(dev, "Error loading default %s\n", + curr_name); + goto auto_load_all_exit; + } + } + } + +auto_load_all_exit: + return ret; +} + +static int bcm_vk_trigger_autoload(struct bcm_vk *vk) +{ + if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0) + return -EPERM; + + set_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload); + queue_work(vk->wq_thread, &vk->wq_work); + + return 0; +} + +/* + * deferred work queue for draining and auto download. + */ +static void bcm_vk_wq_handler(struct work_struct *work) +{ + struct bcm_vk *vk = container_of(work, struct bcm_vk, wq_work); + struct device *dev = &vk->pdev->dev; + s32 ret; + + /* check wq offload bit map to perform various operations */ + if (test_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload)) { + /* clear bit right the way for notification */ + clear_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload); + bcm_vk_handle_notf(vk); + } + if (test_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload)) { + bcm_vk_auto_load_all_images(vk); + + /* + * at the end of operation, clear AUTO bit and pending + * bit + */ + clear_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload); + clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload); + } + + /* next, try to drain */ + ret = bcm_to_h_msg_dequeue(vk); + + if (ret == 0) + dev_dbg(dev, "Spurious trigger for workqueue\n"); + else if (ret < 0) + bcm_vk_blk_drv_access(vk); +} + +static long bcm_vk_load_image(struct bcm_vk *vk, + const struct vk_image __user *arg) +{ + struct device *dev = &vk->pdev->dev; + const char *image_name; + struct vk_image image; + u32 next_loadable; + enum soc_idx idx; + int image_idx; + int ret = -EPERM; + + if (copy_from_user(&image, arg, sizeof(image))) + return -EACCES; + + if ((image.type != VK_IMAGE_TYPE_BOOT1) && + (image.type != VK_IMAGE_TYPE_BOOT2)) { + dev_err(dev, "invalid image.type %u\n", image.type); + return ret; + } + + next_loadable = bcm_vk_next_boot_image(vk); + if (next_loadable != image.type) { + dev_err(dev, "Next expected image %u, Loading %u\n", + next_loadable, image.type); + return ret; + } + + /* + * if something is pending download already. This could only happen + * for now when the driver is being loaded, or if someone has issued + * another download command in another shell. + */ + if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0) { + dev_err(dev, "Download operation already pending.\n"); + return ret; + } + + image_name = image.filename; + if (image_name[0] == '\0') { + /* Use default image name if NULL */ + idx = get_soc_idx(vk); + if (idx == VK_IDX_INVALID) + goto err_idx; + + /* Image idx starts with boot1 */ + image_idx = image.type - VK_IMAGE_TYPE_BOOT1; + image_name = get_load_fw_name(vk, &image_tab[idx][image_idx]); + if (!image_name) { + dev_err(dev, "No suitable image found for type %d", + image.type); + ret = -ENOENT; + goto err_idx; + } + } else { + /* Ensure filename is NULL terminated */ + image.filename[sizeof(image.filename) - 1] = '\0'; + } + ret = bcm_vk_load_image_by_type(vk, image.type, image_name); + dev_info(dev, "Load %s, ret %d\n", image_name, ret); +err_idx: + clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload); + + return ret; +} + +static int bcm_vk_reset_successful(struct bcm_vk *vk) +{ + struct device *dev = &vk->pdev->dev; + u32 fw_status, reset_reason; + int ret = -EAGAIN; + + /* + * Reset could be triggered when the card in several state: + * i) in bootROM + * ii) after boot1 + * iii) boot2 running + * + * i) & ii) - no status bits will be updated. If vkboot1 + * runs automatically after reset, it will update the reason + * to be unknown reason + * iii) - reboot reason match + deinit done. + */ + fw_status = vkread32(vk, BAR_0, VK_BAR_FWSTS); + /* immediate exit if interface goes down */ + if (BCM_VK_INTF_IS_DOWN(fw_status)) { + dev_err(dev, "PCIe Intf Down!\n"); + goto reset_exit; + } + + reset_reason = (fw_status & VK_FWSTS_RESET_REASON_MASK); + if ((reset_reason == VK_FWSTS_RESET_MBOX_DB) || + (reset_reason == VK_FWSTS_RESET_UNKNOWN)) + ret = 0; + + /* + * if some of the deinit bits are set, but done + * bit is not, this is a failure if triggered while boot2 is running + */ + if ((fw_status & VK_FWSTS_DEINIT_TRIGGERED) && + !(fw_status & VK_FWSTS_RESET_DONE)) + ret = -EAGAIN; + +reset_exit: + dev_dbg(dev, "FW status = 0x%x ret %d\n", fw_status, ret); + + return ret; +} + +static void bcm_to_v_reset_doorbell(struct bcm_vk *vk, u32 db_val) +{ + vkwrite32(vk, db_val, BAR_0, VK_BAR0_RESET_DB_BASE); +} + +static int bcm_vk_trigger_reset(struct bcm_vk *vk) +{ + u32 i; + u32 value, boot_status; + bool is_stdalone, is_boot2; + static const u32 bar0_reg_clr_list[] = { BAR_OS_UPTIME, + BAR_INTF_VER, + BAR_CARD_VOLTAGE, + BAR_CARD_TEMPERATURE, + BAR_CARD_PWR_AND_THRE }; + + /* clean up before pressing the door bell */ + bcm_vk_drain_msg_on_reset(vk); + vkwrite32(vk, 0, BAR_1, VK_BAR1_MSGQ_DEF_RDY); + /* make tag '\0' terminated */ + vkwrite32(vk, 0, BAR_1, VK_BAR1_BOOT1_VER_TAG); + + for (i = 0; i < VK_BAR1_DAUTH_MAX; i++) { + vkwrite32(vk, 0, BAR_1, VK_BAR1_DAUTH_STORE_ADDR(i)); + vkwrite32(vk, 0, BAR_1, VK_BAR1_DAUTH_VALID_ADDR(i)); + } + for (i = 0; i < VK_BAR1_SOTP_REVID_MAX; i++) + vkwrite32(vk, 0, BAR_1, VK_BAR1_SOTP_REVID_ADDR(i)); + + memset(&vk->card_info, 0, sizeof(vk->card_info)); + memset(&vk->peerlog_info, 0, sizeof(vk->peerlog_info)); + memset(&vk->proc_mon_info, 0, sizeof(vk->proc_mon_info)); + memset(&vk->alert_cnts, 0, sizeof(vk->alert_cnts)); + + /* + * When boot request fails, the CODE_PUSH_OFFSET stays persistent. + * Allowing us to debug the failure. When we call reset, + * we should clear CODE_PUSH_OFFSET so ROM does not execute + * boot again (and fails again) and instead waits for a new + * codepush. And, if previous boot has encountered error, need + * to clear the entry values + */ + boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); + if (boot_status & BOOT_ERR_MASK) { + dev_info(&vk->pdev->dev, + "Card in boot error 0x%x, clear CODEPUSH val\n", + boot_status); + value = 0; + } else { + value = vkread32(vk, BAR_0, BAR_CODEPUSH_SBL); + value &= CODEPUSH_MASK; + } + vkwrite32(vk, value, BAR_0, BAR_CODEPUSH_SBL); + + /* special reset handling */ + is_stdalone = boot_status & BOOT_STDALONE_RUNNING; + is_boot2 = (boot_status & BOOT_STATE_MASK) == BOOT2_RUNNING; + if (vk->peer_alert.flags & ERR_LOG_RAMDUMP) { + /* + * if card is in ramdump mode, it is hitting an error. Don't + * reset the reboot reason as it will contain valid info that + * is important - simply use special reset + */ + vkwrite32(vk, VK_BAR0_RESET_RAMPDUMP, BAR_0, VK_BAR_FWSTS); + return VK_BAR0_RESET_RAMPDUMP; + } else if (is_stdalone && !is_boot2) { + dev_info(&vk->pdev->dev, "Hard reset on Standalone mode"); + bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_HARD); + return VK_BAR0_RESET_DB_HARD; + } + + /* reset fw_status with proper reason, and press db */ + vkwrite32(vk, VK_FWSTS_RESET_MBOX_DB, BAR_0, VK_BAR_FWSTS); + bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_SOFT); + + /* clear other necessary registers and alert records */ + for (i = 0; i < ARRAY_SIZE(bar0_reg_clr_list); i++) + vkwrite32(vk, 0, BAR_0, bar0_reg_clr_list[i]); + memset(&vk->host_alert, 0, sizeof(vk->host_alert)); + memset(&vk->peer_alert, 0, sizeof(vk->peer_alert)); + /* clear 4096 bits of bitmap */ + bitmap_clear(vk->bmap, 0, VK_MSG_ID_BITMAP_SIZE); + + return 0; +} + +static long bcm_vk_reset(struct bcm_vk *vk, struct vk_reset __user *arg) +{ + struct device *dev = &vk->pdev->dev; + struct vk_reset reset; + int ret = 0; + u32 ramdump_reset; + int special_reset; + + if (copy_from_user(&reset, arg, sizeof(struct vk_reset))) + return -EFAULT; + + /* check if any download is in-progress, if so return error */ + if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0) { + dev_err(dev, "Download operation pending - skip reset.\n"); + return -EPERM; + } + + ramdump_reset = vk->peer_alert.flags & ERR_LOG_RAMDUMP; + dev_info(dev, "Issue Reset %s\n", + ramdump_reset ? "in ramdump mode" : ""); + + /* + * The following is the sequence of reset: + * - send card level graceful shut down + * - wait enough time for VK to handle its business, stopping DMA etc + * - kill host apps + * - Trigger interrupt with DB + */ + bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_GRACEFUL, 0, 0); + + spin_lock(&vk->ctx_lock); + if (!vk->reset_pid) { + vk->reset_pid = task_pid_nr(current); + } else { + dev_err(dev, "Reset already launched by process pid %d\n", + vk->reset_pid); + ret = -EACCES; + } + spin_unlock(&vk->ctx_lock); + if (ret) + goto err_exit; + + bcm_vk_blk_drv_access(vk); + special_reset = bcm_vk_trigger_reset(vk); + + /* + * Wait enough time for card os to deinit + * and populate the reset reason. + */ + msleep(BCM_VK_DEINIT_TIME_MS); + + if (special_reset) { + /* if it is special ramdump reset, return the type to user */ + reset.arg2 = special_reset; + if (copy_to_user(arg, &reset, sizeof(reset))) + ret = -EFAULT; + } else { + ret = bcm_vk_reset_successful(vk); + } + +err_exit: + clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload); + return ret; +} + +static int bcm_vk_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct bcm_vk_ctx *ctx = file->private_data; + struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); + unsigned long pg_size; + + /* only BAR2 is mmap possible, which is bar num 4 due to 64bit */ +#define VK_MMAPABLE_BAR 4 + + pg_size = ((pci_resource_len(vk->pdev, VK_MMAPABLE_BAR) - 1) + >> PAGE_SHIFT) + 1; + if (vma->vm_pgoff + vma_pages(vma) > pg_size) + return -EINVAL; + + vma->vm_pgoff += (pci_resource_start(vk->pdev, VK_MMAPABLE_BAR) + >> PAGE_SHIFT); + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + vma->vm_end - vma->vm_start, + vma->vm_page_prot); +} + +static long bcm_vk_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + long ret = -EINVAL; + struct bcm_vk_ctx *ctx = file->private_data; + struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); + void __user *argp = (void __user *)arg; + + dev_dbg(&vk->pdev->dev, + "ioctl, cmd=0x%02x, arg=0x%02lx\n", + cmd, arg); + + mutex_lock(&vk->mutex); + + switch (cmd) { + case VK_IOCTL_LOAD_IMAGE: + ret = bcm_vk_load_image(vk, argp); + break; + + case VK_IOCTL_RESET: + ret = bcm_vk_reset(vk, argp); + break; + + default: + break; + } + + mutex_unlock(&vk->mutex); + + return ret; +} + +static const struct file_operations bcm_vk_fops = { + .owner = THIS_MODULE, + .open = bcm_vk_open, + .read = bcm_vk_read, + .write = bcm_vk_write, + .poll = bcm_vk_poll, + .release = bcm_vk_release, + .mmap = bcm_vk_mmap, + .unlocked_ioctl = bcm_vk_ioctl, +}; + +static int bcm_vk_on_panic(struct notifier_block *nb, + unsigned long e, void *p) +{ + struct bcm_vk *vk = container_of(nb, struct bcm_vk, panic_nb); + + bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_HARD); + + return 0; +} + +static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + int err; + int i; + int id; + int irq; + char name[20]; + struct bcm_vk *vk; + struct device *dev = &pdev->dev; + struct miscdevice *misc_device; + u32 boot_status; + + /* allocate vk structure which is tied to kref for freeing */ + vk = kzalloc(sizeof(*vk), GFP_KERNEL); + if (!vk) + return -ENOMEM; + + kref_init(&vk->kref); + if (nr_ib_sgl_blk > BCM_VK_IB_SGL_BLK_MAX) { + dev_warn(dev, "Inband SGL blk %d limited to max %d\n", + nr_ib_sgl_blk, BCM_VK_IB_SGL_BLK_MAX); + nr_ib_sgl_blk = BCM_VK_IB_SGL_BLK_MAX; + } + vk->ib_sgl_size = nr_ib_sgl_blk * VK_MSGQ_BLK_SIZE; + mutex_init(&vk->mutex); + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Cannot enable PCI device\n"); + goto err_free_exit; + } + vk->pdev = pci_dev_get(pdev); + + err = pci_request_regions(pdev, DRV_MODULE_NAME); + if (err) { + dev_err(dev, "Cannot obtain PCI resources\n"); + goto err_disable_pdev; + } + + /* make sure DMA is good */ + err = dma_set_mask_and_coherent(&pdev->dev, + DMA_BIT_MASK(BCM_VK_DMA_BITS)); + if (err) { + dev_err(dev, "failed to set DMA mask\n"); + goto err_disable_pdev; + } + + /* The tdma is a scratch area for some DMA testings. */ + if (nr_scratch_pages) { + vk->tdma_vaddr = dma_alloc_coherent + (dev, + nr_scratch_pages * PAGE_SIZE, + &vk->tdma_addr, GFP_KERNEL); + if (!vk->tdma_vaddr) { + err = -ENOMEM; + goto err_disable_pdev; + } + } + + pci_set_master(pdev); + pci_set_drvdata(pdev, vk); + + irq = pci_alloc_irq_vectors(pdev, + 1, + VK_MSIX_IRQ_MAX, + PCI_IRQ_MSI | PCI_IRQ_MSIX); + + if (irq < VK_MSIX_IRQ_MIN_REQ) { + dev_err(dev, "failed to get min %d MSIX interrupts, irq(%d)\n", + VK_MSIX_IRQ_MIN_REQ, irq); + err = (irq >= 0) ? -EINVAL : irq; + goto err_disable_pdev; + } + + if (irq != VK_MSIX_IRQ_MAX) + dev_warn(dev, "Number of IRQs %d allocated - requested(%d).\n", + irq, VK_MSIX_IRQ_MAX); + + for (i = 0; i < MAX_BAR; i++) { + /* multiple by 2 for 64 bit BAR mapping */ + vk->bar[i] = pci_ioremap_bar(pdev, i * 2); + if (!vk->bar[i]) { + dev_err(dev, "failed to remap BAR%d\n", i); + err = -ENOMEM; + goto err_iounmap; + } + } + + for (vk->num_irqs = 0; + vk->num_irqs < VK_MSIX_MSGQ_MAX; + vk->num_irqs++) { + err = devm_request_irq(dev, pci_irq_vector(pdev, vk->num_irqs), + bcm_vk_msgq_irqhandler, + IRQF_SHARED, DRV_MODULE_NAME, vk); + if (err) { + dev_err(dev, "failed to request msgq IRQ %d for MSIX %d\n", + pdev->irq + vk->num_irqs, vk->num_irqs + 1); + goto err_irq; + } + } + /* one irq for notification from VK */ + err = devm_request_irq(dev, pci_irq_vector(pdev, vk->num_irqs), + bcm_vk_notf_irqhandler, + IRQF_SHARED, DRV_MODULE_NAME, vk); + if (err) { + dev_err(dev, "failed to request notf IRQ %d for MSIX %d\n", + pdev->irq + vk->num_irqs, vk->num_irqs + 1); + goto err_irq; + } + vk->num_irqs++; + + for (i = 0; + (i < VK_MSIX_TTY_MAX) && (vk->num_irqs < irq); + i++, vk->num_irqs++) { + err = devm_request_irq(dev, pci_irq_vector(pdev, vk->num_irqs), + bcm_vk_tty_irqhandler, + IRQF_SHARED, DRV_MODULE_NAME, vk); + if (err) { + dev_err(dev, "failed request tty IRQ %d for MSIX %d\n", + pdev->irq + vk->num_irqs, vk->num_irqs + 1); + goto err_irq; + } + bcm_vk_tty_set_irq_enabled(vk, i); + } + + id = ida_simple_get(&bcm_vk_ida, 0, 0, GFP_KERNEL); + if (id < 0) { + err = id; + dev_err(dev, "unable to get id\n"); + goto err_irq; + } + + vk->devid = id; + snprintf(name, sizeof(name), DRV_MODULE_NAME ".%d", id); + misc_device = &vk->miscdev; + misc_device->minor = MISC_DYNAMIC_MINOR; + misc_device->name = kstrdup(name, GFP_KERNEL); + if (!misc_device->name) { + err = -ENOMEM; + goto err_ida_remove; + } + misc_device->fops = &bcm_vk_fops, + + err = misc_register(misc_device); + if (err) { + dev_err(dev, "failed to register device\n"); + goto err_kfree_name; + } + + INIT_WORK(&vk->wq_work, bcm_vk_wq_handler); + + /* create dedicated workqueue */ + vk->wq_thread = create_singlethread_workqueue(name); + if (!vk->wq_thread) { + dev_err(dev, "Fail to create workqueue thread\n"); + err = -ENOMEM; + goto err_misc_deregister; + } + + err = bcm_vk_msg_init(vk); + if (err) { + dev_err(dev, "failed to init msg queue info\n"); + goto err_destroy_workqueue; + } + + /* sync other info */ + bcm_vk_sync_card_info(vk); + + /* register for panic notifier */ + vk->panic_nb.notifier_call = bcm_vk_on_panic; + err = atomic_notifier_chain_register(&panic_notifier_list, + &vk->panic_nb); + if (err) { + dev_err(dev, "Fail to register panic notifier\n"); + goto err_destroy_workqueue; + } + + snprintf(name, sizeof(name), KBUILD_MODNAME ".%d_ttyVK", id); + err = bcm_vk_tty_init(vk, name); + if (err) + goto err_unregister_panic_notifier; + + /* + * lets trigger an auto download. We don't want to do it serially here + * because at probing time, it is not supposed to block for a long time. + */ + boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); + if (auto_load) { + if ((boot_status & BOOT_STATE_MASK) == BROM_RUNNING) { + err = bcm_vk_trigger_autoload(vk); + if (err) + goto err_bcm_vk_tty_exit; + } else { + dev_err(dev, + "Auto-load skipped - BROM not in proper state (0x%x)\n", + boot_status); + } + } + + /* enable hb */ + bcm_vk_hb_init(vk); + + dev_dbg(dev, "BCM-VK:%u created\n", id); + + return 0; + +err_bcm_vk_tty_exit: + bcm_vk_tty_exit(vk); + +err_unregister_panic_notifier: + atomic_notifier_chain_unregister(&panic_notifier_list, + &vk->panic_nb); + +err_destroy_workqueue: + destroy_workqueue(vk->wq_thread); + +err_misc_deregister: + misc_deregister(misc_device); + +err_kfree_name: + kfree(misc_device->name); + misc_device->name = NULL; + +err_ida_remove: + ida_simple_remove(&bcm_vk_ida, id); + +err_irq: + for (i = 0; i < vk->num_irqs; i++) + devm_free_irq(dev, pci_irq_vector(pdev, i), vk); + + pci_disable_msix(pdev); + pci_disable_msi(pdev); + +err_iounmap: + for (i = 0; i < MAX_BAR; i++) { + if (vk->bar[i]) + pci_iounmap(pdev, vk->bar[i]); + } + pci_release_regions(pdev); + +err_disable_pdev: + if (vk->tdma_vaddr) + dma_free_coherent(&pdev->dev, nr_scratch_pages * PAGE_SIZE, + vk->tdma_vaddr, vk->tdma_addr); + + pci_free_irq_vectors(pdev); + pci_disable_device(pdev); + pci_dev_put(pdev); + +err_free_exit: + kfree(vk); + + return err; +} + +void bcm_vk_release_data(struct kref *kref) +{ + struct bcm_vk *vk = container_of(kref, struct bcm_vk, kref); + struct pci_dev *pdev = vk->pdev; + + dev_dbg(&pdev->dev, "BCM-VK:%d release data 0x%p\n", vk->devid, vk); + pci_dev_put(pdev); + kfree(vk); +} + +static void bcm_vk_remove(struct pci_dev *pdev) +{ + int i; + struct bcm_vk *vk = pci_get_drvdata(pdev); + struct miscdevice *misc_device = &vk->miscdev; + + bcm_vk_hb_deinit(vk); + + /* + * Trigger a reset to card and wait enough time for UCODE to rerun, + * which re-initialize the card into its default state. + * This ensures when driver is re-enumerated it will start from + * a completely clean state. + */ + bcm_vk_trigger_reset(vk); + usleep_range(BCM_VK_UCODE_BOOT_US, BCM_VK_UCODE_BOOT_MAX_US); + + /* unregister panic notifier */ + atomic_notifier_chain_unregister(&panic_notifier_list, + &vk->panic_nb); + + bcm_vk_msg_remove(vk); + bcm_vk_tty_exit(vk); + + if (vk->tdma_vaddr) + dma_free_coherent(&pdev->dev, nr_scratch_pages * PAGE_SIZE, + vk->tdma_vaddr, vk->tdma_addr); + + /* remove if name is set which means misc dev registered */ + if (misc_device->name) { + misc_deregister(misc_device); + kfree(misc_device->name); + ida_simple_remove(&bcm_vk_ida, vk->devid); + } + for (i = 0; i < vk->num_irqs; i++) + devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i), vk); + + pci_disable_msix(pdev); + pci_disable_msi(pdev); + + cancel_work_sync(&vk->wq_work); + destroy_workqueue(vk->wq_thread); + bcm_vk_tty_wq_exit(vk); + + for (i = 0; i < MAX_BAR; i++) { + if (vk->bar[i]) + pci_iounmap(pdev, vk->bar[i]); + } + + dev_dbg(&pdev->dev, "BCM-VK:%d released\n", vk->devid); + + pci_release_regions(pdev); + pci_free_irq_vectors(pdev); + pci_disable_device(pdev); + + kref_put(&vk->kref, bcm_vk_release_data); +} + +static void bcm_vk_shutdown(struct pci_dev *pdev) +{ + struct bcm_vk *vk = pci_get_drvdata(pdev); + u32 reg, boot_stat; + + reg = vkread32(vk, BAR_0, BAR_BOOT_STATUS); + boot_stat = reg & BOOT_STATE_MASK; + + if (boot_stat == BOOT1_RUNNING) { + /* simply trigger a reset interrupt to park it */ + bcm_vk_trigger_reset(vk); + } else if (boot_stat == BROM_NOT_RUN) { + int err; + u16 lnksta; + + /* + * The boot status only reflects boot condition since last reset + * As ucode will run only once to configure pcie, if multiple + * resets happen, we lost track if ucode has run or not. + * Here, read the current link speed and use that to + * sync up the bootstatus properly so that on reboot-back-up, + * it has the proper state to start with autoload + */ + err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); + if (!err && + (lnksta & PCI_EXP_LNKSTA_CLS) != PCI_EXP_LNKSTA_CLS_2_5GB) { + reg |= BROM_STATUS_COMPLETE; + vkwrite32(vk, reg, BAR_0, BAR_BOOT_STATUS); + } + } +} + +static const struct pci_device_id bcm_vk_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_VALKYRIE), }, + { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_VIPER), }, + { } +}; +MODULE_DEVICE_TABLE(pci, bcm_vk_ids); + +static struct pci_driver pci_driver = { + .name = DRV_MODULE_NAME, + .id_table = bcm_vk_ids, + .probe = bcm_vk_probe, + .remove = bcm_vk_remove, + .shutdown = bcm_vk_shutdown, +}; +module_pci_driver(pci_driver); + +MODULE_DESCRIPTION("Broadcom VK Host Driver"); +MODULE_AUTHOR("Scott Branden <scott.branden@broadcom.com>"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION("1.0"); diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c new file mode 100644 index 000000000000..f40cf08a6192 --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -0,0 +1,1357 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2018-2020 Broadcom. + */ + +#include <linux/delay.h> +#include <linux/fs.h> +#include <linux/hash.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/poll.h> +#include <linux/sizes.h> +#include <linux/spinlock.h> +#include <linux/timer.h> + +#include "bcm_vk.h" +#include "bcm_vk_msg.h" +#include "bcm_vk_sg.h" + +/* functions to manipulate the transport id in msg block */ +#define BCM_VK_MSG_Q_SHIFT 4 +#define BCM_VK_MSG_Q_MASK 0xF +#define BCM_VK_MSG_ID_MASK 0xFFF + +#define BCM_VK_DMA_DRAIN_MAX_MS 2000 + +/* number x q_size will be the max number of msg processed per loop */ +#define BCM_VK_MSG_PROC_MAX_LOOP 2 + +/* module parameter */ +static bool hb_mon = true; +module_param(hb_mon, bool, 0444); +MODULE_PARM_DESC(hb_mon, "Monitoring heartbeat continuously.\n"); +static int batch_log = 1; +module_param(batch_log, int, 0444); +MODULE_PARM_DESC(batch_log, "Max num of logs per batch operation.\n"); + +static bool hb_mon_is_on(void) +{ + return hb_mon; +} + +static u32 get_q_num(const struct vk_msg_blk *msg) +{ + u32 q_num = msg->trans_id & BCM_VK_MSG_Q_MASK; + + if (q_num >= VK_MSGQ_PER_CHAN_MAX) + q_num = VK_MSGQ_NUM_DEFAULT; + return q_num; +} + +static void set_q_num(struct vk_msg_blk *msg, u32 q_num) +{ + u32 trans_q; + + if (q_num >= VK_MSGQ_PER_CHAN_MAX) + trans_q = VK_MSGQ_NUM_DEFAULT; + else + trans_q = q_num; + + msg->trans_id = (msg->trans_id & ~BCM_VK_MSG_Q_MASK) | trans_q; +} + +static u32 get_msg_id(const struct vk_msg_blk *msg) +{ + return ((msg->trans_id >> BCM_VK_MSG_Q_SHIFT) & BCM_VK_MSG_ID_MASK); +} + +static void set_msg_id(struct vk_msg_blk *msg, u32 val) +{ + msg->trans_id = (val << BCM_VK_MSG_Q_SHIFT) | get_q_num(msg); +} + +static u32 msgq_inc(const struct bcm_vk_sync_qinfo *qinfo, u32 idx, u32 inc) +{ + return ((idx + inc) & qinfo->q_mask); +} + +static +struct vk_msg_blk __iomem *msgq_blk_addr(const struct bcm_vk_sync_qinfo *qinfo, + u32 idx) +{ + return qinfo->q_start + (VK_MSGQ_BLK_SIZE * idx); +} + +static u32 msgq_occupied(const struct bcm_vk_msgq __iomem *msgq, + const struct bcm_vk_sync_qinfo *qinfo) +{ + u32 wr_idx, rd_idx; + + wr_idx = readl_relaxed(&msgq->wr_idx); + rd_idx = readl_relaxed(&msgq->rd_idx); + + return ((wr_idx - rd_idx) & qinfo->q_mask); +} + +static +u32 msgq_avail_space(const struct bcm_vk_msgq __iomem *msgq, + const struct bcm_vk_sync_qinfo *qinfo) +{ + return (qinfo->q_size - msgq_occupied(msgq, qinfo) - 1); +} + +/* number of retries when enqueue message fails before returning EAGAIN */ +#define BCM_VK_H2VK_ENQ_RETRY 10 +#define BCM_VK_H2VK_ENQ_RETRY_DELAY_MS 50 + +bool bcm_vk_drv_access_ok(struct bcm_vk *vk) +{ + return (!!atomic_read(&vk->msgq_inited)); +} + +void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask) +{ + struct bcm_vk_alert *alert = &vk->host_alert; + unsigned long flags; + + /* use irqsave version as this maybe called inside timer interrupt */ + spin_lock_irqsave(&vk->host_alert_lock, flags); + alert->notfs |= bit_mask; + spin_unlock_irqrestore(&vk->host_alert_lock, flags); + + if (test_and_set_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload) == 0) + queue_work(vk->wq_thread, &vk->wq_work); +} + +/* + * Heartbeat related defines + * The heartbeat from host is a last resort. If stuck condition happens + * on the card, firmware is supposed to detect it. Therefore, the heartbeat + * values used will be more relaxed on the driver, which need to be bigger + * than the watchdog timeout on the card. The watchdog timeout on the card + * is 20s, with a jitter of 2s => 22s. We use a value of 27s here. + */ +#define BCM_VK_HB_TIMER_S 3 +#define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ) +#define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S) + +static void bcm_vk_hb_poll(struct timer_list *t) +{ + u32 uptime_s; + struct bcm_vk_hb_ctrl *hb = container_of(t, struct bcm_vk_hb_ctrl, + timer); + struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl); + + if (bcm_vk_drv_access_ok(vk) && hb_mon_is_on()) { + /* read uptime from register and compare */ + uptime_s = vkread32(vk, BAR_0, BAR_OS_UPTIME); + + if (uptime_s == hb->last_uptime) + hb->lost_cnt++; + else /* reset to avoid accumulation */ + hb->lost_cnt = 0; + + dev_dbg(&vk->pdev->dev, "Last uptime %d current %d, lost %d\n", + hb->last_uptime, uptime_s, hb->lost_cnt); + + /* + * if the interface goes down without any activity, a value + * of 0xFFFFFFFF will be continuously read, and the detection + * will be happened eventually. + */ + hb->last_uptime = uptime_s; + } else { + /* reset heart beat lost cnt */ + hb->lost_cnt = 0; + } + + /* next, check if heartbeat exceeds limit */ + if (hb->lost_cnt > BCM_VK_HB_LOST_MAX) { + dev_err(&vk->pdev->dev, "Heartbeat Misses %d times, %d s!\n", + BCM_VK_HB_LOST_MAX, + BCM_VK_HB_LOST_MAX * BCM_VK_HB_TIMER_S); + + bcm_vk_blk_drv_access(vk); + bcm_vk_set_host_alert(vk, ERR_LOG_HOST_HB_FAIL); + } + /* re-arm timer */ + mod_timer(&hb->timer, jiffies + BCM_VK_HB_TIMER_VALUE); +} + +void bcm_vk_hb_init(struct bcm_vk *vk) +{ + struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; + + timer_setup(&hb->timer, bcm_vk_hb_poll, 0); + mod_timer(&hb->timer, jiffies + BCM_VK_HB_TIMER_VALUE); +} + +void bcm_vk_hb_deinit(struct bcm_vk *vk) +{ + struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; + + del_timer(&hb->timer); +} + +static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk, + unsigned int start, + unsigned int nbits) +{ + spin_lock(&vk->msg_id_lock); + bitmap_clear(vk->bmap, start, nbits); + spin_unlock(&vk->msg_id_lock); +} + +/* + * allocate a ctx per file struct + */ +static struct bcm_vk_ctx *bcm_vk_get_ctx(struct bcm_vk *vk, const pid_t pid) +{ + u32 i; + struct bcm_vk_ctx *ctx = NULL; + u32 hash_idx = hash_32(pid, VK_PID_HT_SHIFT_BIT); + + spin_lock(&vk->ctx_lock); + + /* check if it is in reset, if so, don't allow */ + if (vk->reset_pid) { + dev_err(&vk->pdev->dev, + "No context allowed during reset by pid %d\n", + vk->reset_pid); + + goto in_reset_exit; + } + + for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) { + if (!vk->ctx[i].in_use) { + vk->ctx[i].in_use = true; + ctx = &vk->ctx[i]; + break; + } + } + + if (!ctx) { + dev_err(&vk->pdev->dev, "All context in use\n"); + + goto all_in_use_exit; + } + + /* set the pid and insert it to hash table */ + ctx->pid = pid; + ctx->hash_idx = hash_idx; + list_add_tail(&ctx->node, &vk->pid_ht[hash_idx].head); + + /* increase kref */ + kref_get(&vk->kref); + + /* clear counter */ + atomic_set(&ctx->pend_cnt, 0); + atomic_set(&ctx->dma_cnt, 0); + init_waitqueue_head(&ctx->rd_wq); + +all_in_use_exit: +in_reset_exit: + spin_unlock(&vk->ctx_lock); + + return ctx; +} + +static u16 bcm_vk_get_msg_id(struct bcm_vk *vk) +{ + u16 rc = VK_MSG_ID_OVERFLOW; + u16 test_bit_count = 0; + + spin_lock(&vk->msg_id_lock); + while (test_bit_count < (VK_MSG_ID_BITMAP_SIZE - 1)) { + /* + * first time come in this loop, msg_id will be 0 + * and the first one tested will be 1. We skip + * VK_SIMPLEX_MSG_ID (0) for one way host2vk + * communication + */ + vk->msg_id++; + if (vk->msg_id == VK_MSG_ID_BITMAP_SIZE) + vk->msg_id = 1; + + if (test_bit(vk->msg_id, vk->bmap)) { + test_bit_count++; + continue; + } + rc = vk->msg_id; + bitmap_set(vk->bmap, vk->msg_id, 1); + break; + } + spin_unlock(&vk->msg_id_lock); + + return rc; +} + +static int bcm_vk_free_ctx(struct bcm_vk *vk, struct bcm_vk_ctx *ctx) +{ + u32 idx; + u32 hash_idx; + pid_t pid; + struct bcm_vk_ctx *entry; + int count = 0; + + if (!ctx) { + dev_err(&vk->pdev->dev, "NULL context detected\n"); + return -EINVAL; + } + idx = ctx->idx; + pid = ctx->pid; + + spin_lock(&vk->ctx_lock); + + if (!vk->ctx[idx].in_use) { + dev_err(&vk->pdev->dev, "context[%d] not in use!\n", idx); + } else { + vk->ctx[idx].in_use = false; + vk->ctx[idx].miscdev = NULL; + + /* Remove it from hash list and see if it is the last one. */ + list_del(&ctx->node); + hash_idx = ctx->hash_idx; + list_for_each_entry(entry, &vk->pid_ht[hash_idx].head, node) { + if (entry->pid == pid) + count++; + } + } + + spin_unlock(&vk->ctx_lock); + + return count; +} + +static void bcm_vk_free_wkent(struct device *dev, struct bcm_vk_wkent *entry) +{ + int proc_cnt; + + bcm_vk_sg_free(dev, entry->dma, VK_DMA_MAX_ADDRS, &proc_cnt); + if (proc_cnt) + atomic_dec(&entry->ctx->dma_cnt); + + kfree(entry->to_h_msg); + kfree(entry); +} + +static void bcm_vk_drain_all_pend(struct device *dev, + struct bcm_vk_msg_chan *chan, + struct bcm_vk_ctx *ctx) +{ + u32 num; + struct bcm_vk_wkent *entry, *tmp; + struct bcm_vk *vk; + struct list_head del_q; + + if (ctx) + vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); + + INIT_LIST_HEAD(&del_q); + spin_lock(&chan->pendq_lock); + for (num = 0; num < chan->q_nr; num++) { + list_for_each_entry_safe(entry, tmp, &chan->pendq[num], node) { + if ((!ctx) || (entry->ctx->idx == ctx->idx)) { + list_del(&entry->node); + list_add_tail(&entry->node, &del_q); + } + } + } + spin_unlock(&chan->pendq_lock); + + /* batch clean up */ + num = 0; + list_for_each_entry_safe(entry, tmp, &del_q, node) { + list_del(&entry->node); + num++; + if (ctx) { + struct vk_msg_blk *msg; + int bit_set; + bool responded; + u32 msg_id; + + /* if it is specific ctx, log for any stuck */ + msg = entry->to_v_msg; + msg_id = get_msg_id(msg); + bit_set = test_bit(msg_id, vk->bmap); + responded = entry->to_h_msg ? true : false; + if (num <= batch_log) + dev_info(dev, + "Drained: fid %u size %u msg 0x%x(seq-%x) ctx 0x%x[fd-%d] args:[0x%x 0x%x] resp %s, bmap %d\n", + msg->function_id, msg->size, + msg_id, entry->seq_num, + msg->context_id, entry->ctx->idx, + msg->cmd, msg->arg, + responded ? "T" : "F", bit_set); + if (responded) + atomic_dec(&ctx->pend_cnt); + else if (bit_set) + bcm_vk_msgid_bitmap_clear(vk, msg_id, 1); + } + bcm_vk_free_wkent(dev, entry); + } + if (num && ctx) + dev_info(dev, "Total drained items %d [fd-%d]\n", + num, ctx->idx); +} + +void bcm_vk_drain_msg_on_reset(struct bcm_vk *vk) +{ + bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, NULL); + bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, NULL); +} + +/* + * Function to sync up the messages queue info that is provided by BAR1 + */ +int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync) +{ + struct bcm_vk_msgq __iomem *msgq; + struct device *dev = &vk->pdev->dev; + u32 msgq_off; + u32 num_q; + struct bcm_vk_msg_chan *chan_list[] = {&vk->to_v_msg_chan, + &vk->to_h_msg_chan}; + struct bcm_vk_msg_chan *chan; + int i, j; + int ret = 0; + + /* + * If the driver is loaded at startup where vk OS is not up yet, + * the msgq-info may not be available until a later time. In + * this case, we skip and the sync function is supposed to be + * called again. + */ + if (!bcm_vk_msgq_marker_valid(vk)) { + dev_info(dev, "BAR1 msgq marker not initialized.\n"); + return -EAGAIN; + } + + msgq_off = vkread32(vk, BAR_1, VK_BAR1_MSGQ_CTRL_OFF); + + /* each side is always half the total */ + num_q = vkread32(vk, BAR_1, VK_BAR1_MSGQ_NR) / 2; + if (!num_q || (num_q > VK_MSGQ_PER_CHAN_MAX)) { + dev_err(dev, + "Advertised msgq %d error - max %d allowed\n", + num_q, VK_MSGQ_PER_CHAN_MAX); + return -EINVAL; + } + + vk->to_v_msg_chan.q_nr = num_q; + vk->to_h_msg_chan.q_nr = num_q; + + /* first msgq location */ + msgq = vk->bar[BAR_1] + msgq_off; + + /* + * if this function is called when it is already inited, + * something is wrong + */ + if (bcm_vk_drv_access_ok(vk) && !force_sync) { + dev_err(dev, "Msgq info already in sync\n"); + return -EPERM; + } + + for (i = 0; i < ARRAY_SIZE(chan_list); i++) { + chan = chan_list[i]; + memset(chan->sync_qinfo, 0, sizeof(chan->sync_qinfo)); + + for (j = 0; j < num_q; j++) { + struct bcm_vk_sync_qinfo *qinfo; + u32 msgq_start; + u32 msgq_size; + u32 msgq_nxt; + u32 msgq_db_offset, q_db_offset; + + chan->msgq[j] = msgq; + msgq_start = readl_relaxed(&msgq->start); + msgq_size = readl_relaxed(&msgq->size); + msgq_nxt = readl_relaxed(&msgq->nxt); + msgq_db_offset = readl_relaxed(&msgq->db_offset); + q_db_offset = (msgq_db_offset & ((1 << DB_SHIFT) - 1)); + if (q_db_offset == (~msgq_db_offset >> DB_SHIFT)) + msgq_db_offset = q_db_offset; + else + /* fall back to default */ + msgq_db_offset = VK_BAR0_Q_DB_BASE(j); + + dev_info(dev, + "MsgQ[%d] type %d num %d, @ 0x%x, db_offset 0x%x rd_idx %d wr_idx %d, size %d, nxt 0x%x\n", + j, + readw_relaxed(&msgq->type), + readw_relaxed(&msgq->num), + msgq_start, + msgq_db_offset, + readl_relaxed(&msgq->rd_idx), + readl_relaxed(&msgq->wr_idx), + msgq_size, + msgq_nxt); + + qinfo = &chan->sync_qinfo[j]; + /* formulate and record static info */ + qinfo->q_start = vk->bar[BAR_1] + msgq_start; + qinfo->q_size = msgq_size; + /* set low threshold as 50% or 1/2 */ + qinfo->q_low = qinfo->q_size >> 1; + qinfo->q_mask = qinfo->q_size - 1; + qinfo->q_db_offset = msgq_db_offset; + + msgq++; + } + } + atomic_set(&vk->msgq_inited, 1); + + return ret; +} + +static int bcm_vk_msg_chan_init(struct bcm_vk_msg_chan *chan) +{ + u32 i; + + mutex_init(&chan->msgq_mutex); + spin_lock_init(&chan->pendq_lock); + for (i = 0; i < VK_MSGQ_MAX_NR; i++) + INIT_LIST_HEAD(&chan->pendq[i]); + + return 0; +} + +static void bcm_vk_append_pendq(struct bcm_vk_msg_chan *chan, u16 q_num, + struct bcm_vk_wkent *entry) +{ + struct bcm_vk_ctx *ctx; + + spin_lock(&chan->pendq_lock); + list_add_tail(&entry->node, &chan->pendq[q_num]); + if (entry->to_h_msg) { + ctx = entry->ctx; + atomic_inc(&ctx->pend_cnt); + wake_up_interruptible(&ctx->rd_wq); + } + spin_unlock(&chan->pendq_lock); +} + +static u32 bcm_vk_append_ib_sgl(struct bcm_vk *vk, + struct bcm_vk_wkent *entry, + struct _vk_data *data, + unsigned int num_planes) +{ + unsigned int i; + unsigned int item_cnt = 0; + struct device *dev = &vk->pdev->dev; + struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan; + struct vk_msg_blk *msg = &entry->to_v_msg[0]; + struct bcm_vk_msgq __iomem *msgq; + struct bcm_vk_sync_qinfo *qinfo; + u32 ib_sgl_size = 0; + u8 *buf = (u8 *)&entry->to_v_msg[entry->to_v_blks]; + u32 avail; + u32 q_num; + + /* check if high watermark is hit, and if so, skip */ + q_num = get_q_num(msg); + msgq = chan->msgq[q_num]; + qinfo = &chan->sync_qinfo[q_num]; + avail = msgq_avail_space(msgq, qinfo); + if (avail < qinfo->q_low) { + dev_dbg(dev, "Skip inserting inband SGL, [0x%x/0x%x]\n", + avail, qinfo->q_size); + return 0; + } + + for (i = 0; i < num_planes; i++) { + if (data[i].address && + (ib_sgl_size + data[i].size) <= vk->ib_sgl_size) { + item_cnt++; + memcpy(buf, entry->dma[i].sglist, data[i].size); + ib_sgl_size += data[i].size; + buf += data[i].size; + } + } + + dev_dbg(dev, "Num %u sgl items appended, size 0x%x, room 0x%x\n", + item_cnt, ib_sgl_size, vk->ib_sgl_size); + + /* round up size */ + ib_sgl_size = (ib_sgl_size + VK_MSGQ_BLK_SIZE - 1) + >> VK_MSGQ_BLK_SZ_SHIFT; + + return ib_sgl_size; +} + +void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val) +{ + struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan; + struct bcm_vk_sync_qinfo *qinfo = &chan->sync_qinfo[q_num]; + + vkwrite32(vk, db_val, BAR_0, qinfo->q_db_offset); +} + +static int bcm_to_v_msg_enqueue(struct bcm_vk *vk, struct bcm_vk_wkent *entry) +{ + static u32 seq_num; + struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan; + struct device *dev = &vk->pdev->dev; + struct vk_msg_blk *src = &entry->to_v_msg[0]; + + struct vk_msg_blk __iomem *dst; + struct bcm_vk_msgq __iomem *msgq; + struct bcm_vk_sync_qinfo *qinfo; + u32 q_num = get_q_num(src); + u32 wr_idx; /* local copy */ + u32 i; + u32 avail; + u32 retry; + + if (entry->to_v_blks != src->size + 1) { + dev_err(dev, "number of blks %d not matching %d MsgId[0x%x]: func %d ctx 0x%x\n", + entry->to_v_blks, + src->size + 1, + get_msg_id(src), + src->function_id, + src->context_id); + return -EMSGSIZE; + } + + msgq = chan->msgq[q_num]; + qinfo = &chan->sync_qinfo[q_num]; + + mutex_lock(&chan->msgq_mutex); + + avail = msgq_avail_space(msgq, qinfo); + + /* if not enough space, return EAGAIN and let app handles it */ + retry = 0; + while ((avail < entry->to_v_blks) && + (retry++ < BCM_VK_H2VK_ENQ_RETRY)) { + mutex_unlock(&chan->msgq_mutex); + + msleep(BCM_VK_H2VK_ENQ_RETRY_DELAY_MS); + mutex_lock(&chan->msgq_mutex); + avail = msgq_avail_space(msgq, qinfo); + } + if (retry > BCM_VK_H2VK_ENQ_RETRY) { + mutex_unlock(&chan->msgq_mutex); + return -EAGAIN; + } + + /* at this point, mutex is taken and there is enough space */ + entry->seq_num = seq_num++; /* update debug seq number */ + wr_idx = readl_relaxed(&msgq->wr_idx); + + if (wr_idx >= qinfo->q_size) { + dev_crit(dev, "Invalid wr_idx 0x%x => max 0x%x!", + wr_idx, qinfo->q_size); + bcm_vk_blk_drv_access(vk); + bcm_vk_set_host_alert(vk, ERR_LOG_HOST_PCIE_DWN); + goto idx_err; + } + + dst = msgq_blk_addr(qinfo, wr_idx); + for (i = 0; i < entry->to_v_blks; i++) { + memcpy_toio(dst, src, sizeof(*dst)); + + src++; + wr_idx = msgq_inc(qinfo, wr_idx, 1); + dst = msgq_blk_addr(qinfo, wr_idx); + } + + /* flush the write pointer */ + writel(wr_idx, &msgq->wr_idx); + + /* log new info for debugging */ + dev_dbg(dev, + "MsgQ[%d] [Rd Wr] = [%d %d] blks inserted %d - Q = [u-%d a-%d]/%d\n", + readl_relaxed(&msgq->num), + readl_relaxed(&msgq->rd_idx), + wr_idx, + entry->to_v_blks, + msgq_occupied(msgq, qinfo), + msgq_avail_space(msgq, qinfo), + readl_relaxed(&msgq->size)); + /* + * press door bell based on queue number. 1 is added to the wr_idx + * to avoid the value of 0 appearing on the VK side to distinguish + * from initial value. + */ + bcm_to_v_q_doorbell(vk, q_num, wr_idx + 1); +idx_err: + mutex_unlock(&chan->msgq_mutex); + return 0; +} + +int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type, + const pid_t pid, const u32 q_num) +{ + int rc = 0; + struct bcm_vk_wkent *entry; + struct device *dev = &vk->pdev->dev; + + /* + * check if the marker is still good. Sometimes, the PCIe interface may + * have gone done, and if so and we ship down thing based on broken + * values, kernel may panic. + */ + if (!bcm_vk_msgq_marker_valid(vk)) { + dev_info(dev, "PCIe comm chan - invalid marker (0x%x)!\n", + vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY)); + return -EINVAL; + } + + entry = kzalloc(sizeof(*entry) + + sizeof(struct vk_msg_blk), GFP_KERNEL); + if (!entry) + return -ENOMEM; + + /* fill up necessary data */ + entry->to_v_msg[0].function_id = VK_FID_SHUTDOWN; + set_q_num(&entry->to_v_msg[0], q_num); + set_msg_id(&entry->to_v_msg[0], VK_SIMPLEX_MSG_ID); + entry->to_v_blks = 1; /* always 1 block */ + + entry->to_v_msg[0].cmd = shut_type; + entry->to_v_msg[0].arg = pid; + + rc = bcm_to_v_msg_enqueue(vk, entry); + if (rc) + dev_err(dev, + "Sending shutdown message to q %d for pid %d fails.\n", + get_q_num(&entry->to_v_msg[0]), pid); + + kfree(entry); + + return rc; +} + +static int bcm_vk_handle_last_sess(struct bcm_vk *vk, const pid_t pid, + const u32 q_num) +{ + int rc = 0; + struct device *dev = &vk->pdev->dev; + + /* + * don't send down or do anything if message queue is not initialized + * and if it is the reset session, clear it. + */ + if (!bcm_vk_drv_access_ok(vk)) { + if (vk->reset_pid == pid) + vk->reset_pid = 0; + return -EPERM; + } + + dev_dbg(dev, "No more sessions, shut down pid %d\n", pid); + + /* only need to do it if it is not the reset process */ + if (vk->reset_pid != pid) + rc = bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_PID, pid, q_num); + else + /* put reset_pid to 0 if it is exiting last session */ + vk->reset_pid = 0; + + return rc; +} + +static struct bcm_vk_wkent *bcm_vk_dequeue_pending(struct bcm_vk *vk, + struct bcm_vk_msg_chan *chan, + u16 q_num, + u16 msg_id) +{ + bool found = false; + struct bcm_vk_wkent *entry; + + spin_lock(&chan->pendq_lock); + list_for_each_entry(entry, &chan->pendq[q_num], node) { + if (get_msg_id(&entry->to_v_msg[0]) == msg_id) { + list_del(&entry->node); + found = true; + bcm_vk_msgid_bitmap_clear(vk, msg_id, 1); + break; + } + } + spin_unlock(&chan->pendq_lock); + return ((found) ? entry : NULL); +} + +s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk) +{ + struct device *dev = &vk->pdev->dev; + struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan; + struct vk_msg_blk *data; + struct vk_msg_blk __iomem *src; + struct vk_msg_blk *dst; + struct bcm_vk_msgq __iomem *msgq; + struct bcm_vk_sync_qinfo *qinfo; + struct bcm_vk_wkent *entry; + u32 rd_idx, wr_idx; + u32 q_num, msg_id, j; + u32 num_blks; + s32 total = 0; + int cnt = 0; + int msg_processed = 0; + int max_msg_to_process; + bool exit_loop; + + /* + * drain all the messages from the queues, and find its pending + * entry in the to_v queue, based on msg_id & q_num, and move the + * entry to the to_h pending queue, waiting for user space + * program to extract + */ + mutex_lock(&chan->msgq_mutex); + + for (q_num = 0; q_num < chan->q_nr; q_num++) { + msgq = chan->msgq[q_num]; + qinfo = &chan->sync_qinfo[q_num]; + max_msg_to_process = BCM_VK_MSG_PROC_MAX_LOOP * qinfo->q_size; + + rd_idx = readl_relaxed(&msgq->rd_idx); + wr_idx = readl_relaxed(&msgq->wr_idx); + msg_processed = 0; + exit_loop = false; + while ((rd_idx != wr_idx) && !exit_loop) { + u8 src_size; + + /* + * Make a local copy and get pointer to src blk + * The rd_idx is masked before getting the pointer to + * avoid out of bound access in case the interface goes + * down. It will end up pointing to the last block in + * the buffer, but subsequent src->size check would be + * able to catch this. + */ + src = msgq_blk_addr(qinfo, rd_idx & qinfo->q_mask); + src_size = readb(&src->size); + + if ((rd_idx >= qinfo->q_size) || + (src_size > (qinfo->q_size - 1))) { + dev_crit(dev, + "Invalid rd_idx 0x%x or size 0x%x => max 0x%x!", + rd_idx, src_size, qinfo->q_size); + bcm_vk_blk_drv_access(vk); + bcm_vk_set_host_alert(vk, + ERR_LOG_HOST_PCIE_DWN); + goto idx_err; + } + + num_blks = src_size + 1; + data = kzalloc(num_blks * VK_MSGQ_BLK_SIZE, GFP_KERNEL); + if (data) { + /* copy messages and linearize it */ + dst = data; + for (j = 0; j < num_blks; j++) { + memcpy_fromio(dst, src, sizeof(*dst)); + + dst++; + rd_idx = msgq_inc(qinfo, rd_idx, 1); + src = msgq_blk_addr(qinfo, rd_idx); + } + total++; + } else { + /* + * if we could not allocate memory in kernel, + * that is fatal. + */ + dev_crit(dev, "Kernel mem allocation failure.\n"); + total = -ENOMEM; + goto idx_err; + } + + /* flush rd pointer after a message is dequeued */ + writel(rd_idx, &msgq->rd_idx); + + /* log new info for debugging */ + dev_dbg(dev, + "MsgQ[%d] [Rd Wr] = [%d %d] blks extracted %d - Q = [u-%d a-%d]/%d\n", + readl_relaxed(&msgq->num), + rd_idx, + wr_idx, + num_blks, + msgq_occupied(msgq, qinfo), + msgq_avail_space(msgq, qinfo), + readl_relaxed(&msgq->size)); + + /* + * No need to search if it is an autonomous one-way + * message from driver, as these messages do not bear + * a to_v pending item. Currently, only the shutdown + * message falls into this category. + */ + if (data->function_id == VK_FID_SHUTDOWN) { + kfree(data); + continue; + } + + msg_id = get_msg_id(data); + /* lookup original message in to_v direction */ + entry = bcm_vk_dequeue_pending(vk, + &vk->to_v_msg_chan, + q_num, + msg_id); + + /* + * if there is message to does not have prior send, + * this is the location to add here + */ + if (entry) { + entry->to_h_blks = num_blks; + entry->to_h_msg = data; + bcm_vk_append_pendq(&vk->to_h_msg_chan, + q_num, entry); + + } else { + if (cnt++ < batch_log) + dev_info(dev, + "Could not find MsgId[0x%x] for resp func %d bmap %d\n", + msg_id, data->function_id, + test_bit(msg_id, vk->bmap)); + kfree(data); + } + /* Fetch wr_idx to handle more back-to-back events */ + wr_idx = readl(&msgq->wr_idx); + + /* + * cap the max so that even we try to handle more back-to-back events, + * so that it won't hold CPU too long or in case rd/wr idexes are + * corrupted which triggers infinite looping. + */ + if (++msg_processed >= max_msg_to_process) { + dev_warn(dev, "Q[%d] Per loop processing exceeds %d\n", + q_num, max_msg_to_process); + exit_loop = true; + } + } + } +idx_err: + mutex_unlock(&chan->msgq_mutex); + dev_dbg(dev, "total %d drained from queues\n", total); + + return total; +} + +/* + * init routine for all required data structures + */ +static int bcm_vk_data_init(struct bcm_vk *vk) +{ + int i; + + spin_lock_init(&vk->ctx_lock); + for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) { + vk->ctx[i].in_use = false; + vk->ctx[i].idx = i; /* self identity */ + vk->ctx[i].miscdev = NULL; + } + spin_lock_init(&vk->msg_id_lock); + spin_lock_init(&vk->host_alert_lock); + vk->msg_id = 0; + + /* initialize hash table */ + for (i = 0; i < VK_PID_HT_SZ; i++) + INIT_LIST_HEAD(&vk->pid_ht[i].head); + + return 0; +} + +irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id) +{ + struct bcm_vk *vk = dev_id; + + if (!bcm_vk_drv_access_ok(vk)) { + dev_err(&vk->pdev->dev, + "Interrupt %d received when msgq not inited\n", irq); + goto skip_schedule_work; + } + + queue_work(vk->wq_thread, &vk->wq_work); + +skip_schedule_work: + return IRQ_HANDLED; +} + +int bcm_vk_open(struct inode *inode, struct file *p_file) +{ + struct bcm_vk_ctx *ctx; + struct miscdevice *miscdev = (struct miscdevice *)p_file->private_data; + struct bcm_vk *vk = container_of(miscdev, struct bcm_vk, miscdev); + struct device *dev = &vk->pdev->dev; + int rc = 0; + + /* get a context and set it up for file */ + ctx = bcm_vk_get_ctx(vk, task_tgid_nr(current)); + if (!ctx) { + dev_err(dev, "Error allocating context\n"); + rc = -ENOMEM; + } else { + /* + * set up context and replace private data with context for + * other methods to use. Reason for the context is because + * it is allowed for multiple sessions to open the sysfs, and + * for each file open, when upper layer query the response, + * only those that are tied to a specific open should be + * returned. The context->idx will be used for such binding + */ + ctx->miscdev = miscdev; + p_file->private_data = ctx; + dev_dbg(dev, "ctx_returned with idx %d, pid %d\n", + ctx->idx, ctx->pid); + } + return rc; +} + +ssize_t bcm_vk_read(struct file *p_file, + char __user *buf, + size_t count, + loff_t *f_pos) +{ + ssize_t rc = -ENOMSG; + struct bcm_vk_ctx *ctx = p_file->private_data; + struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, + miscdev); + struct device *dev = &vk->pdev->dev; + struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan; + struct bcm_vk_wkent *entry = NULL; + u32 q_num; + u32 rsp_length; + bool found = false; + + if (!bcm_vk_drv_access_ok(vk)) + return -EPERM; + + dev_dbg(dev, "Buf count %zu\n", count); + found = false; + + /* + * search through the pendq on the to_h chan, and return only those + * that belongs to the same context. Search is always from the high to + * the low priority queues + */ + spin_lock(&chan->pendq_lock); + for (q_num = 0; q_num < chan->q_nr; q_num++) { + list_for_each_entry(entry, &chan->pendq[q_num], node) { + if (entry->ctx->idx == ctx->idx) { + if (count >= + (entry->to_h_blks * VK_MSGQ_BLK_SIZE)) { + list_del(&entry->node); + atomic_dec(&ctx->pend_cnt); + found = true; + } else { + /* buffer not big enough */ + rc = -EMSGSIZE; + } + goto read_loop_exit; + } + } + } +read_loop_exit: + spin_unlock(&chan->pendq_lock); + + if (found) { + /* retrieve the passed down msg_id */ + set_msg_id(&entry->to_h_msg[0], entry->usr_msg_id); + rsp_length = entry->to_h_blks * VK_MSGQ_BLK_SIZE; + if (copy_to_user(buf, entry->to_h_msg, rsp_length) == 0) + rc = rsp_length; + + bcm_vk_free_wkent(dev, entry); + } else if (rc == -EMSGSIZE) { + struct vk_msg_blk tmp_msg = entry->to_h_msg[0]; + + /* + * in this case, return just the first block, so + * that app knows what size it is looking for. + */ + set_msg_id(&tmp_msg, entry->usr_msg_id); + tmp_msg.size = entry->to_h_blks - 1; + if (copy_to_user(buf, &tmp_msg, VK_MSGQ_BLK_SIZE) != 0) { + dev_err(dev, "Error return 1st block in -EMSGSIZE\n"); + rc = -EFAULT; + } + } + return rc; +} + +ssize_t bcm_vk_write(struct file *p_file, + const char __user *buf, + size_t count, + loff_t *f_pos) +{ + ssize_t rc; + struct bcm_vk_ctx *ctx = p_file->private_data; + struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, + miscdev); + struct bcm_vk_msgq __iomem *msgq; + struct device *dev = &vk->pdev->dev; + struct bcm_vk_wkent *entry; + u32 sgl_extra_blks; + u32 q_num; + u32 msg_size; + u32 msgq_size; + + if (!bcm_vk_drv_access_ok(vk)) + return -EPERM; + + dev_dbg(dev, "Msg count %zu\n", count); + + /* first, do sanity check where count should be multiple of basic blk */ + if (count & (VK_MSGQ_BLK_SIZE - 1)) { + dev_err(dev, "Failure with size %zu not multiple of %zu\n", + count, VK_MSGQ_BLK_SIZE); + rc = -EINVAL; + goto write_err; + } + + /* allocate the work entry + buffer for size count and inband sgl */ + entry = kzalloc(sizeof(*entry) + count + vk->ib_sgl_size, + GFP_KERNEL); + if (!entry) { + rc = -ENOMEM; + goto write_err; + } + + /* now copy msg from user space, and then formulate the work entry */ + if (copy_from_user(&entry->to_v_msg[0], buf, count)) { + rc = -EFAULT; + goto write_free_ent; + } + + entry->to_v_blks = count >> VK_MSGQ_BLK_SZ_SHIFT; + entry->ctx = ctx; + + /* do a check on the blk size which could not exceed queue space */ + q_num = get_q_num(&entry->to_v_msg[0]); + msgq = vk->to_v_msg_chan.msgq[q_num]; + msgq_size = readl_relaxed(&msgq->size); + if (entry->to_v_blks + (vk->ib_sgl_size >> VK_MSGQ_BLK_SZ_SHIFT) + > (msgq_size - 1)) { + dev_err(dev, "Blk size %d exceed max queue size allowed %d\n", + entry->to_v_blks, msgq_size - 1); + rc = -EINVAL; + goto write_free_ent; + } + + /* Use internal message id */ + entry->usr_msg_id = get_msg_id(&entry->to_v_msg[0]); + rc = bcm_vk_get_msg_id(vk); + if (rc == VK_MSG_ID_OVERFLOW) { + dev_err(dev, "msg_id overflow\n"); + rc = -EOVERFLOW; + goto write_free_ent; + } + set_msg_id(&entry->to_v_msg[0], rc); + ctx->q_num = q_num; + + dev_dbg(dev, + "[Q-%d]Message ctx id %d, usr_msg_id 0x%x sent msg_id 0x%x\n", + ctx->q_num, ctx->idx, entry->usr_msg_id, + get_msg_id(&entry->to_v_msg[0])); + + if (entry->to_v_msg[0].function_id == VK_FID_TRANS_BUF) { + /* Convert any pointers to sg list */ + unsigned int num_planes; + int dir; + struct _vk_data *data; + + /* + * check if we are in reset, if so, no buffer transfer is + * allowed and return error. + */ + if (vk->reset_pid) { + dev_dbg(dev, "No Transfer allowed during reset, pid %d.\n", + ctx->pid); + rc = -EACCES; + goto write_free_msgid; + } + + num_planes = entry->to_v_msg[0].cmd & VK_CMD_PLANES_MASK; + if ((entry->to_v_msg[0].cmd & VK_CMD_MASK) == VK_CMD_DOWNLOAD) + dir = DMA_FROM_DEVICE; + else + dir = DMA_TO_DEVICE; + + /* Calculate vk_data location */ + /* Go to end of the message */ + msg_size = entry->to_v_msg[0].size; + if (msg_size > entry->to_v_blks) { + rc = -EMSGSIZE; + goto write_free_msgid; + } + + data = (struct _vk_data *)&entry->to_v_msg[msg_size + 1]; + + /* Now back up to the start of the pointers */ + data -= num_planes; + + /* Convert user addresses to DMA SG List */ + rc = bcm_vk_sg_alloc(dev, entry->dma, dir, data, num_planes); + if (rc) + goto write_free_msgid; + + atomic_inc(&ctx->dma_cnt); + /* try to embed inband sgl */ + sgl_extra_blks = bcm_vk_append_ib_sgl(vk, entry, data, + num_planes); + entry->to_v_blks += sgl_extra_blks; + entry->to_v_msg[0].size += sgl_extra_blks; + } else if (entry->to_v_msg[0].function_id == VK_FID_INIT && + entry->to_v_msg[0].context_id == VK_NEW_CTX) { + /* + * Init happens in 2 stages, only the first stage contains the + * pid that needs translating. + */ + pid_t org_pid, pid; + + /* + * translate the pid into the unique host space as user + * may run sessions inside containers or process + * namespaces. + */ +#define VK_MSG_PID_MASK 0xffffff00 +#define VK_MSG_PID_SH 8 + org_pid = (entry->to_v_msg[0].arg & VK_MSG_PID_MASK) + >> VK_MSG_PID_SH; + + pid = task_tgid_nr(current); + entry->to_v_msg[0].arg = + (entry->to_v_msg[0].arg & ~VK_MSG_PID_MASK) | + (pid << VK_MSG_PID_SH); + if (org_pid != pid) + dev_dbg(dev, "In PID 0x%x(%d), converted PID 0x%x(%d)\n", + org_pid, org_pid, pid, pid); + } + + /* + * store work entry to pending queue until a response is received. + * This needs to be done before enqueuing the message + */ + bcm_vk_append_pendq(&vk->to_v_msg_chan, q_num, entry); + + rc = bcm_to_v_msg_enqueue(vk, entry); + if (rc) { + dev_err(dev, "Fail to enqueue msg to to_v queue\n"); + + /* remove message from pending list */ + entry = bcm_vk_dequeue_pending + (vk, + &vk->to_v_msg_chan, + q_num, + get_msg_id(&entry->to_v_msg[0])); + goto write_free_ent; + } + + return count; + +write_free_msgid: + bcm_vk_msgid_bitmap_clear(vk, get_msg_id(&entry->to_v_msg[0]), 1); +write_free_ent: + kfree(entry); +write_err: + return rc; +} + +__poll_t bcm_vk_poll(struct file *p_file, struct poll_table_struct *wait) +{ + __poll_t ret = 0; + int cnt; + struct bcm_vk_ctx *ctx = p_file->private_data; + struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); + struct device *dev = &vk->pdev->dev; + + poll_wait(p_file, &ctx->rd_wq, wait); + + cnt = atomic_read(&ctx->pend_cnt); + if (cnt) { + ret = (__force __poll_t)(POLLIN | POLLRDNORM); + if (cnt < 0) { + dev_err(dev, "Error cnt %d, setting back to 0", cnt); + atomic_set(&ctx->pend_cnt, 0); + } + } + + return ret; +} + +int bcm_vk_release(struct inode *inode, struct file *p_file) +{ + int ret; + struct bcm_vk_ctx *ctx = p_file->private_data; + struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); + struct device *dev = &vk->pdev->dev; + pid_t pid = ctx->pid; + int dma_cnt; + unsigned long timeout, start_time; + + /* + * if there are outstanding DMA transactions, need to delay long enough + * to ensure that the card side would have stopped touching the host buffer + * and its SGL list. A race condition could happen if the host app is killed + * abruptly, eg kill -9, while some DMA transfer orders are still inflight. + * Nothing could be done except for a delay as host side is running in a + * completely async fashion. + */ + start_time = jiffies; + timeout = start_time + msecs_to_jiffies(BCM_VK_DMA_DRAIN_MAX_MS); + do { + if (time_after(jiffies, timeout)) { + dev_warn(dev, "%d dma still pending for [fd-%d] pid %d\n", + dma_cnt, ctx->idx, pid); + break; + } + dma_cnt = atomic_read(&ctx->dma_cnt); + cpu_relax(); + cond_resched(); + } while (dma_cnt); + dev_dbg(dev, "Draining for [fd-%d] pid %d - delay %d ms\n", + ctx->idx, pid, jiffies_to_msecs(jiffies - start_time)); + + bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, ctx); + bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, ctx); + + ret = bcm_vk_free_ctx(vk, ctx); + if (ret == 0) + ret = bcm_vk_handle_last_sess(vk, pid, ctx->q_num); + else + ret = 0; + + kref_put(&vk->kref, bcm_vk_release_data); + + return ret; +} + +int bcm_vk_msg_init(struct bcm_vk *vk) +{ + struct device *dev = &vk->pdev->dev; + int ret; + + if (bcm_vk_data_init(vk)) { + dev_err(dev, "Error initializing internal data structures\n"); + return -EINVAL; + } + + if (bcm_vk_msg_chan_init(&vk->to_v_msg_chan) || + bcm_vk_msg_chan_init(&vk->to_h_msg_chan)) { + dev_err(dev, "Error initializing communication channel\n"); + return -EIO; + } + + /* read msgq info if ready */ + ret = bcm_vk_sync_msgq(vk, false); + if (ret && (ret != -EAGAIN)) { + dev_err(dev, "Error reading comm msg Q info\n"); + return -EIO; + } + + return 0; +} + +void bcm_vk_msg_remove(struct bcm_vk *vk) +{ + bcm_vk_blk_drv_access(vk); + + /* drain all pending items */ + bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, NULL); + bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, NULL); +} + diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.h b/drivers/misc/bcm-vk/bcm_vk_msg.h new file mode 100644 index 000000000000..4eaad84825d6 --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk_msg.h @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2018-2020 Broadcom. + */ + +#ifndef BCM_VK_MSG_H +#define BCM_VK_MSG_H + +#include <uapi/linux/misc/bcm_vk.h> +#include "bcm_vk_sg.h" + +/* Single message queue control structure */ +struct bcm_vk_msgq { + u16 type; /* queue type */ + u16 num; /* queue number */ + u32 start; /* offset in BAR1 where the queue memory starts */ + + u32 rd_idx; /* read idx */ + u32 wr_idx; /* write idx */ + + u32 size; /* + * size, which is in number of 16byte blocks, + * to align with the message data structure. + */ + u32 nxt; /* + * nxt offset to the next msg queue struct. + * This is to provide flexibity for alignment purposes. + */ + +/* Least significant 16 bits in below field hold doorbell register offset */ +#define DB_SHIFT 16 + + u32 db_offset; /* queue doorbell register offset in BAR0 */ + + u32 rsvd; +}; + +/* + * Structure to record static info from the msgq sync. We keep local copy + * for some of these variables for both performance + checking purpose. + */ +struct bcm_vk_sync_qinfo { + void __iomem *q_start; + u32 q_size; + u32 q_mask; + u32 q_low; + u32 q_db_offset; +}; + +#define VK_MSGQ_MAX_NR 4 /* Maximum number of message queues */ + +/* + * message block - basic unit in the message where a message's size is always + * N x sizeof(basic_block) + */ +struct vk_msg_blk { + u8 function_id; +#define VK_FID_TRANS_BUF 5 +#define VK_FID_SHUTDOWN 8 +#define VK_FID_INIT 9 + u8 size; /* size of the message in number of vk_msg_blk's */ + u16 trans_id; /* transport id, queue & msg_id */ + u32 context_id; +#define VK_NEW_CTX 0 + u32 cmd; +#define VK_CMD_PLANES_MASK 0x000f /* number of planes to up/download */ +#define VK_CMD_UPLOAD 0x0400 /* memory transfer to vk */ +#define VK_CMD_DOWNLOAD 0x0500 /* memory transfer from vk */ +#define VK_CMD_MASK 0x0f00 /* command mask */ + u32 arg; +}; + +/* vk_msg_blk is 16 bytes fixed */ +#define VK_MSGQ_BLK_SIZE (sizeof(struct vk_msg_blk)) +/* shift for fast division of basic msg blk size */ +#define VK_MSGQ_BLK_SZ_SHIFT 4 + +/* use msg_id 0 for any simplex host2vk communication */ +#define VK_SIMPLEX_MSG_ID 0 + +/* context per session opening of sysfs */ +struct bcm_vk_ctx { + struct list_head node; /* use for linkage in Hash Table */ + unsigned int idx; + bool in_use; + pid_t pid; + u32 hash_idx; + u32 q_num; /* queue number used by the stream */ + struct miscdevice *miscdev; + atomic_t pend_cnt; /* number of items pending to be read from host */ + atomic_t dma_cnt; /* any dma transaction outstanding */ + wait_queue_head_t rd_wq; +}; + +/* pid hash table entry */ +struct bcm_vk_ht_entry { + struct list_head head; +}; + +#define VK_DMA_MAX_ADDRS 4 /* Max 4 DMA Addresses */ +/* structure for house keeping a single work entry */ +struct bcm_vk_wkent { + struct list_head node; /* for linking purpose */ + struct bcm_vk_ctx *ctx; + + /* Store up to 4 dma pointers */ + struct bcm_vk_dma dma[VK_DMA_MAX_ADDRS]; + + u32 to_h_blks; /* response */ + struct vk_msg_blk *to_h_msg; + + /* + * put the to_v_msg at the end so that we could simply append to_v msg + * to the end of the allocated block + */ + u32 usr_msg_id; + u32 to_v_blks; + u32 seq_num; + struct vk_msg_blk to_v_msg[0]; +}; + +/* queue stats counters */ +struct bcm_vk_qs_cnts { + u32 cnt; /* general counter, used to limit output */ + u32 acc_sum; + u32 max_occ; /* max during a sampling period */ + u32 max_abs; /* the abs max since reset */ +}; + +/* control channel structure for either to_v or to_h communication */ +struct bcm_vk_msg_chan { + u32 q_nr; + /* Mutex to access msgq */ + struct mutex msgq_mutex; + /* pointing to BAR locations */ + struct bcm_vk_msgq __iomem *msgq[VK_MSGQ_MAX_NR]; + /* Spinlock to access pending queue */ + spinlock_t pendq_lock; + /* for temporary storing pending items, one for each queue */ + struct list_head pendq[VK_MSGQ_MAX_NR]; + /* static queue info from the sync */ + struct bcm_vk_sync_qinfo sync_qinfo[VK_MSGQ_MAX_NR]; +}; + +/* totol number of message q allowed by the driver */ +#define VK_MSGQ_PER_CHAN_MAX 3 +#define VK_MSGQ_NUM_DEFAULT (VK_MSGQ_PER_CHAN_MAX - 1) + +/* total number of supported ctx, 32 ctx each for 5 components */ +#define VK_CMPT_CTX_MAX (32 * 5) + +/* hash table defines to store the opened FDs */ +#define VK_PID_HT_SHIFT_BIT 7 /* 128 */ +#define VK_PID_HT_SZ BIT(VK_PID_HT_SHIFT_BIT) + +/* The following are offsets of DDR info provided by the vk card */ +#define VK_BAR0_SEG_SIZE (4 * SZ_1K) /* segment size for BAR0 */ + +/* shutdown types supported */ +#define VK_SHUTDOWN_PID 1 +#define VK_SHUTDOWN_GRACEFUL 2 + +#endif diff --git a/drivers/misc/bcm-vk/bcm_vk_sg.c b/drivers/misc/bcm-vk/bcm_vk_sg.c new file mode 100644 index 000000000000..2e9daaf3e492 --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk_sg.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2018-2020 Broadcom. + */ +#include <linux/dma-mapping.h> +#include <linux/mm.h> +#include <linux/pagemap.h> +#include <linux/pgtable.h> +#include <linux/vmalloc.h> + +#include <asm/page.h> +#include <asm/unaligned.h> + +#include <uapi/linux/misc/bcm_vk.h> + +#include "bcm_vk.h" +#include "bcm_vk_msg.h" +#include "bcm_vk_sg.h" + +/* + * Valkyrie has a hardware limitation of 16M transfer size. + * So limit the SGL chunks to 16M. + */ +#define BCM_VK_MAX_SGL_CHUNK SZ_16M + +static int bcm_vk_dma_alloc(struct device *dev, + struct bcm_vk_dma *dma, + int dir, + struct _vk_data *vkdata); +static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma); + +/* Uncomment to dump SGLIST */ +/* #define BCM_VK_DUMP_SGLIST */ + +static int bcm_vk_dma_alloc(struct device *dev, + struct bcm_vk_dma *dma, + int direction, + struct _vk_data *vkdata) +{ + dma_addr_t addr, sg_addr; + int err; + int i; + int offset; + u32 size; + u32 remaining_size; + u32 transfer_size; + u64 data; + unsigned long first, last; + struct _vk_data *sgdata; + + /* Get 64-bit user address */ + data = get_unaligned(&vkdata->address); + + /* offset into first page */ + offset = offset_in_page(data); + + /* Calculate number of pages */ + first = (data & PAGE_MASK) >> PAGE_SHIFT; + last = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT; + dma->nr_pages = last - first + 1; + + /* Allocate DMA pages */ + dma->pages = kmalloc_array(dma->nr_pages, + sizeof(struct page *), + GFP_KERNEL); + if (!dma->pages) + return -ENOMEM; + + dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n", + data, vkdata->size, dma->nr_pages); + + dma->direction = direction; + + /* Get user pages into memory */ + err = get_user_pages_fast(data & PAGE_MASK, + dma->nr_pages, + direction == DMA_FROM_DEVICE, + dma->pages); + if (err != dma->nr_pages) { + dma->nr_pages = (err >= 0) ? err : 0; + dev_err(dev, "get_user_pages_fast, err=%d [%d]\n", + err, dma->nr_pages); + return err < 0 ? err : -EINVAL; + } + + /* Max size of sg list is 1 per mapped page + fields at start */ + dma->sglen = (dma->nr_pages * sizeof(*sgdata)) + + (sizeof(u32) * SGLIST_VKDATA_START); + + /* Allocate sglist */ + dma->sglist = dma_alloc_coherent(dev, + dma->sglen, + &dma->handle, + GFP_KERNEL); + if (!dma->sglist) + return -ENOMEM; + + dma->sglist[SGLIST_NUM_SG] = 0; + dma->sglist[SGLIST_TOTALSIZE] = vkdata->size; + remaining_size = vkdata->size; + sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START]; + + /* Map all pages into DMA */ + size = min_t(size_t, PAGE_SIZE - offset, remaining_size); + remaining_size -= size; + sg_addr = dma_map_page(dev, + dma->pages[0], + offset, + size, + dma->direction); + transfer_size = size; + if (unlikely(dma_mapping_error(dev, sg_addr))) { + __free_page(dma->pages[0]); + return -EIO; + } + + for (i = 1; i < dma->nr_pages; i++) { + size = min_t(size_t, PAGE_SIZE, remaining_size); + remaining_size -= size; + addr = dma_map_page(dev, + dma->pages[i], + 0, + size, + dma->direction); + if (unlikely(dma_mapping_error(dev, addr))) { + __free_page(dma->pages[i]); + return -EIO; + } + + /* + * Compress SG list entry when pages are contiguous + * and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK + */ + if ((addr == (sg_addr + transfer_size)) && + ((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) { + /* pages are contiguous, add to same sg entry */ + transfer_size += size; + } else { + /* pages are not contiguous, write sg entry */ + sgdata->size = transfer_size; + put_unaligned(sg_addr, (u64 *)&sgdata->address); + dma->sglist[SGLIST_NUM_SG]++; + + /* start new sg entry */ + sgdata++; + sg_addr = addr; + transfer_size = size; + } + } + /* Write last sg list entry */ + sgdata->size = transfer_size; + put_unaligned(sg_addr, (u64 *)&sgdata->address); + dma->sglist[SGLIST_NUM_SG]++; + + /* Update pointers and size field to point to sglist */ + put_unaligned((u64)dma->handle, &vkdata->address); + vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) + + (sizeof(u32) * SGLIST_VKDATA_START); + +#ifdef BCM_VK_DUMP_SGLIST + dev_dbg(dev, + "sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n", + (u64)dma->sglist, + dma->handle, + dma->sglen, + vkdata->size); + for (i = 0; i < vkdata->size / sizeof(u32); i++) + dev_dbg(dev, "i:0x%x 0x%x\n", i, dma->sglist[i]); +#endif + + return 0; +} + +int bcm_vk_sg_alloc(struct device *dev, + struct bcm_vk_dma *dma, + int dir, + struct _vk_data *vkdata, + int num) +{ + int i; + int rc = -EINVAL; + + /* Convert user addresses to DMA SG List */ + for (i = 0; i < num; i++) { + if (vkdata[i].size && vkdata[i].address) { + /* + * If both size and address are non-zero + * then DMA alloc. + */ + rc = bcm_vk_dma_alloc(dev, + &dma[i], + dir, + &vkdata[i]); + } else if (vkdata[i].size || + vkdata[i].address) { + /* + * If one of size and address are zero + * there is a problem. + */ + dev_err(dev, + "Invalid vkdata %x 0x%x 0x%llx\n", + i, vkdata[i].size, vkdata[i].address); + rc = -EINVAL; + } else { + /* + * If size and address are both zero + * don't convert, but return success. + */ + rc = 0; + } + + if (rc) + goto fail_alloc; + } + return rc; + +fail_alloc: + while (i > 0) { + i--; + if (dma[i].sglist) + bcm_vk_dma_free(dev, &dma[i]); + } + return rc; +} + +static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma) +{ + dma_addr_t addr; + int i; + int num_sg; + u32 size; + struct _vk_data *vkdata; + + dev_dbg(dev, "free sglist=%p sglen=0x%x\n", dma->sglist, dma->sglen); + + /* Unmap all pages in the sglist */ + num_sg = dma->sglist[SGLIST_NUM_SG]; + vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START]; + for (i = 0; i < num_sg; i++) { + size = vkdata[i].size; + addr = get_unaligned(&vkdata[i].address); + + dma_unmap_page(dev, addr, size, dma->direction); + } + + /* Free allocated sglist */ + dma_free_coherent(dev, dma->sglen, dma->sglist, dma->handle); + + /* Release lock on all pages */ + for (i = 0; i < dma->nr_pages; i++) + put_page(dma->pages[i]); + + /* Free allocated dma pages */ + kfree(dma->pages); + dma->sglist = NULL; + + return 0; +} + +int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num, + int *proc_cnt) +{ + int i; + + *proc_cnt = 0; + /* Unmap and free all pages and sglists */ + for (i = 0; i < num; i++) { + if (dma[i].sglist) { + bcm_vk_dma_free(dev, &dma[i]); + *proc_cnt += 1; + } + } + + return 0; +} diff --git a/drivers/misc/bcm-vk/bcm_vk_sg.h b/drivers/misc/bcm-vk/bcm_vk_sg.h new file mode 100644 index 000000000000..81b3d0976ddb --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk_sg.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2018-2020 Broadcom. + */ + +#ifndef BCM_VK_SG_H +#define BCM_VK_SG_H + +#include <linux/dma-mapping.h> + +struct bcm_vk_dma { + /* for userland buffer */ + struct page **pages; + int nr_pages; + + /* common */ + dma_addr_t handle; + /* + * sglist is of the following LE format + * [U32] num_sg = number of sg addresses (N) + * [U32] totalsize = totalsize of data being transferred in sglist + * [U32] size[0] = size of data in address0 + * [U32] addr_l[0] = lower 32-bits of address0 + * [U32] addr_h[0] = higher 32-bits of address0 + * .. + * [U32] size[N-1] = size of data in addressN-1 + * [U32] addr_l[N-1] = lower 32-bits of addressN-1 + * [U32] addr_h[N-1] = higher 32-bits of addressN-1 + */ + u32 *sglist; +#define SGLIST_NUM_SG 0 +#define SGLIST_TOTALSIZE 1 +#define SGLIST_VKDATA_START 2 + + int sglen; /* Length (bytes) of sglist */ + int direction; +}; + +struct _vk_data { + u32 size; /* data size in bytes */ + u64 address; /* Pointer to data */ +} __packed; + +/* + * Scatter-gather DMA buffer API. + * + * These functions provide a simple way to create a page list and a + * scatter-gather list from userspace address and map the memory + * for DMA operation. + */ +int bcm_vk_sg_alloc(struct device *dev, + struct bcm_vk_dma *dma, + int dir, + struct _vk_data *vkdata, + int num); + +int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num, + int *proc_cnt); + +#endif + diff --git a/drivers/misc/bcm-vk/bcm_vk_tty.c b/drivers/misc/bcm-vk/bcm_vk_tty.c new file mode 100644 index 000000000000..4d02692ecfc7 --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk_tty.c @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2018-2020 Broadcom. + */ + +#include <linux/tty.h> +#include <linux/tty_driver.h> +#include <linux/tty_flip.h> + +#include "bcm_vk.h" + +/* TTYVK base offset is 0x30000 into BAR1 */ +#define BAR1_TTYVK_BASE_OFFSET 0x300000 +/* Each TTYVK channel (TO or FROM) is 0x10000 */ +#define BAR1_TTYVK_CHAN_OFFSET 0x100000 +/* Each TTYVK channel has TO and FROM, hence the * 2 */ +#define BAR1_TTYVK_BASE(index) (BAR1_TTYVK_BASE_OFFSET + \ + ((index) * BAR1_TTYVK_CHAN_OFFSET * 2)) +/* TO TTYVK channel base comes before FROM for each index */ +#define TO_TTYK_BASE(index) BAR1_TTYVK_BASE(index) +#define FROM_TTYK_BASE(index) (BAR1_TTYVK_BASE(index) + \ + BAR1_TTYVK_CHAN_OFFSET) + +struct bcm_vk_tty_chan { + u32 reserved; + u32 size; + u32 wr; + u32 rd; + u32 *data; +}; + +#define VK_BAR_CHAN(v, DIR, e) ((v)->DIR##_offset \ + + offsetof(struct bcm_vk_tty_chan, e)) +#define VK_BAR_CHAN_SIZE(v, DIR) VK_BAR_CHAN(v, DIR, size) +#define VK_BAR_CHAN_WR(v, DIR) VK_BAR_CHAN(v, DIR, wr) +#define VK_BAR_CHAN_RD(v, DIR) VK_BAR_CHAN(v, DIR, rd) +#define VK_BAR_CHAN_DATA(v, DIR, off) (VK_BAR_CHAN(v, DIR, data) + (off)) + +#define VK_BAR0_REGSEG_TTY_DB_OFFSET 0x86c + +/* Poll every 1/10 of second - temp hack till we use MSI interrupt */ +#define SERIAL_TIMER_VALUE (HZ / 10) + +static void bcm_vk_tty_poll(struct timer_list *t) +{ + struct bcm_vk *vk = from_timer(vk, t, serial_timer); + + queue_work(vk->tty_wq_thread, &vk->tty_wq_work); + mod_timer(&vk->serial_timer, jiffies + SERIAL_TIMER_VALUE); +} + +irqreturn_t bcm_vk_tty_irqhandler(int irq, void *dev_id) +{ + struct bcm_vk *vk = dev_id; + + queue_work(vk->tty_wq_thread, &vk->tty_wq_work); + + return IRQ_HANDLED; +} + +static void bcm_vk_tty_wq_handler(struct work_struct *work) +{ + struct bcm_vk *vk = container_of(work, struct bcm_vk, tty_wq_work); + struct bcm_vk_tty *vktty; + int card_status; + int count; + unsigned char c; + int i; + int wr; + + card_status = vkread32(vk, BAR_0, BAR_CARD_STATUS); + if (BCM_VK_INTF_IS_DOWN(card_status)) + return; + + for (i = 0; i < BCM_VK_NUM_TTY; i++) { + count = 0; + /* Check the card status that the tty channel is ready */ + if ((card_status & BIT(i)) == 0) + continue; + + vktty = &vk->tty[i]; + + /* Don't increment read index if tty app is closed */ + if (!vktty->is_opened) + continue; + + /* Fetch the wr offset in buffer from VK */ + wr = vkread32(vk, BAR_1, VK_BAR_CHAN_WR(vktty, from)); + + /* safe to ignore until bar read gives proper size */ + if (vktty->from_size == 0) + continue; + + if (wr >= vktty->from_size) { + dev_err(&vk->pdev->dev, + "ERROR: wq handler ttyVK%d wr:0x%x > 0x%x\n", + i, wr, vktty->from_size); + /* Need to signal and close device in this case */ + continue; + } + + /* + * Simple read of circular buffer and + * insert into tty flip buffer + */ + while (vk->tty[i].rd != wr) { + c = vkread8(vk, BAR_1, + VK_BAR_CHAN_DATA(vktty, from, vktty->rd)); + vktty->rd++; + if (vktty->rd >= vktty->from_size) + vktty->rd = 0; + tty_insert_flip_char(&vktty->port, c, TTY_NORMAL); + count++; + } + + if (count) { + tty_flip_buffer_push(&vktty->port); + + /* Update read offset from shadow register to card */ + vkwrite32(vk, vktty->rd, BAR_1, + VK_BAR_CHAN_RD(vktty, from)); + } + } +} + +static int bcm_vk_tty_open(struct tty_struct *tty, struct file *file) +{ + int card_status; + struct bcm_vk *vk; + struct bcm_vk_tty *vktty; + int index; + + /* initialize the pointer in case something fails */ + tty->driver_data = NULL; + + vk = (struct bcm_vk *)dev_get_drvdata(tty->dev); + index = tty->index; + + if (index >= BCM_VK_NUM_TTY) + return -EINVAL; + + vktty = &vk->tty[index]; + + vktty->pid = task_pid_nr(current); + vktty->to_offset = TO_TTYK_BASE(index); + vktty->from_offset = FROM_TTYK_BASE(index); + + /* Do not allow tty device to be opened if tty on card not ready */ + card_status = vkread32(vk, BAR_0, BAR_CARD_STATUS); + if (BCM_VK_INTF_IS_DOWN(card_status) || ((card_status & BIT(index)) == 0)) + return -EBUSY; + + /* + * Get shadow registers of the buffer sizes and the "to" write offset + * and "from" read offset + */ + vktty->to_size = vkread32(vk, BAR_1, VK_BAR_CHAN_SIZE(vktty, to)); + vktty->wr = vkread32(vk, BAR_1, VK_BAR_CHAN_WR(vktty, to)); + vktty->from_size = vkread32(vk, BAR_1, VK_BAR_CHAN_SIZE(vktty, from)); + vktty->rd = vkread32(vk, BAR_1, VK_BAR_CHAN_RD(vktty, from)); + vktty->is_opened = true; + + if (tty->count == 1 && !vktty->irq_enabled) { + timer_setup(&vk->serial_timer, bcm_vk_tty_poll, 0); + mod_timer(&vk->serial_timer, jiffies + SERIAL_TIMER_VALUE); + } + return 0; +} + +static void bcm_vk_tty_close(struct tty_struct *tty, struct file *file) +{ + struct bcm_vk *vk = dev_get_drvdata(tty->dev); + + if (tty->index >= BCM_VK_NUM_TTY) + return; + + vk->tty[tty->index].is_opened = false; + + if (tty->count == 1) + del_timer_sync(&vk->serial_timer); +} + +static void bcm_vk_tty_doorbell(struct bcm_vk *vk, u32 db_val) +{ + vkwrite32(vk, db_val, BAR_0, + VK_BAR0_REGSEG_DB_BASE + VK_BAR0_REGSEG_TTY_DB_OFFSET); +} + +static int bcm_vk_tty_write(struct tty_struct *tty, + const unsigned char *buffer, + int count) +{ + int index; + struct bcm_vk *vk; + struct bcm_vk_tty *vktty; + int i; + + index = tty->index; + vk = dev_get_drvdata(tty->dev); + vktty = &vk->tty[index]; + + /* Simple write each byte to circular buffer */ + for (i = 0; i < count; i++) { + vkwrite8(vk, buffer[i], BAR_1, + VK_BAR_CHAN_DATA(vktty, to, vktty->wr)); + vktty->wr++; + if (vktty->wr >= vktty->to_size) + vktty->wr = 0; + } + /* Update write offset from shadow register to card */ + vkwrite32(vk, vktty->wr, BAR_1, VK_BAR_CHAN_WR(vktty, to)); + bcm_vk_tty_doorbell(vk, 0); + + return count; +} + +static int bcm_vk_tty_write_room(struct tty_struct *tty) +{ + struct bcm_vk *vk = dev_get_drvdata(tty->dev); + + return vk->tty[tty->index].to_size - 1; +} + +static const struct tty_operations serial_ops = { + .open = bcm_vk_tty_open, + .close = bcm_vk_tty_close, + .write = bcm_vk_tty_write, + .write_room = bcm_vk_tty_write_room, +}; + +int bcm_vk_tty_init(struct bcm_vk *vk, char *name) +{ + int i; + int err; + struct tty_driver *tty_drv; + struct device *dev = &vk->pdev->dev; + + tty_drv = tty_alloc_driver + (BCM_VK_NUM_TTY, + TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV); + if (IS_ERR(tty_drv)) + return PTR_ERR(tty_drv); + + /* Save struct tty_driver for uninstalling the device */ + vk->tty_drv = tty_drv; + + /* initialize the tty driver */ + tty_drv->driver_name = KBUILD_MODNAME; + tty_drv->name = kstrdup(name, GFP_KERNEL); + if (!tty_drv->name) { + err = -ENOMEM; + goto err_put_tty_driver; + } + tty_drv->type = TTY_DRIVER_TYPE_SERIAL; + tty_drv->subtype = SERIAL_TYPE_NORMAL; + tty_drv->init_termios = tty_std_termios; + tty_set_operations(tty_drv, &serial_ops); + + /* register the tty driver */ + err = tty_register_driver(tty_drv); + if (err) { + dev_err(dev, "tty_register_driver failed\n"); + goto err_kfree_tty_name; + } + + for (i = 0; i < BCM_VK_NUM_TTY; i++) { + struct device *tty_dev; + + tty_port_init(&vk->tty[i].port); + tty_dev = tty_port_register_device(&vk->tty[i].port, tty_drv, + i, dev); + if (IS_ERR(tty_dev)) { + err = PTR_ERR(tty_dev); + goto unwind; + } + dev_set_drvdata(tty_dev, vk); + vk->tty[i].is_opened = false; + } + + INIT_WORK(&vk->tty_wq_work, bcm_vk_tty_wq_handler); + vk->tty_wq_thread = create_singlethread_workqueue("tty"); + if (!vk->tty_wq_thread) { + dev_err(dev, "Fail to create tty workqueue thread\n"); + err = -ENOMEM; + goto unwind; + } + return 0; + +unwind: + while (--i >= 0) + tty_port_unregister_device(&vk->tty[i].port, tty_drv, i); + tty_unregister_driver(tty_drv); + +err_kfree_tty_name: + kfree(tty_drv->name); + tty_drv->name = NULL; + +err_put_tty_driver: + put_tty_driver(tty_drv); + + return err; +} + +void bcm_vk_tty_exit(struct bcm_vk *vk) +{ + int i; + + del_timer_sync(&vk->serial_timer); + for (i = 0; i < BCM_VK_NUM_TTY; ++i) { + tty_port_unregister_device(&vk->tty[i].port, + vk->tty_drv, + i); + tty_port_destroy(&vk->tty[i].port); + } + tty_unregister_driver(vk->tty_drv); + + kfree(vk->tty_drv->name); + vk->tty_drv->name = NULL; + + put_tty_driver(vk->tty_drv); +} + +void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk) +{ + struct bcm_vk_tty *vktty; + int i; + + for (i = 0; i < BCM_VK_NUM_TTY; ++i) { + vktty = &vk->tty[i]; + if (vktty->pid) + kill_pid(find_vpid(vktty->pid), SIGKILL, 1); + } +} + +void bcm_vk_tty_wq_exit(struct bcm_vk *vk) +{ + cancel_work_sync(&vk->tty_wq_work); + destroy_workqueue(vk->tty_wq_thread); +} diff --git a/drivers/misc/cardreader/rts5227.c b/drivers/misc/cardreader/rts5227.c index 8859011672cb..8200af22b529 100644 --- a/drivers/misc/cardreader/rts5227.c +++ b/drivers/misc/cardreader/rts5227.c @@ -398,6 +398,11 @@ static int rts522a_extra_init_hw(struct rtsx_pcr *pcr) { rts5227_extra_init_hw(pcr); + /* Power down OCP for power consumption */ + if (!pcr->card_exist) + rtsx_pci_write_register(pcr, FPDCTL, OC_POWER_DOWN, + OC_POWER_DOWN); + rtsx_pci_write_register(pcr, FUNC_FORCE_CTL, FUNC_FORCE_UPME_XMT_DBG, FUNC_FORCE_UPME_XMT_DBG); rtsx_pci_write_register(pcr, PCLK_CTL, 0x04, 0x04); diff --git a/drivers/misc/cardreader/rtsx_pcr.c b/drivers/misc/cardreader/rtsx_pcr.c index 5a491d2cd1ae..273311184669 100644 --- a/drivers/misc/cardreader/rtsx_pcr.c +++ b/drivers/misc/cardreader/rtsx_pcr.c @@ -59,12 +59,6 @@ static const struct pci_device_id rtsx_pci_ids[] = { MODULE_DEVICE_TABLE(pci, rtsx_pci_ids); -static inline void rtsx_pci_disable_aspm(struct rtsx_pcr *pcr) -{ - pcie_capability_clear_and_set_word(pcr->pci, PCI_EXP_LNKCTL, - PCI_EXP_LNKCTL_ASPMC, 0); -} - static int rtsx_comm_set_ltr_latency(struct rtsx_pcr *pcr, u32 latency) { rtsx_pci_write_register(pcr, MSGTXDATA0, @@ -1805,7 +1799,6 @@ static int rtsx_pci_runtime_resume(struct device *device) struct pci_dev *pcidev = to_pci_dev(device); struct pcr_handle *handle; struct rtsx_pcr *pcr; - int ret = 0; handle = pci_get_drvdata(pcidev); pcr = handle->pcr; @@ -1830,7 +1823,7 @@ static int rtsx_pci_runtime_resume(struct device *device) schedule_delayed_work(&pcr->idle_work, msecs_to_jiffies(200)); mutex_unlock(&pcr->pcr_mutex); - return ret; + return 0; } #else /* CONFIG_PM */ diff --git a/drivers/misc/cxl/sysfs.c b/drivers/misc/cxl/sysfs.c index d97a243ad30c..c173a5e88c91 100644 --- a/drivers/misc/cxl/sysfs.c +++ b/drivers/misc/cxl/sysfs.c @@ -178,7 +178,7 @@ static ssize_t perst_reloads_same_image_store(struct device *device, if ((rc != 1) || !(val == 1 || val == 0)) return -EINVAL; - adapter->perst_same_image = (val == 1 ? true : false); + adapter->perst_same_image = (val == 1); return count; } diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c index 7c45f82b4302..80114f4c80ad 100644 --- a/drivers/misc/eeprom/eeprom_93xx46.c +++ b/drivers/misc/eeprom/eeprom_93xx46.c @@ -35,6 +35,10 @@ static const struct eeprom_93xx46_devtype_data atmel_at93c46d_data = { EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH, }; +static const struct eeprom_93xx46_devtype_data microchip_93lc46b_data = { + .quirks = EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE, +}; + struct eeprom_93xx46_dev { struct spi_device *spi; struct eeprom_93xx46_platform_data *pdata; @@ -55,6 +59,11 @@ static inline bool has_quirk_instruction_length(struct eeprom_93xx46_dev *edev) return edev->pdata->quirks & EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH; } +static inline bool has_quirk_extra_read_cycle(struct eeprom_93xx46_dev *edev) +{ + return edev->pdata->quirks & EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE; +} + static int eeprom_93xx46_read(void *priv, unsigned int off, void *val, size_t count) { @@ -96,6 +105,11 @@ static int eeprom_93xx46_read(void *priv, unsigned int off, dev_dbg(&edev->spi->dev, "read cmd 0x%x, %d Hz\n", cmd_addr, edev->spi->max_speed_hz); + if (has_quirk_extra_read_cycle(edev)) { + cmd_addr <<= 1; + bits += 1; + } + spi_message_init(&m); t[0].tx_buf = (char *)&cmd_addr; @@ -363,6 +377,7 @@ static void select_deassert(void *context) static const struct of_device_id eeprom_93xx46_of_table[] = { { .compatible = "eeprom-93xx46", }, { .compatible = "atmel,at93c46d", .data = &atmel_at93c46d_data, }, + { .compatible = "microchip,93lc46b", .data = µchip_93lc46b_data, }, {} }; MODULE_DEVICE_TABLE(of, eeprom_93xx46_of_table); @@ -512,3 +527,5 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Driver for 93xx46 EEPROMs"); MODULE_AUTHOR("Anatolij Gustschin <agust@denx.de>"); MODULE_ALIAS("spi:93xx46"); +MODULE_ALIAS("spi:eeprom-93xx46"); +MODULE_ALIAS("spi:93lc46b"); diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index 70eb5ed942d0..f12e909034ac 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -520,12 +520,13 @@ fastrpc_map_dma_buf(struct dma_buf_attachment *attachment, { struct fastrpc_dma_buf_attachment *a = attachment->priv; struct sg_table *table; + int ret; table = &a->sgt; - if (!dma_map_sgtable(attachment->dev, table, dir, 0)) - return ERR_PTR(-ENOMEM); - + ret = dma_map_sgtable(attachment->dev, table, dir, 0); + if (ret) + table = ERR_PTR(ret); return table; } diff --git a/drivers/misc/habanalabs/common/Makefile b/drivers/misc/habanalabs/common/Makefile index eccd8c7dc62d..5d8b48288cf4 100644 --- a/drivers/misc/habanalabs/common/Makefile +++ b/drivers/misc/habanalabs/common/Makefile @@ -1,7 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-only + +include $(src)/common/mmu/Makefile +habanalabs-y += $(HL_COMMON_MMU_FILES) + +include $(src)/common/pci/Makefile +habanalabs-y += $(HL_COMMON_PCI_FILES) + HL_COMMON_FILES := common/habanalabs_drv.o common/device.o common/context.o \ common/asid.o common/habanalabs_ioctl.o \ common/command_buffer.o common/hw_queue.o common/irq.o \ common/sysfs.o common/hwmon.o common/memory.o \ - common/command_submission.o common/mmu.o common/mmu_v1.o \ - common/firmware_if.o common/pci.o + common/command_submission.o common/firmware_if.o diff --git a/drivers/misc/habanalabs/common/asid.c b/drivers/misc/habanalabs/common/asid.c index a2fdf31cf27c..ede04c032b6e 100644 --- a/drivers/misc/habanalabs/common/asid.c +++ b/drivers/misc/habanalabs/common/asid.c @@ -50,8 +50,10 @@ unsigned long hl_asid_alloc(struct hl_device *hdev) void hl_asid_free(struct hl_device *hdev, unsigned long asid) { - if (WARN((asid == 0 || asid >= hdev->asic_prop.max_asid), - "Invalid ASID %lu", asid)) + if (asid == HL_KERNEL_ASID_ID || asid >= hdev->asic_prop.max_asid) { + dev_crit(hdev->dev, "Invalid ASID %lu", asid); return; + } + clear_bit(asid, hdev->asid_bitmap); } diff --git a/drivers/misc/habanalabs/common/command_buffer.c b/drivers/misc/habanalabs/common/command_buffer.c index 6f6a904ab6ca..d9adb9a5e4d8 100644 --- a/drivers/misc/habanalabs/common/command_buffer.c +++ b/drivers/misc/habanalabs/common/command_buffer.c @@ -635,10 +635,12 @@ struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size, cb_handle >>= PAGE_SHIFT; cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) cb_handle); - /* hl_cb_get should never fail here so use kernel WARN */ - WARN(!cb, "Kernel CB handle invalid 0x%x\n", (u32) cb_handle); - if (!cb) + /* hl_cb_get should never fail here */ + if (!cb) { + dev_crit(hdev->dev, "Kernel CB handle invalid 0x%x\n", + (u32) cb_handle); goto destroy_cb; + } return cb; diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index b2b3d2b0f808..7bd4a03b3429 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -48,8 +48,8 @@ void hl_sob_reset_error(struct kref *ref) struct hl_device *hdev = hw_sob->hdev; dev_crit(hdev->dev, - "SOB release shouldn't be called here, q_idx: %d, sob_id: %d\n", - hw_sob->q_idx, hw_sob->sob_id); + "SOB release shouldn't be called here, q_idx: %d, sob_id: %d\n", + hw_sob->q_idx, hw_sob->sob_id); } /** @@ -149,9 +149,10 @@ void hl_fence_get(struct hl_fence *fence) kref_get(&fence->refcount); } -static void hl_fence_init(struct hl_fence *fence) +static void hl_fence_init(struct hl_fence *fence, u64 sequence) { kref_init(&fence->refcount); + fence->cs_sequence = sequence; fence->error = 0; fence->timestamp = ktime_set(0, 0); init_completion(&fence->completion); @@ -184,6 +185,28 @@ static void cs_job_put(struct hl_cs_job *job) kref_put(&job->refcount, cs_job_do_release); } +bool cs_needs_completion(struct hl_cs *cs) +{ + /* In case this is a staged CS, only the last CS in sequence should + * get a completion, any non staged CS will always get a completion + */ + if (cs->staged_cs && !cs->staged_last) + return false; + + return true; +} + +bool cs_needs_timeout(struct hl_cs *cs) +{ + /* In case this is a staged CS, only the first CS in sequence should + * get a timeout, any non staged CS will always get a timeout + */ + if (cs->staged_cs && !cs->staged_first) + return false; + + return true; +} + static bool is_cb_patched(struct hl_device *hdev, struct hl_cs_job *job) { /* @@ -225,6 +248,7 @@ static int cs_parser(struct hl_fpriv *hpriv, struct hl_cs_job *job) parser.queue_type = job->queue_type; parser.is_kernel_allocated_cb = job->is_kernel_allocated_cb; job->patched_cb = NULL; + parser.completion = cs_needs_completion(job->cs); rc = hdev->asic_funcs->cs_parser(hdev, &parser); @@ -290,13 +314,153 @@ static void complete_job(struct hl_device *hdev, struct hl_cs_job *job) hl_debugfs_remove_job(hdev, job); - if (job->queue_type == QUEUE_TYPE_EXT || - job->queue_type == QUEUE_TYPE_HW) + /* We decrement reference only for a CS that gets completion + * because the reference was incremented only for this kind of CS + * right before it was scheduled. + * + * In staged submission, only the last CS marked as 'staged_last' + * gets completion, hence its release function will be called from here. + * As for all the rest CS's in the staged submission which do not get + * completion, their CS reference will be decremented by the + * 'staged_last' CS during the CS release flow. + * All relevant PQ CI counters will be incremented during the CS release + * flow by calling 'hl_hw_queue_update_ci'. + */ + if (cs_needs_completion(cs) && + (job->queue_type == QUEUE_TYPE_EXT || + job->queue_type == QUEUE_TYPE_HW)) cs_put(cs); cs_job_put(job); } +/* + * hl_staged_cs_find_first - locate the first CS in this staged submission + * + * @hdev: pointer to device structure + * @cs_seq: staged submission sequence number + * + * @note: This function must be called under 'hdev->cs_mirror_lock' + * + * Find and return a CS pointer with the given sequence + */ +struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq) +{ + struct hl_cs *cs; + + list_for_each_entry_reverse(cs, &hdev->cs_mirror_list, mirror_node) + if (cs->staged_cs && cs->staged_first && + cs->sequence == cs_seq) + return cs; + + return NULL; +} + +/* + * is_staged_cs_last_exists - returns true if the last CS in sequence exists + * + * @hdev: pointer to device structure + * @cs: staged submission member + * + */ +bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs) +{ + struct hl_cs *last_entry; + + last_entry = list_last_entry(&cs->staged_cs_node, struct hl_cs, + staged_cs_node); + + if (last_entry->staged_last) + return true; + + return false; +} + +/* + * staged_cs_get - get CS reference if this CS is a part of a staged CS + * + * @hdev: pointer to device structure + * @cs: current CS + * @cs_seq: staged submission sequence number + * + * Increment CS reference for every CS in this staged submission except for + * the CS which get completion. + */ +static void staged_cs_get(struct hl_device *hdev, struct hl_cs *cs) +{ + /* Only the last CS in this staged submission will get a completion. + * We must increment the reference for all other CS's in this + * staged submission. + * Once we get a completion we will release the whole staged submission. + */ + if (!cs->staged_last) + cs_get(cs); +} + +/* + * staged_cs_put - put a CS in case it is part of staged submission + * + * @hdev: pointer to device structure + * @cs: CS to put + * + * This function decrements a CS reference (for a non completion CS) + */ +static void staged_cs_put(struct hl_device *hdev, struct hl_cs *cs) +{ + /* We release all CS's in a staged submission except the last + * CS which we have never incremented its reference. + */ + if (!cs_needs_completion(cs)) + cs_put(cs); +} + +static void cs_handle_tdr(struct hl_device *hdev, struct hl_cs *cs) +{ + bool next_entry_found = false; + struct hl_cs *next; + + if (!cs_needs_timeout(cs)) + return; + + spin_lock(&hdev->cs_mirror_lock); + + /* We need to handle tdr only once for the complete staged submission. + * Hence, we choose the CS that reaches this function first which is + * the CS marked as 'staged_last'. + */ + if (cs->staged_cs && cs->staged_last) + cs = hl_staged_cs_find_first(hdev, cs->staged_sequence); + + spin_unlock(&hdev->cs_mirror_lock); + + /* Don't cancel TDR in case this CS was timedout because we might be + * running from the TDR context + */ + if (cs && (cs->timedout || + hdev->timeout_jiffies == MAX_SCHEDULE_TIMEOUT)) + return; + + if (cs && cs->tdr_active) + cancel_delayed_work_sync(&cs->work_tdr); + + spin_lock(&hdev->cs_mirror_lock); + + /* queue TDR for next CS */ + list_for_each_entry(next, &hdev->cs_mirror_list, mirror_node) + if (cs_needs_timeout(next)) { + next_entry_found = true; + break; + } + + if (next_entry_found && !next->tdr_active) { + next->tdr_active = true; + schedule_delayed_work(&next->work_tdr, + hdev->timeout_jiffies); + } + + spin_unlock(&hdev->cs_mirror_lock); +} + static void cs_do_release(struct kref *ref) { struct hl_cs *cs = container_of(ref, struct hl_cs, refcount); @@ -346,36 +510,37 @@ static void cs_do_release(struct kref *ref) hdev->asic_funcs->hw_queues_unlock(hdev); - /* Need to update CI for internal queues */ - hl_int_hw_queue_update_ci(cs); + /* Need to update CI for all queue jobs that does not get completion */ + hl_hw_queue_update_ci(cs); /* remove CS from CS mirror list */ spin_lock(&hdev->cs_mirror_lock); list_del_init(&cs->mirror_node); spin_unlock(&hdev->cs_mirror_lock); - /* Don't cancel TDR in case this CS was timedout because we might be - * running from the TDR context - */ - if (!cs->timedout && hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) { - struct hl_cs *next; - - if (cs->tdr_active) - cancel_delayed_work_sync(&cs->work_tdr); + cs_handle_tdr(hdev, cs); - spin_lock(&hdev->cs_mirror_lock); - - /* queue TDR for next CS */ - next = list_first_entry_or_null(&hdev->cs_mirror_list, - struct hl_cs, mirror_node); + if (cs->staged_cs) { + /* the completion CS decrements reference for the entire + * staged submission + */ + if (cs->staged_last) { + struct hl_cs *staged_cs, *tmp; - if (next && !next->tdr_active) { - next->tdr_active = true; - schedule_delayed_work(&next->work_tdr, - hdev->timeout_jiffies); + list_for_each_entry_safe(staged_cs, tmp, + &cs->staged_cs_node, staged_cs_node) + staged_cs_put(hdev, staged_cs); } - spin_unlock(&hdev->cs_mirror_lock); + /* A staged CS will be a member in the list only after it + * was submitted. We used 'cs_mirror_lock' when inserting + * it to list so we will use it again when removing it + */ + if (cs->submitted) { + spin_lock(&hdev->cs_mirror_lock); + list_del(&cs->staged_cs_node); + spin_unlock(&hdev->cs_mirror_lock); + } } out: @@ -461,7 +626,8 @@ static void cs_timedout(struct work_struct *work) } static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, - enum hl_cs_type cs_type, struct hl_cs **cs_new) + enum hl_cs_type cs_type, u64 user_sequence, + struct hl_cs **cs_new) { struct hl_cs_counters_atomic *cntr; struct hl_fence *other = NULL; @@ -478,6 +644,9 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, return -ENOMEM; } + /* increment refcnt for context */ + hl_ctx_get(hdev, ctx); + cs->ctx = ctx; cs->submitted = false; cs->completed = false; @@ -507,6 +676,18 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, (hdev->asic_prop.max_pending_cs - 1)]; if (other && !completion_done(&other->completion)) { + /* If the following statement is true, it means we have reached + * a point in which only part of the staged submission was + * submitted and we don't have enough room in the 'cs_pending' + * array for the rest of the submission. + * This causes a deadlock because this CS will never be + * completed as it depends on future CS's for completion. + */ + if (other->cs_sequence == user_sequence) + dev_crit_ratelimited(hdev->dev, + "Staged CS %llu deadlock due to lack of resources", + user_sequence); + dev_dbg_ratelimited(hdev->dev, "Rejecting CS because of too many in-flights CS\n"); atomic64_inc(&ctx->cs_counters.max_cs_in_flight_drop_cnt); @@ -525,7 +706,7 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, } /* init hl_fence */ - hl_fence_init(&cs_cmpl->base_fence); + hl_fence_init(&cs_cmpl->base_fence, cs_cmpl->cs_seq); cs->sequence = cs_cmpl->cs_seq; @@ -549,6 +730,7 @@ free_fence: kfree(cs_cmpl); free_cs: kfree(cs); + hl_ctx_put(ctx); return rc; } @@ -556,6 +738,8 @@ static void cs_rollback(struct hl_device *hdev, struct hl_cs *cs) { struct hl_cs_job *job, *tmp; + staged_cs_put(hdev, cs); + list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node) complete_job(hdev, job); } @@ -565,7 +749,9 @@ void hl_cs_rollback_all(struct hl_device *hdev) int i; struct hl_cs *cs, *tmp; - /* flush all completions */ + /* flush all completions before iterating over the CS mirror list in + * order to avoid a race with the release functions + */ for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) flush_workqueue(hdev->cq_wq[i]); @@ -574,12 +760,24 @@ void hl_cs_rollback_all(struct hl_device *hdev) cs_get(cs); cs->aborted = true; dev_warn_ratelimited(hdev->dev, "Killing CS %d.%llu\n", - cs->ctx->asid, cs->sequence); + cs->ctx->asid, cs->sequence); cs_rollback(hdev, cs); cs_put(cs); } } +void hl_pending_cb_list_flush(struct hl_ctx *ctx) +{ + struct hl_pending_cb *pending_cb, *tmp; + + list_for_each_entry_safe(pending_cb, tmp, + &ctx->pending_cb_list, cb_node) { + list_del(&pending_cb->cb_node); + hl_cb_put(pending_cb->cb); + kfree(pending_cb); + } +} + static void job_wq_completion(struct work_struct *work) { struct hl_cs_job *job = container_of(work, struct hl_cs_job, @@ -734,6 +932,12 @@ static int hl_cs_sanity_checks(struct hl_fpriv *hpriv, union hl_cs_args *args) return -EBUSY; } + if ((args->in.cs_flags & HL_CS_FLAGS_STAGED_SUBMISSION) && + !hdev->supports_staged_submission) { + dev_err(hdev->dev, "staged submission not supported"); + return -EPERM; + } + cs_type_flags = args->in.cs_flags & HL_CS_FLAGS_TYPE_MASK; if (unlikely(cs_type_flags && !is_power_of_2(cs_type_flags))) { @@ -805,10 +1009,38 @@ static int hl_cs_copy_chunk_array(struct hl_device *hdev, return 0; } +static int cs_staged_submission(struct hl_device *hdev, struct hl_cs *cs, + u64 sequence, u32 flags) +{ + if (!(flags & HL_CS_FLAGS_STAGED_SUBMISSION)) + return 0; + + cs->staged_last = !!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_LAST); + cs->staged_first = !!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST); + + if (cs->staged_first) { + /* Staged CS sequence is the first CS sequence */ + INIT_LIST_HEAD(&cs->staged_cs_node); + cs->staged_sequence = cs->sequence; + } else { + /* User sequence will be validated in 'hl_hw_queue_schedule_cs' + * under the cs_mirror_lock + */ + cs->staged_sequence = sequence; + } + + /* Increment CS reference if needed */ + staged_cs_get(hdev, cs); + + cs->staged_cs = true; + + return 0; +} + static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, - u32 num_chunks, u64 *cs_seq, bool timestamp) + u32 num_chunks, u64 *cs_seq, u32 flags) { - bool int_queues_only = true; + bool staged_mid, int_queues_only = true; struct hl_device *hdev = hpriv->hdev; struct hl_cs_chunk *cs_chunk_array; struct hl_cs_counters_atomic *cntr; @@ -816,9 +1048,11 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, struct hl_cs_job *job; struct hl_cs *cs; struct hl_cb *cb; + u64 user_sequence; int rc, i; cntr = &hdev->aggregated_cs_counters; + user_sequence = *cs_seq; *cs_seq = ULLONG_MAX; rc = hl_cs_copy_chunk_array(hdev, &cs_chunk_array, chunks, num_chunks, @@ -826,20 +1060,26 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, if (rc) goto out; - /* increment refcnt for context */ - hl_ctx_get(hdev, hpriv->ctx); + if ((flags & HL_CS_FLAGS_STAGED_SUBMISSION) && + !(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST)) + staged_mid = true; + else + staged_mid = false; - rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT, &cs); - if (rc) { - hl_ctx_put(hpriv->ctx); + rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT, + staged_mid ? user_sequence : ULLONG_MAX, &cs); + if (rc) goto free_cs_chunk_array; - } - cs->timestamp = !!timestamp; + cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP); *cs_seq = cs->sequence; hl_debugfs_add_cs(cs); + rc = cs_staged_submission(hdev, cs, user_sequence, flags); + if (rc) + goto free_cs_object; + /* Validate ALL the CS chunks before submitting the CS */ for (i = 0 ; i < num_chunks ; i++) { struct hl_cs_chunk *chunk = &cs_chunk_array[i]; @@ -899,8 +1139,9 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, * Only increment for JOB on external or H/W queues, because * only for those JOBs we get completion */ - if (job->queue_type == QUEUE_TYPE_EXT || - job->queue_type == QUEUE_TYPE_HW) + if (cs_needs_completion(cs) && + (job->queue_type == QUEUE_TYPE_EXT || + job->queue_type == QUEUE_TYPE_HW)) cs_get(cs); hl_debugfs_add_job(hdev, job); @@ -916,11 +1157,14 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, } } - if (int_queues_only) { + /* We allow a CS with any queue type combination as long as it does + * not get a completion + */ + if (int_queues_only && cs_needs_completion(cs)) { atomic64_inc(&ctx->cs_counters.validation_drop_cnt); atomic64_inc(&cntr->validation_drop_cnt); dev_err(hdev->dev, - "Reject CS %d.%llu because only internal queues jobs are present\n", + "Reject CS %d.%llu since it contains only internal queues jobs and needs completion\n", cs->ctx->asid, cs->sequence); rc = -EINVAL; goto free_cs_object; @@ -954,6 +1198,129 @@ out: return rc; } +static int pending_cb_create_job(struct hl_device *hdev, struct hl_ctx *ctx, + struct hl_cs *cs, struct hl_cb *cb, u32 size, u32 hw_queue_id) +{ + struct hw_queue_properties *hw_queue_prop; + struct hl_cs_counters_atomic *cntr; + struct hl_cs_job *job; + + hw_queue_prop = &hdev->asic_prop.hw_queues_props[hw_queue_id]; + cntr = &hdev->aggregated_cs_counters; + + job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true); + if (!job) { + atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt); + atomic64_inc(&cntr->out_of_mem_drop_cnt); + dev_err(hdev->dev, "Failed to allocate a new job\n"); + return -ENOMEM; + } + + job->id = 0; + job->cs = cs; + job->user_cb = cb; + atomic_inc(&job->user_cb->cs_cnt); + job->user_cb_size = size; + job->hw_queue_id = hw_queue_id; + job->patched_cb = job->user_cb; + job->job_cb_size = job->user_cb_size; + + /* increment refcount as for external queues we get completion */ + cs_get(cs); + + cs->jobs_in_queue_cnt[job->hw_queue_id]++; + + list_add_tail(&job->cs_node, &cs->job_list); + + hl_debugfs_add_job(hdev, job); + + return 0; +} + +static int hl_submit_pending_cb(struct hl_fpriv *hpriv) +{ + struct hl_device *hdev = hpriv->hdev; + struct hl_ctx *ctx = hpriv->ctx; + struct hl_pending_cb *pending_cb, *tmp; + struct list_head local_cb_list; + struct hl_cs *cs; + struct hl_cb *cb; + u32 hw_queue_id; + u32 cb_size; + int process_list, rc = 0; + + if (list_empty(&ctx->pending_cb_list)) + return 0; + + process_list = atomic_cmpxchg(&ctx->thread_pending_cb_token, 1, 0); + + /* Only a single thread is allowed to process the list */ + if (!process_list) + return 0; + + if (list_empty(&ctx->pending_cb_list)) + goto free_pending_cb_token; + + /* move all list elements to a local list */ + INIT_LIST_HEAD(&local_cb_list); + spin_lock(&ctx->pending_cb_lock); + list_for_each_entry_safe(pending_cb, tmp, &ctx->pending_cb_list, + cb_node) + list_move_tail(&pending_cb->cb_node, &local_cb_list); + spin_unlock(&ctx->pending_cb_lock); + + rc = allocate_cs(hdev, ctx, CS_TYPE_DEFAULT, ULLONG_MAX, &cs); + if (rc) + goto add_list_elements; + + hl_debugfs_add_cs(cs); + + /* Iterate through pending cb list, create jobs and add to CS */ + list_for_each_entry(pending_cb, &local_cb_list, cb_node) { + cb = pending_cb->cb; + cb_size = pending_cb->cb_size; + hw_queue_id = pending_cb->hw_queue_id; + + rc = pending_cb_create_job(hdev, ctx, cs, cb, cb_size, + hw_queue_id); + if (rc) + goto free_cs_object; + } + + rc = hl_hw_queue_schedule_cs(cs); + if (rc) { + if (rc != -EAGAIN) + dev_err(hdev->dev, + "Failed to submit CS %d.%llu (%d)\n", + ctx->asid, cs->sequence, rc); + goto free_cs_object; + } + + /* pending cb was scheduled successfully */ + list_for_each_entry_safe(pending_cb, tmp, &local_cb_list, cb_node) { + list_del(&pending_cb->cb_node); + kfree(pending_cb); + } + + cs_put(cs); + + goto free_pending_cb_token; + +free_cs_object: + cs_rollback(hdev, cs); + cs_put(cs); +add_list_elements: + spin_lock(&ctx->pending_cb_lock); + list_for_each_entry_safe_reverse(pending_cb, tmp, &local_cb_list, + cb_node) + list_move(&pending_cb->cb_node, &ctx->pending_cb_list); + spin_unlock(&ctx->pending_cb_lock); +free_pending_cb_token: + atomic_set(&ctx->thread_pending_cb_token, 1); + + return rc; +} + static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args, u64 *cs_seq) { @@ -1003,7 +1370,7 @@ static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args, rc = 0; } else { rc = cs_ioctl_default(hpriv, chunks, num_chunks, - cs_seq, false); + cs_seq, 0); } mutex_unlock(&hpriv->restore_phase_mutex); @@ -1275,15 +1642,11 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, } } - /* increment refcnt for context */ - hl_ctx_get(hdev, ctx); - - rc = allocate_cs(hdev, ctx, cs_type, &cs); + rc = allocate_cs(hdev, ctx, cs_type, ULLONG_MAX, &cs); if (rc) { if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_COLLECTIVE_WAIT) hl_fence_put(sig_fence); - hl_ctx_put(ctx); goto free_cs_chunk_array; } @@ -1346,7 +1709,7 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) enum hl_cs_type cs_type; u64 cs_seq = ULONG_MAX; void __user *chunks; - u32 num_chunks; + u32 num_chunks, flags; int rc; rc = hl_cs_sanity_checks(hpriv, args); @@ -1357,10 +1720,20 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) if (rc) goto out; + rc = hl_submit_pending_cb(hpriv); + if (rc) + goto out; + cs_type = hl_cs_get_cs_type(args->in.cs_flags & ~HL_CS_FLAGS_FORCE_RESTORE); chunks = (void __user *) (uintptr_t) args->in.chunks_execute; num_chunks = args->in.num_chunks_execute; + flags = args->in.cs_flags; + + /* In case this is a staged CS, user should supply the CS sequence */ + if ((flags & HL_CS_FLAGS_STAGED_SUBMISSION) && + !(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST)) + cs_seq = args->in.seq; switch (cs_type) { case CS_TYPE_SIGNAL: @@ -1371,7 +1744,7 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) break; default: rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq, - args->in.cs_flags & HL_CS_FLAGS_TIMESTAMP); + args->in.cs_flags); break; } diff --git a/drivers/misc/habanalabs/common/context.c b/drivers/misc/habanalabs/common/context.c index f65e6559149b..cda871afb8f4 100644 --- a/drivers/misc/habanalabs/common/context.c +++ b/drivers/misc/habanalabs/common/context.c @@ -12,9 +12,14 @@ static void hl_ctx_fini(struct hl_ctx *ctx) { struct hl_device *hdev = ctx->hdev; - u64 idle_mask = 0; + u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; int i; + /* Release all allocated pending cb's, those cb's were never + * scheduled so it is safe to release them here + */ + hl_pending_cb_list_flush(ctx); + /* * If we arrived here, there are no jobs waiting for this context * on its queues so we can safely remove it. @@ -50,12 +55,15 @@ static void hl_ctx_fini(struct hl_ctx *ctx) if ((!hdev->pldm) && (hdev->pdev) && (!hdev->asic_funcs->is_device_idle(hdev, - &idle_mask, NULL))) + idle_mask, + HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL))) dev_notice(hdev->dev, - "device not idle after user context is closed (0x%llx)\n", - idle_mask); + "device not idle after user context is closed (0x%llx, 0x%llx)\n", + idle_mask[0], idle_mask[1]); } else { dev_dbg(hdev->dev, "closing kernel context\n"); + hdev->asic_funcs->ctx_fini(ctx); + hl_vm_ctx_fini(ctx); hl_mmu_ctx_fini(ctx); } } @@ -140,8 +148,11 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx) kref_init(&ctx->refcount); ctx->cs_sequence = 1; + INIT_LIST_HEAD(&ctx->pending_cb_list); + spin_lock_init(&ctx->pending_cb_lock); spin_lock_init(&ctx->cs_lock); atomic_set(&ctx->thread_ctx_switch_token, 1); + atomic_set(&ctx->thread_pending_cb_token, 1); ctx->thread_ctx_switch_wait_token = 0; ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs, sizeof(struct hl_fence *), @@ -151,11 +162,18 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx) if (is_kernel_ctx) { ctx->asid = HL_KERNEL_ASID_ID; /* Kernel driver gets ASID 0 */ - rc = hl_mmu_ctx_init(ctx); + rc = hl_vm_ctx_init(ctx); if (rc) { - dev_err(hdev->dev, "Failed to init mmu ctx module\n"); + dev_err(hdev->dev, "Failed to init mem ctx module\n"); + rc = -ENOMEM; goto err_free_cs_pending; } + + rc = hdev->asic_funcs->ctx_init(ctx); + if (rc) { + dev_err(hdev->dev, "ctx_init failed\n"); + goto err_vm_ctx_fini; + } } else { ctx->asid = hl_asid_alloc(hdev); if (!ctx->asid) { @@ -194,7 +212,8 @@ err_cb_va_pool_fini: err_vm_ctx_fini: hl_vm_ctx_fini(ctx); err_asid_free: - hl_asid_free(hdev, ctx->asid); + if (ctx->asid != HL_KERNEL_ASID_ID) + hl_asid_free(hdev, ctx->asid); err_free_cs_pending: kfree(ctx->cs_pending); diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c index cef716643979..df847a6d19f4 100644 --- a/drivers/misc/habanalabs/common/debugfs.c +++ b/drivers/misc/habanalabs/common/debugfs.c @@ -310,8 +310,8 @@ static int mmu_show(struct seq_file *s, void *data) struct hl_dbg_device_entry *dev_entry = entry->dev_entry; struct hl_device *hdev = dev_entry->hdev; struct hl_ctx *ctx; - struct hl_mmu_hop_info hops_info; - u64 virt_addr = dev_entry->mmu_addr; + struct hl_mmu_hop_info hops_info = {0}; + u64 virt_addr = dev_entry->mmu_addr, phys_addr; int i; if (!hdev->mmu_enable) @@ -333,8 +333,19 @@ static int mmu_show(struct seq_file *s, void *data) return 0; } - seq_printf(s, "asid: %u, virt_addr: 0x%llx\n", - dev_entry->mmu_asid, dev_entry->mmu_addr); + phys_addr = hops_info.hop_info[hops_info.used_hops - 1].hop_pte_val; + + if (hops_info.scrambled_vaddr && + (dev_entry->mmu_addr != hops_info.scrambled_vaddr)) + seq_printf(s, + "asid: %u, virt_addr: 0x%llx, scrambled virt_addr: 0x%llx,\nphys_addr: 0x%llx, scrambled_phys_addr: 0x%llx\n", + dev_entry->mmu_asid, dev_entry->mmu_addr, + hops_info.scrambled_vaddr, + hops_info.unscrambled_paddr, phys_addr); + else + seq_printf(s, + "asid: %u, virt_addr: 0x%llx, phys_addr: 0x%llx\n", + dev_entry->mmu_asid, dev_entry->mmu_addr, phys_addr); for (i = 0 ; i < hops_info.used_hops ; i++) { seq_printf(s, "hop%d_addr: 0x%llx\n", @@ -403,7 +414,7 @@ static int engines_show(struct seq_file *s, void *data) return 0; } - hdev->asic_funcs->is_device_idle(hdev, NULL, s); + hdev->asic_funcs->is_device_idle(hdev, NULL, 0, s); return 0; } @@ -865,6 +876,17 @@ static ssize_t hl_stop_on_err_write(struct file *f, const char __user *buf, return count; } +static ssize_t hl_security_violations_read(struct file *f, char __user *buf, + size_t count, loff_t *ppos) +{ + struct hl_dbg_device_entry *entry = file_inode(f)->i_private; + struct hl_device *hdev = entry->hdev; + + hdev->asic_funcs->ack_protection_bits_errors(hdev); + + return 0; +} + static const struct file_operations hl_data32b_fops = { .owner = THIS_MODULE, .read = hl_data_read32, @@ -922,6 +944,11 @@ static const struct file_operations hl_stop_on_err_fops = { .write = hl_stop_on_err_write }; +static const struct file_operations hl_security_violations_fops = { + .owner = THIS_MODULE, + .read = hl_security_violations_read +}; + static const struct hl_info_list hl_debugfs_list[] = { {"command_buffers", command_buffers_show, NULL}, {"command_submission", command_submission_show, NULL}, @@ -1071,6 +1098,12 @@ void hl_debugfs_add_device(struct hl_device *hdev) dev_entry, &hl_stop_on_err_fops); + debugfs_create_file("dump_security_violations", + 0644, + dev_entry->root, + dev_entry, + &hl_security_violations_fops); + for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) { ent = debugfs_create_file(hl_debugfs_list[i].name, diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index 69d04eca767f..15fcb5c31c4b 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -142,6 +142,9 @@ static int hl_mmap(struct file *filp, struct vm_area_struct *vma) switch (vm_pgoff & HL_MMAP_TYPE_MASK) { case HL_MMAP_TYPE_CB: return hl_cb_mmap(hpriv, vma); + + case HL_MMAP_TYPE_BLOCK: + return hl_hw_block_mmap(hpriv, vma); } return -EINVAL; @@ -373,7 +376,6 @@ static int device_early_init(struct hl_device *hdev) mutex_init(&hdev->send_cpu_message_lock); mutex_init(&hdev->debug_lock); - mutex_init(&hdev->mmu_cache_lock); INIT_LIST_HEAD(&hdev->cs_mirror_list); spin_lock_init(&hdev->cs_mirror_lock); INIT_LIST_HEAD(&hdev->fpriv_list); @@ -414,7 +416,6 @@ static void device_early_fini(struct hl_device *hdev) { int i; - mutex_destroy(&hdev->mmu_cache_lock); mutex_destroy(&hdev->debug_lock); mutex_destroy(&hdev->send_cpu_message_lock); @@ -1158,12 +1159,20 @@ kill_processes: atomic_set(&hdev->in_reset, 0); hdev->needs_reset = false; - if (hard_reset) + dev_notice(hdev->dev, "Successfully finished resetting the device\n"); + + if (hard_reset) { hdev->hard_reset_cnt++; - else - hdev->soft_reset_cnt++; - dev_warn(hdev->dev, "Successfully finished resetting the device\n"); + /* After reset is done, we are ready to receive events from + * the F/W. We can't do it before because we will ignore events + * and if those events are fatal, we won't know about it and + * the device will be operational although it shouldn't be + */ + hdev->asic_funcs->enable_events_from_fw(hdev); + } else { + hdev->soft_reset_cnt++; + } return 0; @@ -1314,11 +1323,16 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) hdev->compute_ctx = NULL; + hl_debugfs_add_device(hdev); + + /* debugfs nodes are created in hl_ctx_init so it must be called after + * hl_debugfs_add_device. + */ rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); if (rc) { dev_err(hdev->dev, "failed to initialize kernel context\n"); kfree(hdev->kernel_ctx); - goto mmu_fini; + goto remove_device_from_debugfs; } rc = hl_cb_pool_init(hdev); @@ -1327,8 +1341,6 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) goto release_ctx; } - hl_debugfs_add_device(hdev); - /* * From this point, in case of an error, add char devices and create * sysfs nodes as part of the error flow, to allow debugging. @@ -1411,12 +1423,21 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) hdev->init_done = true; + /* After initialization is done, we are ready to receive events from + * the F/W. We can't do it before because we will ignore events and if + * those events are fatal, we won't know about it and the device will + * be operational although it shouldn't be + */ + hdev->asic_funcs->enable_events_from_fw(hdev); + return 0; release_ctx: if (hl_ctx_put(hdev->kernel_ctx) != 1) dev_err(hdev->dev, "kernel ctx is still alive on initialization failure\n"); +remove_device_from_debugfs: + hl_debugfs_remove_device(hdev); mmu_fini: hl_mmu_fini(hdev); eq_fini: @@ -1482,7 +1503,8 @@ void hl_device_fini(struct hl_device *hdev) usleep_range(50, 200); rc = atomic_cmpxchg(&hdev->in_reset, 0, 1); if (ktime_compare(ktime_get(), timeout) > 0) { - WARN(1, "Failed to remove device because reset function did not finish\n"); + dev_crit(hdev->dev, + "Failed to remove device because reset function did not finish\n"); return; } } @@ -1515,8 +1537,6 @@ void hl_device_fini(struct hl_device *hdev) device_late_fini(hdev); - hl_debugfs_remove_device(hdev); - /* * Halt the engines and disable interrupts so we won't get any more * completions from H/W and we won't have any accesses from the @@ -1548,6 +1568,8 @@ void hl_device_fini(struct hl_device *hdev) if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1)) dev_err(hdev->dev, "kernel ctx is still alive\n"); + hl_debugfs_remove_device(hdev); + hl_vm_fini(hdev); hl_mmu_fini(hdev); diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c index c9a12980218a..09706c571e95 100644 --- a/drivers/misc/habanalabs/common/firmware_if.c +++ b/drivers/misc/habanalabs/common/firmware_if.c @@ -90,9 +90,10 @@ int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode) int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, u16 len, u32 timeout, u64 *result) { + struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id]; struct cpucp_packet *pkt; dma_addr_t pkt_dma_addr; - u32 tmp; + u32 tmp, expected_ack_val; int rc = 0; pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len, @@ -115,14 +116,23 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, goto out; } + /* set fence to a non valid value */ + pkt->fence = UINT_MAX; + rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, len, pkt_dma_addr); if (rc) { dev_err(hdev->dev, "Failed to send CB on CPU PQ (%d)\n", rc); goto out; } + if (hdev->asic_prop.fw_app_security_map & + CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN) + expected_ack_val = queue->pi; + else + expected_ack_val = CPUCP_PACKET_FENCE_VAL; + rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp, - (tmp == CPUCP_PACKET_FENCE_VAL), 1000, + (tmp == expected_ack_val), 1000, timeout, true); hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id); @@ -279,8 +289,74 @@ int hl_fw_send_heartbeat(struct hl_device *hdev) return rc; } +static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, + u32 cpu_security_boot_status_reg) +{ + u32 err_val, security_val; + + /* Some of the firmware status codes are deprecated in newer f/w + * versions. In those versions, the errors are reported + * in different registers. Therefore, we need to check those + * registers and print the exact errors. Moreover, there + * may be multiple errors, so we need to report on each error + * separately. Some of the error codes might indicate a state + * that is not an error per-se, but it is an error in production + * environment + */ + err_val = RREG32(boot_err0_reg); + if (!(err_val & CPU_BOOT_ERR0_ENABLED)) + return 0; + + if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) + dev_err(hdev->dev, + "Device boot error - DRAM initialization failed\n"); + if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) + dev_err(hdev->dev, "Device boot error - FIT image corrupted\n"); + if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) + dev_err(hdev->dev, + "Device boot error - Thermal Sensor initialization failed\n"); + if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) + dev_warn(hdev->dev, + "Device boot warning - Skipped DRAM initialization\n"); + + if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) { + if (hdev->bmc_enable) + dev_warn(hdev->dev, + "Device boot error - Skipped waiting for BMC\n"); + else + err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED; + } + + if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) + dev_err(hdev->dev, + "Device boot error - Serdes data from BMC not available\n"); + if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) + dev_err(hdev->dev, + "Device boot error - NIC F/W initialization failed\n"); + if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) + dev_warn(hdev->dev, + "Device boot warning - security not ready\n"); + if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) + dev_err(hdev->dev, "Device boot error - security failure\n"); + if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) + dev_err(hdev->dev, "Device boot error - eFuse failure\n"); + if (err_val & CPU_BOOT_ERR0_PLL_FAIL) + dev_err(hdev->dev, "Device boot error - PLL failure\n"); + + security_val = RREG32(cpu_security_boot_status_reg); + if (security_val & CPU_BOOT_DEV_STS0_ENABLED) + dev_dbg(hdev->dev, "Device security status %#x\n", + security_val); + + if (err_val & ~CPU_BOOT_ERR0_ENABLED) + return -EIO; + + return 0; +} + int hl_fw_cpucp_info_get(struct hl_device *hdev, - u32 cpu_security_boot_status_reg) + u32 cpu_security_boot_status_reg, + u32 boot_err0_reg) { struct asic_fixed_properties *prop = &hdev->asic_prop; struct cpucp_packet pkt = {}; @@ -314,6 +390,12 @@ int hl_fw_cpucp_info_get(struct hl_device *hdev, goto out; } + rc = fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg); + if (rc) { + dev_err(hdev->dev, "Errors in device boot\n"); + goto out; + } + memcpy(&prop->cpucp_info, cpucp_info_cpu_addr, sizeof(prop->cpucp_info)); @@ -483,58 +565,6 @@ int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u16 pll_index, return rc; } -static void fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, - u32 cpu_security_boot_status_reg) -{ - u32 err_val, security_val; - - /* Some of the firmware status codes are deprecated in newer f/w - * versions. In those versions, the errors are reported - * in different registers. Therefore, we need to check those - * registers and print the exact errors. Moreover, there - * may be multiple errors, so we need to report on each error - * separately. Some of the error codes might indicate a state - * that is not an error per-se, but it is an error in production - * environment - */ - err_val = RREG32(boot_err0_reg); - if (!(err_val & CPU_BOOT_ERR0_ENABLED)) - return; - - if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) - dev_err(hdev->dev, - "Device boot error - DRAM initialization failed\n"); - if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) - dev_err(hdev->dev, "Device boot error - FIT image corrupted\n"); - if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) - dev_err(hdev->dev, - "Device boot error - Thermal Sensor initialization failed\n"); - if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) - dev_warn(hdev->dev, - "Device boot warning - Skipped DRAM initialization\n"); - if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) - dev_warn(hdev->dev, - "Device boot error - Skipped waiting for BMC\n"); - if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) - dev_err(hdev->dev, - "Device boot error - Serdes data from BMC not available\n"); - if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) - dev_err(hdev->dev, - "Device boot error - NIC F/W initialization failed\n"); - if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) - dev_warn(hdev->dev, - "Device boot warning - security not ready\n"); - if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) - dev_err(hdev->dev, "Device boot error - security failure\n"); - if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) - dev_err(hdev->dev, "Device boot error - eFuse failure\n"); - - security_val = RREG32(cpu_security_boot_status_reg); - if (security_val & CPU_BOOT_DEV_STS0_ENABLED) - dev_dbg(hdev->dev, "Device security status %#x\n", - security_val); -} - static void detect_cpu_boot_status(struct hl_device *hdev, u32 status) { /* Some of the status codes below are deprecated in newer f/w @@ -659,6 +689,9 @@ int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, prop->fw_security_disabled = true; } + dev_dbg(hdev->dev, "Firmware preboot security status %#x\n", + security_status); + dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n", prop->hard_reset_done_by_fw ? "enabled" : "disabled"); @@ -753,6 +786,10 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, if (prop->fw_boot_cpu_security_map & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) prop->hard_reset_done_by_fw = true; + + dev_dbg(hdev->dev, + "Firmware boot CPU security status %#x\n", + prop->fw_boot_cpu_security_map); } dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n", @@ -826,6 +863,10 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, goto out; } + rc = fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg); + if (rc) + return rc; + /* Clear reset status since we need to read again from app */ prop->hard_reset_done_by_fw = false; @@ -837,6 +878,10 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, if (prop->fw_app_security_map & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) prop->hard_reset_done_by_fw = true; + + dev_dbg(hdev->dev, + "Firmware application CPU security status %#x\n", + prop->fw_app_security_map); } dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n", @@ -844,6 +889,8 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, dev_info(hdev->dev, "Successfully loaded firmware to device\n"); + return 0; + out: fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg); diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 41af347157e0..d933878b24d1 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -28,17 +28,18 @@ #define HL_NAME "habanalabs" /* Use upper bits of mmap offset to store habana driver specific information. - * bits[63:62] - Encode mmap type + * bits[63:61] - Encode mmap type * bits[45:0] - mmap offset value * * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these * defines are w.r.t to PAGE_SIZE */ -#define HL_MMAP_TYPE_SHIFT (62 - PAGE_SHIFT) -#define HL_MMAP_TYPE_MASK (0x3ull << HL_MMAP_TYPE_SHIFT) +#define HL_MMAP_TYPE_SHIFT (61 - PAGE_SHIFT) +#define HL_MMAP_TYPE_MASK (0x7ull << HL_MMAP_TYPE_SHIFT) +#define HL_MMAP_TYPE_BLOCK (0x4ull << HL_MMAP_TYPE_SHIFT) #define HL_MMAP_TYPE_CB (0x2ull << HL_MMAP_TYPE_SHIFT) -#define HL_MMAP_OFFSET_VALUE_MASK (0x3FFFFFFFFFFFull >> PAGE_SHIFT) +#define HL_MMAP_OFFSET_VALUE_MASK (0x1FFFFFFFFFFFull >> PAGE_SHIFT) #define HL_MMAP_OFFSET_VALUE_GET(off) (off & HL_MMAP_OFFSET_VALUE_MASK) #define HL_PENDING_RESET_PER_SEC 10 @@ -408,6 +409,9 @@ struct hl_mmu_properties { * @sync_stream_first_mon: first monitor available for sync stream use * @first_available_user_sob: first sob available for the user * @first_available_user_mon: first monitor available for the user + * @first_available_user_msix_interrupt: first available msix interrupt + * reserved for the user + * @first_available_cq: first available CQ for the user. * @tpc_enabled_mask: which TPCs are enabled. * @completion_queues_count: number of completion queues. * @fw_security_disabled: true if security measures are disabled in firmware, @@ -416,6 +420,7 @@ struct hl_mmu_properties { * from BOOT_DEV_STS0 * @dram_supports_virtual_memory: is there an MMU towards the DRAM * @hard_reset_done_by_fw: true if firmware is handling hard reset flow + * @num_functional_hbms: number of functional HBMs in each DCORE. */ struct asic_fixed_properties { struct hw_queue_properties *hw_queues_props; @@ -468,18 +473,22 @@ struct asic_fixed_properties { u16 sync_stream_first_mon; u16 first_available_user_sob[HL_MAX_DCORES]; u16 first_available_user_mon[HL_MAX_DCORES]; + u16 first_available_user_msix_interrupt; + u16 first_available_cq[HL_MAX_DCORES]; u8 tpc_enabled_mask; u8 completion_queues_count; u8 fw_security_disabled; u8 fw_security_status_valid; u8 dram_supports_virtual_memory; u8 hard_reset_done_by_fw; + u8 num_functional_hbms; }; /** * struct hl_fence - software synchronization primitive * @completion: fence is implemented using completion * @refcount: refcount for this fence + * @cs_sequence: sequence of the corresponding command submission * @error: mark this fence with error * @timestamp: timestamp upon completion * @@ -487,6 +496,7 @@ struct asic_fixed_properties { struct hl_fence { struct completion completion; struct kref refcount; + u64 cs_sequence; int error; ktime_t timestamp; }; @@ -846,6 +856,19 @@ enum div_select_defs { * @collective_wait_init_cs: Generate collective master/slave packets * and place them in the relevant cs jobs * @collective_wait_create_jobs: allocate collective wait cs jobs + * @scramble_addr: Routine to scramble the address prior of mapping it + * in the MMU. + * @descramble_addr: Routine to de-scramble the address prior of + * showing it to users. + * @ack_protection_bits_errors: ack and dump all security violations + * @get_hw_block_id: retrieve a HW block id to be used by the user to mmap it. + * also returns the size of the block if caller supplies + * a valid pointer for it + * @hw_block_mmap: mmap a HW block with a given id. + * @enable_events_from_fw: send interrupt to firmware to notify them the + * driver is ready to receive asynchronous events. This + * function should be called during the first init and + * after every hard-reset of the device */ struct hl_asic_funcs { int (*early_init)(struct hl_device *hdev); @@ -918,8 +941,8 @@ struct hl_asic_funcs { void (*set_clock_gating)(struct hl_device *hdev); void (*disable_clock_gating)(struct hl_device *hdev); int (*debug_coresight)(struct hl_device *hdev, void *data); - bool (*is_device_idle)(struct hl_device *hdev, u64 *mask, - struct seq_file *s); + bool (*is_device_idle)(struct hl_device *hdev, u64 *mask_arr, + u8 mask_len, struct seq_file *s); int (*soft_reset_late_init)(struct hl_device *hdev); void (*hw_queues_lock)(struct hl_device *hdev); void (*hw_queues_unlock)(struct hl_device *hdev); @@ -955,6 +978,14 @@ struct hl_asic_funcs { int (*collective_wait_create_jobs)(struct hl_device *hdev, struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, u32 collective_engine_id); + u64 (*scramble_addr)(struct hl_device *hdev, u64 addr); + u64 (*descramble_addr)(struct hl_device *hdev, u64 addr); + void (*ack_protection_bits_errors)(struct hl_device *hdev); + int (*get_hw_block_id)(struct hl_device *hdev, u64 block_addr, + u32 *block_size, u32 *block_id); + int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma, + u32 block_id, u32 block_size); + void (*enable_events_from_fw)(struct hl_device *hdev); }; @@ -1012,6 +1043,20 @@ struct hl_cs_counters_atomic { }; /** + * struct hl_pending_cb - pending command buffer structure + * @cb_node: cb node in pending cb list + * @cb: command buffer to send in next submission + * @cb_size: command buffer size + * @hw_queue_id: destination queue id + */ +struct hl_pending_cb { + struct list_head cb_node; + struct hl_cb *cb; + u32 cb_size; + u32 hw_queue_id; +}; + +/** * struct hl_ctx - user/kernel context. * @mem_hash: holds mapping from virtual address to virtual memory area * descriptor (hl_vm_phys_pg_list or hl_userptr). @@ -1026,6 +1071,8 @@ struct hl_cs_counters_atomic { * @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the * MMU hash or walking the PGT requires talking this lock. * @debugfs_list: node in debugfs list of contexts. + * pending_cb_list: list of pending command buffers waiting to be sent upon + * next user command submission context. * @cs_counters: context command submission counters. * @cb_va_pool: device VA pool for command buffers which are mapped to the * device's MMU. @@ -1034,11 +1081,17 @@ struct hl_cs_counters_atomic { * index to cs_pending array. * @dram_default_hops: array that holds all hops addresses needed for default * DRAM mapping. + * @pending_cb_lock: spinlock to protect pending cb list * @cs_lock: spinlock to protect cs_sequence. * @dram_phys_mem: amount of used physical DRAM memory by this context. * @thread_ctx_switch_token: token to prevent multiple threads of the same * context from running the context switch phase. * Only a single thread should run it. + * @thread_pending_cb_token: token to prevent multiple threads from processing + * the pending CB list. Only a single thread should + * process the list since it is protected by a + * spinlock and we don't want to halt the entire + * command submission sequence. * @thread_ctx_switch_wait_token: token to prevent the threads that didn't run * the context switch phase from moving to their * execution phase before the context switch phase @@ -1057,13 +1110,16 @@ struct hl_ctx { struct mutex mem_hash_lock; struct mutex mmu_lock; struct list_head debugfs_list; + struct list_head pending_cb_list; struct hl_cs_counters_atomic cs_counters; struct gen_pool *cb_va_pool; u64 cs_sequence; u64 *dram_default_hops; + spinlock_t pending_cb_lock; spinlock_t cs_lock; atomic64_t dram_phys_mem; atomic_t thread_ctx_switch_token; + atomic_t thread_pending_cb_token; u32 thread_ctx_switch_wait_token; u32 asid; u32 handle; @@ -1124,8 +1180,11 @@ struct hl_userptr { * @finish_work: workqueue object to run when CS is completed by H/W. * @work_tdr: delayed work node for TDR. * @mirror_node : node in device mirror list of command submissions. + * @staged_cs_node: node in the staged cs list. * @debugfs_list: node in debugfs list of command submissions. * @sequence: the sequence number of this CS. + * @staged_sequence: the sequence of the staged submission this CS is part of, + * relevant only if staged_cs is set. * @type: CS_TYPE_*. * @submitted: true if CS was submitted to H/W. * @completed: true if CS was completed by device. @@ -1133,7 +1192,11 @@ struct hl_userptr { * @tdr_active: true if TDR was activated for this CS (to prevent * double TDR activation). * @aborted: true if CS was aborted due to some device error. - * @timestamp: true if a timestmap must be captured upon completion + * @timestamp: true if a timestmap must be captured upon completion. + * @staged_last: true if this is the last staged CS and needs completion. + * @staged_first: true if this is the first staged CS and we need to receive + * timeout for this CS. + * @staged_cs: true if this CS is part of a staged submission. */ struct hl_cs { u16 *jobs_in_queue_cnt; @@ -1146,8 +1209,10 @@ struct hl_cs { struct work_struct finish_work; struct delayed_work work_tdr; struct list_head mirror_node; + struct list_head staged_cs_node; struct list_head debugfs_list; u64 sequence; + u64 staged_sequence; enum hl_cs_type type; u8 submitted; u8 completed; @@ -1155,6 +1220,9 @@ struct hl_cs { u8 tdr_active; u8 aborted; u8 timestamp; + u8 staged_last; + u8 staged_first; + u8 staged_cs; }; /** @@ -1225,6 +1293,7 @@ struct hl_cs_job { * MSG_PROT packets. Relevant only for GAUDI as GOYA doesn't * have streams so the engine can't be busy by another * stream. + * @completion: true if we need completion for this CS. */ struct hl_cs_parser { struct hl_cb *user_cb; @@ -1239,6 +1308,7 @@ struct hl_cs_parser { u8 job_id; u8 is_kernel_allocated_cb; u8 contains_dma_pkt; + u8 completion; }; /* @@ -1688,12 +1758,20 @@ struct hl_mmu_per_hop_info { * struct hl_mmu_hop_info - A structure describing the TLB hops and their * hop-entries that were created in order to translate a virtual address to a * physical one. + * @scrambled_vaddr: The value of the virtual address after scrambling. This + * address replaces the original virtual-address when mapped + * in the MMU tables. + * @unscrambled_paddr: The un-scrambled physical address. * @hop_info: Array holding the per-hop information used for the translation. * @used_hops: The number of hops used for the translation. + * @range_type: virtual address range type. */ struct hl_mmu_hop_info { + u64 scrambled_vaddr; + u64 unscrambled_paddr; struct hl_mmu_per_hop_info hop_info[MMU_ARCH_5_HOPS]; u32 used_hops; + enum hl_va_range_type range_type; }; /** @@ -1766,7 +1844,6 @@ struct hl_mmu_funcs { * @asic_funcs: ASIC specific functions. * @asic_specific: ASIC specific information to use only from ASIC files. * @vm: virtual memory manager for MMU. - * @mmu_cache_lock: protects MMU cache invalidation as it can serve one context. * @hwmon_dev: H/W monitor device. * @pm_mng_profile: current power management profile. * @hl_chip_info: ASIC's sensors information. @@ -1844,6 +1921,7 @@ struct hl_mmu_funcs { * user processes * @device_fini_pending: true if device_fini was called and might be * waiting for the reset thread to finish + * @supports_staged_submission: true if staged submissions are supported */ struct hl_device { struct pci_dev *pdev; @@ -1881,7 +1959,6 @@ struct hl_device { const struct hl_asic_funcs *asic_funcs; void *asic_specific; struct hl_vm vm; - struct mutex mmu_cache_lock; struct device *hwmon_dev; enum hl_pm_mng_profile pm_mng_profile; struct hwmon_chip_info *hl_chip_info; @@ -1950,6 +2027,7 @@ struct hl_device { u8 needs_reset; u8 process_kill_trial_cnt; u8 device_fini_pending; + u8 supports_staged_submission; /* Parameters for bring-up */ u64 nic_ports_mask; @@ -2067,7 +2145,7 @@ int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id, int hl_hw_queue_schedule_cs(struct hl_cs *cs); u32 hl_hw_queue_add_ptr(u32 ptr, u16 val); void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id); -void hl_int_hw_queue_update_ci(struct hl_cs *cs); +void hl_hw_queue_update_ci(struct hl_cs *cs); void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset); #define hl_queue_inc_ptr(p) hl_hw_queue_add_ptr(p, 1) @@ -2123,6 +2201,7 @@ int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr, bool map_cb, u64 *handle); int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle); int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma); +int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma); struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr, u32 handle); void hl_cb_put(struct hl_cb *cb); @@ -2136,6 +2215,7 @@ int hl_cb_va_pool_init(struct hl_ctx *ctx); void hl_cb_va_pool_fini(struct hl_ctx *ctx); void hl_cs_rollback_all(struct hl_device *hdev); +void hl_pending_cb_list_flush(struct hl_ctx *ctx); struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev, enum hl_queue_type queue_type, bool is_kernel_allocated_cb); void hl_sob_reset_error(struct kref *ref); @@ -2143,6 +2223,10 @@ int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask); void hl_fence_put(struct hl_fence *fence); void hl_fence_get(struct hl_fence *fence); void cs_get(struct hl_cs *cs); +bool cs_needs_completion(struct hl_cs *cs); +bool cs_needs_timeout(struct hl_cs *cs); +bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs); +struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq); void goya_set_asic_funcs(struct hl_device *hdev); void gaudi_set_asic_funcs(struct hl_device *hdev); @@ -2184,6 +2268,8 @@ void hl_mmu_v1_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu); int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr); int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops); +u64 hl_mmu_scramble_addr(struct hl_device *hdev, u64 addr); +u64 hl_mmu_descramble_addr(struct hl_device *hdev, u64 addr); bool hl_is_dram_va(struct hl_device *hdev, u64 virt_addr); int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, @@ -2201,7 +2287,8 @@ void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr); int hl_fw_send_heartbeat(struct hl_device *hdev); int hl_fw_cpucp_info_get(struct hl_device *hdev, - u32 cpu_security_boot_status_reg); + u32 cpu_security_boot_status_reg, + u32 boot_err0_reg); int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size); int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev, struct hl_info_pci_counters *counters); diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index d25892d61ec9..03af61cecd37 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -57,12 +57,23 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args) hw_ip.device_id = hdev->asic_funcs->get_pci_id(hdev); hw_ip.sram_base_address = prop->sram_user_base_address; - hw_ip.dram_base_address = prop->dram_user_base_address; + hw_ip.dram_base_address = + hdev->mmu_enable && prop->dram_supports_virtual_memory ? + prop->dmmu.start_addr : prop->dram_user_base_address; hw_ip.tpc_enabled_mask = prop->tpc_enabled_mask; hw_ip.sram_size = prop->sram_size - sram_kmd_size; - hw_ip.dram_size = prop->dram_size - dram_kmd_size; + + if (hdev->mmu_enable) + hw_ip.dram_size = + DIV_ROUND_DOWN_ULL(prop->dram_size - dram_kmd_size, + prop->dram_page_size) * + prop->dram_page_size; + else + hw_ip.dram_size = prop->dram_size - dram_kmd_size; + if (hw_ip.dram_size > PAGE_SIZE) hw_ip.dram_enabled = 1; + hw_ip.dram_page_size = prop->dram_page_size; hw_ip.num_of_events = prop->num_of_events; memcpy(hw_ip.cpucp_version, prop->cpucp_info.cpucp_version, @@ -79,6 +90,8 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args) hw_ip.psoc_pci_pll_od = prop->psoc_pci_pll_od; hw_ip.psoc_pci_pll_div_factor = prop->psoc_pci_pll_div_factor; + hw_ip.first_available_interrupt_id = + prop->first_available_user_msix_interrupt; return copy_to_user(out, &hw_ip, min((size_t)size, sizeof(hw_ip))) ? -EFAULT : 0; } @@ -132,9 +145,10 @@ static int hw_idle(struct hl_device *hdev, struct hl_info_args *args) return -EINVAL; hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev, - &hw_idle.busy_engines_mask_ext, NULL); + hw_idle.busy_engines_mask_ext, + HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL); hw_idle.busy_engines_mask = - lower_32_bits(hw_idle.busy_engines_mask_ext); + lower_32_bits(hw_idle.busy_engines_mask_ext[0]); return copy_to_user(out, &hw_idle, min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0; @@ -383,7 +397,8 @@ static int sync_manager_info(struct hl_fpriv *hpriv, struct hl_info_args *args) prop->first_available_user_sob[args->dcore_id]; sm_info.first_available_monitor = prop->first_available_user_mon[args->dcore_id]; - + sm_info.first_available_cq = + prop->first_available_cq[args->dcore_id]; return copy_to_user(out, &sm_info, min_t(size_t, (size_t) max_size, sizeof(sm_info))) ? -EFAULT : 0; diff --git a/drivers/misc/habanalabs/common/hw_queue.c b/drivers/misc/habanalabs/common/hw_queue.c index 76217258780a..0f335182267f 100644 --- a/drivers/misc/habanalabs/common/hw_queue.c +++ b/drivers/misc/habanalabs/common/hw_queue.c @@ -38,7 +38,7 @@ static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len) return (abs(delta) - queue_len); } -void hl_int_hw_queue_update_ci(struct hl_cs *cs) +void hl_hw_queue_update_ci(struct hl_cs *cs) { struct hl_device *hdev = cs->ctx->hdev; struct hl_hw_queue *q; @@ -53,8 +53,13 @@ void hl_int_hw_queue_update_ci(struct hl_cs *cs) if (!hdev->asic_prop.max_queues || q->queue_type == QUEUE_TYPE_HW) return; + /* We must increment CI for every queue that will never get a + * completion, there are 2 scenarios this can happen: + * 1. All queues of a non completion CS will never get a completion. + * 2. Internal queues never gets completion. + */ for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) { - if (q->queue_type == QUEUE_TYPE_INT) + if (!cs_needs_completion(cs) || q->queue_type == QUEUE_TYPE_INT) atomic_add(cs->jobs_in_queue_cnt[i], &q->ci); } } @@ -292,6 +297,10 @@ static void ext_queue_schedule_job(struct hl_cs_job *job) len = job->job_cb_size; ptr = cb->bus_address; + /* Skip completion flow in case this is a non completion CS */ + if (!cs_needs_completion(job->cs)) + goto submit_bd; + cq_pkt.data = cpu_to_le32( ((q->pi << CQ_ENTRY_SHADOW_INDEX_SHIFT) & CQ_ENTRY_SHADOW_INDEX_MASK) | @@ -318,6 +327,7 @@ static void ext_queue_schedule_job(struct hl_cs_job *job) cq->pi = hl_cq_inc_ptr(cq->pi); +submit_bd: ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr); } @@ -525,6 +535,7 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) struct hl_cs_job *job, *tmp; struct hl_hw_queue *q; int rc = 0, i, cq_cnt; + bool first_entry; u32 max_queues; cntr = &hdev->aggregated_cs_counters; @@ -548,7 +559,9 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) switch (q->queue_type) { case QUEUE_TYPE_EXT: rc = ext_queue_sanity_checks(hdev, q, - cs->jobs_in_queue_cnt[i], true); + cs->jobs_in_queue_cnt[i], + cs_needs_completion(cs) ? + true : false); break; case QUEUE_TYPE_INT: rc = int_queue_sanity_checks(hdev, q, @@ -583,12 +596,38 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) hdev->asic_funcs->collective_wait_init_cs(cs); spin_lock(&hdev->cs_mirror_lock); + + /* Verify staged CS exists and add to the staged list */ + if (cs->staged_cs && !cs->staged_first) { + struct hl_cs *staged_cs; + + staged_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence); + if (!staged_cs) { + dev_err(hdev->dev, + "Cannot find staged submission sequence %llu", + cs->staged_sequence); + rc = -EINVAL; + goto unlock_cs_mirror; + } + + if (is_staged_cs_last_exists(hdev, staged_cs)) { + dev_err(hdev->dev, + "Staged submission sequence %llu already submitted", + cs->staged_sequence); + rc = -EINVAL; + goto unlock_cs_mirror; + } + + list_add_tail(&cs->staged_cs_node, &staged_cs->staged_cs_node); + } + list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list); /* Queue TDR if the CS is the first entry and if timeout is wanted */ + first_entry = list_first_entry(&hdev->cs_mirror_list, + struct hl_cs, mirror_node) == cs; if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) && - (list_first_entry(&hdev->cs_mirror_list, - struct hl_cs, mirror_node) == cs)) { + first_entry && cs_needs_timeout(cs)) { cs->tdr_active = true; schedule_delayed_work(&cs->work_tdr, hdev->timeout_jiffies); @@ -623,6 +662,8 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) goto out; +unlock_cs_mirror: + spin_unlock(&hdev->cs_mirror_lock); unroll_cq_resv: q = &hdev->kernel_queues[0]; for (i = 0 ; (i < max_queues) && (cq_cnt > 0) ; i++, q++) { diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index 245c0159159f..1f5910517b0e 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -14,6 +14,9 @@ #define HL_MMU_DEBUG 0 +/* use small pages for supporting non-pow2 (32M/40M/48M) DRAM phys page sizes */ +#define DRAM_POOL_PAGE_SIZE SZ_8M + /* * The va ranges in context object contain a list with the available chunks of * device virtual memory. @@ -38,15 +41,14 @@ */ /* - * alloc_device_memory - allocate device memory - * - * @ctx : current context - * @args : host parameters containing the requested size - * @ret_handle : result handle + * alloc_device_memory() - allocate device memory. + * @ctx: pointer to the context structure. + * @args: host parameters containing the requested size. + * @ret_handle: result handle. * * This function does the following: - * - Allocate the requested size rounded up to 'dram_page_size' pages - * - Return unique handle + * - Allocate the requested size rounded up to 'dram_page_size' pages. + * - Return unique handle for later map/unmap/free. */ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args, u32 *ret_handle) @@ -55,15 +57,14 @@ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args, struct hl_vm *vm = &hdev->vm; struct hl_vm_phys_pg_pack *phys_pg_pack; u64 paddr = 0, total_size, num_pgs, i; - u32 num_curr_pgs, page_size, page_shift; + u32 num_curr_pgs, page_size; int handle, rc; bool contiguous; num_curr_pgs = 0; page_size = hdev->asic_prop.dram_page_size; - page_shift = __ffs(page_size); - num_pgs = (args->alloc.mem_size + (page_size - 1)) >> page_shift; - total_size = num_pgs << page_shift; + num_pgs = DIV_ROUND_UP_ULL(args->alloc.mem_size, page_size); + total_size = num_pgs * page_size; if (!total_size) { dev_err(hdev->dev, "Cannot allocate 0 bytes\n"); @@ -182,17 +183,17 @@ pages_pack_err: return rc; } -/* - * dma_map_host_va - DMA mapping of the given host virtual address. - * @hdev: habanalabs device structure - * @addr: the host virtual address of the memory area - * @size: the size of the memory area - * @p_userptr: pointer to result userptr structure +/** + * dma_map_host_va() - DMA mapping of the given host virtual address. + * @hdev: habanalabs device structure. + * @addr: the host virtual address of the memory area. + * @size: the size of the memory area. + * @p_userptr: pointer to result userptr structure. * * This function does the following: - * - Allocate userptr structure - * - Pin the given host memory using the userptr structure - * - Perform DMA mapping to have the DMA addresses of the pages + * - Allocate userptr structure. + * - Pin the given host memory using the userptr structure. + * - Perform DMA mapping to have the DMA addresses of the pages. */ static int dma_map_host_va(struct hl_device *hdev, u64 addr, u64 size, struct hl_userptr **p_userptr) @@ -236,14 +237,14 @@ userptr_err: return rc; } -/* - * dma_unmap_host_va - DMA unmapping of the given host virtual address. - * @hdev: habanalabs device structure - * @userptr: userptr to free +/** + * dma_unmap_host_va() - DMA unmapping of the given host virtual address. + * @hdev: habanalabs device structure. + * @userptr: userptr to free. * * This function does the following: - * - Unpins the physical pages - * - Frees the userptr structure + * - Unpins the physical pages. + * - Frees the userptr structure. */ static void dma_unmap_host_va(struct hl_device *hdev, struct hl_userptr *userptr) @@ -252,14 +253,13 @@ static void dma_unmap_host_va(struct hl_device *hdev, kfree(userptr); } -/* - * dram_pg_pool_do_release - free DRAM pages pool - * - * @ref : pointer to reference object +/** + * dram_pg_pool_do_release() - free DRAM pages pool + * @ref: pointer to reference object. * * This function does the following: - * - Frees the idr structure of physical pages handles - * - Frees the generic pool of DRAM physical pages + * - Frees the idr structure of physical pages handles. + * - Frees the generic pool of DRAM physical pages. */ static void dram_pg_pool_do_release(struct kref *ref) { @@ -274,15 +274,15 @@ static void dram_pg_pool_do_release(struct kref *ref) gen_pool_destroy(vm->dram_pg_pool); } -/* - * free_phys_pg_pack - free physical page pack - * @hdev: habanalabs device structure - * @phys_pg_pack: physical page pack to free +/** + * free_phys_pg_pack() - free physical page pack. + * @hdev: habanalabs device structure. + * @phys_pg_pack: physical page pack to free. * * This function does the following: * - For DRAM memory only, iterate over the pack and free each physical block - * structure by returning it to the general pool - * - Free the hl_vm_phys_pg_pack structure + * structure by returning it to the general pool. + * - Free the hl_vm_phys_pg_pack structure. */ static void free_phys_pg_pack(struct hl_device *hdev, struct hl_vm_phys_pg_pack *phys_pg_pack) @@ -313,20 +313,20 @@ static void free_phys_pg_pack(struct hl_device *hdev, kfree(phys_pg_pack); } -/* - * free_device_memory - free device memory - * - * @ctx : current context - * @handle : handle of the memory chunk to free +/** + * free_device_memory() - free device memory. + * @ctx: pointer to the context structure. + * @args: host parameters containing the requested size. * * This function does the following: - * - Free the device memory related to the given handle + * - Free the device memory related to the given handle. */ -static int free_device_memory(struct hl_ctx *ctx, u32 handle) +static int free_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args) { struct hl_device *hdev = ctx->hdev; struct hl_vm *vm = &hdev->vm; struct hl_vm_phys_pg_pack *phys_pg_pack; + u32 handle = args->free.handle; spin_lock(&vm->idr_lock); phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); @@ -361,16 +361,15 @@ static int free_device_memory(struct hl_ctx *ctx, u32 handle) return 0; } -/* - * clear_va_list_locked - free virtual addresses list - * - * @hdev : habanalabs device structure - * @va_list : list of virtual addresses to free +/** + * clear_va_list_locked() - free virtual addresses list. + * @hdev: habanalabs device structure. + * @va_list: list of virtual addresses to free. * * This function does the following: - * - Iterate over the list and free each virtual addresses block + * - Iterate over the list and free each virtual addresses block. * - * This function should be called only when va_list lock is taken + * This function should be called only when va_list lock is taken. */ static void clear_va_list_locked(struct hl_device *hdev, struct list_head *va_list) @@ -383,16 +382,15 @@ static void clear_va_list_locked(struct hl_device *hdev, } } -/* - * print_va_list_locked - print virtual addresses list - * - * @hdev : habanalabs device structure - * @va_list : list of virtual addresses to print +/** + * print_va_list_locked() - print virtual addresses list. + * @hdev: habanalabs device structure. + * @va_list: list of virtual addresses to print. * * This function does the following: - * - Iterate over the list and print each virtual addresses block + * - Iterate over the list and print each virtual addresses block. * - * This function should be called only when va_list lock is taken + * This function should be called only when va_list lock is taken. */ static void print_va_list_locked(struct hl_device *hdev, struct list_head *va_list) @@ -409,18 +407,17 @@ static void print_va_list_locked(struct hl_device *hdev, #endif } -/* - * merge_va_blocks_locked - merge a virtual block if possible - * - * @hdev : pointer to the habanalabs device structure - * @va_list : pointer to the virtual addresses block list - * @va_block : virtual block to merge with adjacent blocks +/** + * merge_va_blocks_locked() - merge a virtual block if possible. + * @hdev: pointer to the habanalabs device structure. + * @va_list: pointer to the virtual addresses block list. + * @va_block: virtual block to merge with adjacent blocks. * * This function does the following: * - Merge the given blocks with the adjacent blocks if their virtual ranges - * create a contiguous virtual range + * create a contiguous virtual range. * - * This Function should be called only when va_list lock is taken + * This Function should be called only when va_list lock is taken. */ static void merge_va_blocks_locked(struct hl_device *hdev, struct list_head *va_list, struct hl_vm_va_block *va_block) @@ -445,19 +442,18 @@ static void merge_va_blocks_locked(struct hl_device *hdev, } } -/* - * add_va_block_locked - add a virtual block to the virtual addresses list - * - * @hdev : pointer to the habanalabs device structure - * @va_list : pointer to the virtual addresses block list - * @start : start virtual address - * @end : end virtual address +/** + * add_va_block_locked() - add a virtual block to the virtual addresses list. + * @hdev: pointer to the habanalabs device structure. + * @va_list: pointer to the virtual addresses block list. + * @start: start virtual address. + * @end: end virtual address. * * This function does the following: - * - Add the given block to the virtual blocks list and merge with other - * blocks if a contiguous virtual block can be created + * - Add the given block to the virtual blocks list and merge with other blocks + * if a contiguous virtual block can be created. * - * This Function should be called only when va_list lock is taken + * This Function should be called only when va_list lock is taken. */ static int add_va_block_locked(struct hl_device *hdev, struct list_head *va_list, u64 start, u64 end) @@ -501,16 +497,15 @@ static int add_va_block_locked(struct hl_device *hdev, return 0; } -/* - * add_va_block - wrapper for add_va_block_locked - * - * @hdev : pointer to the habanalabs device structure - * @va_list : pointer to the virtual addresses block list - * @start : start virtual address - * @end : end virtual address +/** + * add_va_block() - wrapper for add_va_block_locked. + * @hdev: pointer to the habanalabs device structure. + * @va_list: pointer to the virtual addresses block list. + * @start: start virtual address. + * @end: end virtual address. * * This function does the following: - * - Takes the list lock and calls add_va_block_locked + * - Takes the list lock and calls add_va_block_locked. */ static inline int add_va_block(struct hl_device *hdev, struct hl_va_range *va_range, u64 start, u64 end) @@ -524,8 +519,9 @@ static inline int add_va_block(struct hl_device *hdev, return rc; } -/* +/** * get_va_block() - get a virtual block for the given size and alignment. + * * @hdev: pointer to the habanalabs device structure. * @va_range: pointer to the virtual addresses range. * @size: requested block size. @@ -534,33 +530,51 @@ static inline int add_va_block(struct hl_device *hdev, * * This function does the following: * - Iterate on the virtual block list to find a suitable virtual block for the - * given size and alignment. + * given size, hint address and alignment. * - Reserve the requested block and update the list. * - Return the start address of the virtual block. */ -static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range, - u64 size, u64 hint_addr, u32 va_block_align) +static u64 get_va_block(struct hl_device *hdev, + struct hl_va_range *va_range, + u64 size, u64 hint_addr, u32 va_block_align) { struct hl_vm_va_block *va_block, *new_va_block = NULL; - u64 valid_start, valid_size, prev_start, prev_end, align_mask, - res_valid_start = 0, res_valid_size = 0; + u64 tmp_hint_addr, valid_start, valid_size, prev_start, prev_end, + align_mask, reserved_valid_start = 0, reserved_valid_size = 0; bool add_prev = false; + bool is_align_pow_2 = is_power_of_2(va_range->page_size); + + if (is_align_pow_2) + align_mask = ~((u64)va_block_align - 1); + else + /* + * with non-power-of-2 range we work only with page granularity + * and the start address is page aligned, + * so no need for alignment checking. + */ + size = DIV_ROUND_UP_ULL(size, va_range->page_size) * + va_range->page_size; - align_mask = ~((u64)va_block_align - 1); + tmp_hint_addr = hint_addr; - /* check if hint_addr is aligned */ - if (hint_addr & (va_block_align - 1)) + /* Check if we need to ignore hint address */ + if ((is_align_pow_2 && (hint_addr & (va_block_align - 1))) || + (!is_align_pow_2 && + do_div(tmp_hint_addr, va_range->page_size))) { + dev_info(hdev->dev, "Hint address 0x%llx will be ignored\n", + hint_addr); hint_addr = 0; + } mutex_lock(&va_range->lock); print_va_list_locked(hdev, &va_range->list); list_for_each_entry(va_block, &va_range->list, node) { - /* calc the first possible aligned addr */ + /* Calc the first possible aligned addr */ valid_start = va_block->start; - if (valid_start & (va_block_align - 1)) { + if (is_align_pow_2 && (valid_start & (va_block_align - 1))) { valid_start &= align_mask; valid_start += va_block_align; if (valid_start > va_block->end) @@ -568,35 +582,41 @@ static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range, } valid_size = va_block->end - valid_start; + if (valid_size < size) + continue; - if (valid_size >= size && - (!new_va_block || valid_size < res_valid_size)) { + /* Pick the minimal length block which has the required size */ + if (!new_va_block || (valid_size < reserved_valid_size)) { new_va_block = va_block; - res_valid_start = valid_start; - res_valid_size = valid_size; + reserved_valid_start = valid_start; + reserved_valid_size = valid_size; } if (hint_addr && hint_addr >= valid_start && - ((hint_addr + size) <= va_block->end)) { + (hint_addr + size) <= va_block->end) { new_va_block = va_block; - res_valid_start = hint_addr; - res_valid_size = valid_size; + reserved_valid_start = hint_addr; + reserved_valid_size = valid_size; break; } } if (!new_va_block) { dev_err(hdev->dev, "no available va block for size %llu\n", - size); + size); goto out; } - if (res_valid_start > new_va_block->start) { + /* + * Check if there is some leftover range due to reserving the new + * va block, then return it to the main virtual addresses list. + */ + if (reserved_valid_start > new_va_block->start) { prev_start = new_va_block->start; - prev_end = res_valid_start - 1; + prev_end = reserved_valid_start - 1; - new_va_block->start = res_valid_start; - new_va_block->size = res_valid_size; + new_va_block->start = reserved_valid_start; + new_va_block->size = reserved_valid_size; add_prev = true; } @@ -617,7 +637,7 @@ static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range, out: mutex_unlock(&va_range->lock); - return res_valid_start; + return reserved_valid_start; } /* @@ -644,9 +664,9 @@ u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, /** * hl_get_va_range_type() - get va_range type for the given address and size. - * @address: The start address of the area we want to validate. - * @size: The size in bytes of the area we want to validate. - * @type: returned va_range type + * @address: the start address of the area we want to validate. + * @size: the size in bytes of the area we want to validate. + * @type: returned va_range type. * * Return: true if the area is inside a valid range, false otherwise. */ @@ -667,16 +687,15 @@ static int hl_get_va_range_type(struct hl_ctx *ctx, u64 address, u64 size, return -EINVAL; } -/* - * hl_unreserve_va_block - wrapper for add_va_block for unreserving a va block - * +/** + * hl_unreserve_va_block() - wrapper for add_va_block to unreserve a va block. * @hdev: pointer to the habanalabs device structure - * @ctx: current context - * @start: start virtual address - * @end: end virtual address + * @ctx: pointer to the context structure. + * @start: start virtual address. + * @end: end virtual address. * * This function does the following: - * - Takes the list lock and calls add_va_block_locked + * - Takes the list lock and calls add_va_block_locked. */ int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, u64 start_addr, u64 size) @@ -701,11 +720,10 @@ int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, return rc; } -/* - * get_sg_info - get number of pages and the DMA address from SG list - * - * @sg : the SG list - * @dma_addr : pointer to DMA address to return +/** + * get_sg_info() - get number of pages and the DMA address from SG list. + * @sg: the SG list. + * @dma_addr: pointer to DMA address to return. * * Calculate the number of consecutive pages described by the SG list. Take the * offset of the address in the first page, add to it the length and round it up @@ -719,17 +737,17 @@ static u32 get_sg_info(struct scatterlist *sg, dma_addr_t *dma_addr) (PAGE_SIZE - 1)) >> PAGE_SHIFT; } -/* - * init_phys_pg_pack_from_userptr - initialize physical page pack from host - * memory - * @ctx: current context - * @userptr: userptr to initialize from - * @pphys_pg_pack: result pointer +/** + * init_phys_pg_pack_from_userptr() - initialize physical page pack from host + * memory + * @ctx: pointer to the context structure. + * @userptr: userptr to initialize from. + * @pphys_pg_pack: result pointer. * * This function does the following: - * - Pin the physical pages related to the given virtual block + * - Pin the physical pages related to the given virtual block. * - Create a physical page pack from the physical pages related to the given - * virtual block + * virtual block. */ static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx, struct hl_userptr *userptr, @@ -821,16 +839,16 @@ page_pack_arr_mem_err: return rc; } -/* - * map_phys_pg_pack - maps the physical page pack. - * @ctx: current context - * @vaddr: start address of the virtual area to map from - * @phys_pg_pack: the pack of physical pages to map to +/** + * map_phys_pg_pack() - maps the physical page pack.. + * @ctx: pointer to the context structure. + * @vaddr: start address of the virtual area to map from. + * @phys_pg_pack: the pack of physical pages to map to. * * This function does the following: - * - Maps each chunk of virtual memory to matching physical chunk - * - Stores number of successful mappings in the given argument - * - Returns 0 on success, error code otherwise + * - Maps each chunk of virtual memory to matching physical chunk. + * - Stores number of successful mappings in the given argument. + * - Returns 0 on success, error code otherwise. */ static int map_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr, struct hl_vm_phys_pg_pack *phys_pg_pack) @@ -875,11 +893,11 @@ err: return rc; } -/* - * unmap_phys_pg_pack - unmaps the physical page pack - * @ctx: current context - * @vaddr: start address of the virtual area to unmap - * @phys_pg_pack: the pack of physical pages to unmap +/** + * unmap_phys_pg_pack() - unmaps the physical page pack. + * @ctx: pointer to the context structure. + * @vaddr: start address of the virtual area to unmap. + * @phys_pg_pack: the pack of physical pages to unmap. */ static void unmap_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr, struct hl_vm_phys_pg_pack *phys_pg_pack) @@ -913,7 +931,7 @@ static void unmap_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr, } static int get_paddr_from_handle(struct hl_ctx *ctx, struct hl_mem_in *args, - u64 *paddr) + u64 *paddr) { struct hl_device *hdev = ctx->hdev; struct hl_vm *vm = &hdev->vm; @@ -936,19 +954,18 @@ static int get_paddr_from_handle(struct hl_ctx *ctx, struct hl_mem_in *args, return 0; } -/* - * map_device_va - map the given memory - * - * @ctx : current context - * @args : host parameters with handle/host virtual address - * @device_addr : pointer to result device virtual address +/** + * map_device_va() - map the given memory. + * @ctx: pointer to the context structure. + * @args: host parameters with handle/host virtual address. + * @device_addr: pointer to result device virtual address. * * This function does the following: * - If given a physical device memory handle, map to a device virtual block - * and return the start address of this block + * and return the start address of this block. * - If given a host virtual address and size, find the related physical pages, * map a device virtual block to this pages and return the start address of - * this block + * this block. */ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, u64 *device_addr) @@ -1034,7 +1051,7 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, hint_addr = args->map_device.hint_addr; - /* DRAM VA alignment is the same as the DRAM page size */ + /* DRAM VA alignment is the same as the MMU page size */ va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM]; va_block_align = hdev->asic_prop.dmmu.page_size; } @@ -1125,24 +1142,26 @@ init_page_pack_err: return rc; } -/* - * unmap_device_va - unmap the given device virtual address - * - * @ctx : current context - * @vaddr : device virtual address to unmap - * @ctx_free : true if in context free flow, false otherwise. +/** + * unmap_device_va() - unmap the given device virtual address. + * @ctx: pointer to the context structure. + * @args: host parameters with device virtual address to unmap. + * @ctx_free: true if in context free flow, false otherwise. * * This function does the following: - * - Unmap the physical pages related to the given virtual address - * - return the device virtual block to the virtual block list + * - unmap the physical pages related to the given virtual address. + * - return the device virtual block to the virtual block list. */ -static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr, bool ctx_free) +static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, + bool ctx_free) { struct hl_device *hdev = ctx->hdev; + struct asic_fixed_properties *prop = &hdev->asic_prop; struct hl_vm_phys_pg_pack *phys_pg_pack = NULL; struct hl_vm_hash_node *hnode = NULL; struct hl_userptr *userptr = NULL; struct hl_va_range *va_range; + u64 vaddr = args->unmap.device_virt_addr; enum vm_type_t *vm_type; bool is_userptr; int rc = 0; @@ -1201,7 +1220,13 @@ static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr, bool ctx_free) goto mapping_cnt_err; } - vaddr &= ~(((u64) phys_pg_pack->page_size) - 1); + if (!is_userptr && !is_power_of_2(phys_pg_pack->page_size)) + vaddr = prop->dram_base_address + + DIV_ROUND_DOWN_ULL(vaddr - prop->dram_base_address, + phys_pg_pack->page_size) * + phys_pg_pack->page_size; + else + vaddr &= ~(((u64) phys_pg_pack->page_size) - 1); mutex_lock(&ctx->mmu_lock); @@ -1264,12 +1289,90 @@ vm_type_err: return rc; } +static int map_block(struct hl_device *hdev, u64 address, u64 *handle, + u32 *size) +{ + u32 block_id = 0; + int rc; + + rc = hdev->asic_funcs->get_hw_block_id(hdev, address, size, &block_id); + + *handle = block_id | HL_MMAP_TYPE_BLOCK; + *handle <<= PAGE_SHIFT; + + return rc; +} + +static void hw_block_vm_close(struct vm_area_struct *vma) +{ + struct hl_ctx *ctx = (struct hl_ctx *) vma->vm_private_data; + + hl_ctx_put(ctx); + vma->vm_private_data = NULL; +} + +static const struct vm_operations_struct hw_block_vm_ops = { + .close = hw_block_vm_close +}; + +/** + * hl_hw_block_mmap() - mmap a hw block to user. + * @hpriv: pointer to the private data of the fd + * @vma: pointer to vm_area_struct of the process + * + * Driver increments context reference for every HW block mapped in order + * to prevent user from closing FD without unmapping first + */ +int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma) +{ + struct hl_device *hdev = hpriv->hdev; + u32 block_id, block_size; + int rc; + + /* We use the page offset to hold the block id and thus we need to clear + * it before doing the mmap itself + */ + block_id = vma->vm_pgoff; + vma->vm_pgoff = 0; + + /* Driver only allows mapping of a complete HW block */ + block_size = vma->vm_end - vma->vm_start; + +#ifdef _HAS_TYPE_ARG_IN_ACCESS_OK + if (!access_ok(VERIFY_WRITE, + (void __user *) (uintptr_t) vma->vm_start, block_size)) { +#else + if (!access_ok((void __user *) (uintptr_t) vma->vm_start, block_size)) { +#endif + dev_err(hdev->dev, + "user pointer is invalid - 0x%lx\n", + vma->vm_start); + + return -EINVAL; + } + + vma->vm_ops = &hw_block_vm_ops; + vma->vm_private_data = hpriv->ctx; + + hl_ctx_get(hdev, hpriv->ctx); + + rc = hdev->asic_funcs->hw_block_mmap(hdev, vma, block_id, block_size); + if (rc) { + hl_ctx_put(hpriv->ctx); + return rc; + } + + vma->vm_pgoff = block_id; + + return 0; +} + static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args) { struct hl_device *hdev = hpriv->hdev; struct hl_ctx *ctx = hpriv->ctx; - u64 device_addr = 0; - u32 handle = 0; + u64 block_handle, device_addr = 0; + u32 handle = 0, block_size; int rc; switch (args->in.op) { @@ -1292,7 +1395,7 @@ static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args) break; case HL_MEM_OP_FREE: - rc = free_device_memory(ctx, args->in.free.handle); + rc = free_device_memory(ctx, &args->in); break; case HL_MEM_OP_MAP: @@ -1301,7 +1404,7 @@ static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args) rc = 0; } else { rc = get_paddr_from_handle(ctx, &args->in, - &device_addr); + &device_addr); } memset(args, 0, sizeof(*args)); @@ -1312,6 +1415,13 @@ static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args) rc = 0; break; + case HL_MEM_OP_MAP_BLOCK: + rc = map_block(hdev, args->in.map_block.block_addr, + &block_handle, &block_size); + args->out.block_handle = block_handle; + args->out.block_size = block_size; + break; + default: dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n"); rc = -ENOTTY; @@ -1328,8 +1438,8 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) union hl_mem_args *args = data; struct hl_device *hdev = hpriv->hdev; struct hl_ctx *ctx = hpriv->ctx; - u64 device_addr = 0; - u32 handle = 0; + u64 block_handle, device_addr = 0; + u32 handle = 0, block_size; int rc; if (!hl_device_operational(hdev, &status)) { @@ -1400,7 +1510,7 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) goto out; } - rc = free_device_memory(ctx, args->in.free.handle); + rc = free_device_memory(ctx, &args->in); break; case HL_MEM_OP_MAP: @@ -1411,8 +1521,14 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) break; case HL_MEM_OP_UNMAP: - rc = unmap_device_va(ctx, args->in.unmap.device_virt_addr, - false); + rc = unmap_device_va(ctx, &args->in, false); + break; + + case HL_MEM_OP_MAP_BLOCK: + rc = map_block(hdev, args->in.map_block.block_addr, + &block_handle, &block_size); + args->out.block_handle = block_handle; + args->out.block_size = block_size; break; default: @@ -1473,16 +1589,16 @@ destroy_pages: return rc; } -/* - * hl_pin_host_memory - pins a chunk of host memory. - * @hdev: pointer to the habanalabs device structure - * @addr: the host virtual address of the memory area - * @size: the size of the memory area - * @userptr: pointer to hl_userptr structure +/** + * hl_pin_host_memory() - pins a chunk of host memory. + * @hdev: pointer to the habanalabs device structure. + * @addr: the host virtual address of the memory area. + * @size: the size of the memory area. + * @userptr: pointer to hl_userptr structure. * * This function does the following: - * - Pins the physical pages - * - Create an SG list from those pages + * - Pins the physical pages. + * - Create an SG list from those pages. */ int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size, struct hl_userptr *userptr) @@ -1571,11 +1687,10 @@ void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr) kfree(userptr->sgt); } -/* - * hl_userptr_delete_list - clear userptr list - * - * @hdev : pointer to the habanalabs device structure - * @userptr_list : pointer to the list to clear +/** + * hl_userptr_delete_list() - clear userptr list. + * @hdev: pointer to the habanalabs device structure. + * @userptr_list: pointer to the list to clear. * * This function does the following: * - Iterates over the list and unpins the host memory and frees the userptr @@ -1594,12 +1709,11 @@ void hl_userptr_delete_list(struct hl_device *hdev, INIT_LIST_HEAD(userptr_list); } -/* - * hl_userptr_is_pinned - returns whether the given userptr is pinned - * - * @hdev : pointer to the habanalabs device structure - * @userptr_list : pointer to the list to clear - * @userptr : pointer to userptr to check +/** + * hl_userptr_is_pinned() - returns whether the given userptr is pinned. + * @hdev: pointer to the habanalabs device structure. + * @userptr_list: pointer to the list to clear. + * @userptr: pointer to userptr to check. * * This function does the following: * - Iterates over the list and checks if the given userptr is in it, means is @@ -1617,12 +1731,12 @@ bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr, return false; } -/* - * va_range_init - initialize virtual addresses range - * @hdev: pointer to the habanalabs device structure - * @va_range: pointer to the range to initialize - * @start: range start address - * @end: range end address +/** + * va_range_init() - initialize virtual addresses range. + * @hdev: pointer to the habanalabs device structure. + * @va_range: pointer to the range to initialize. + * @start: range start address. + * @end: range end address. * * This function does the following: * - Initializes the virtual addresses list of the given range with the given @@ -1635,15 +1749,21 @@ static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range, INIT_LIST_HEAD(&va_range->list); - /* PAGE_SIZE alignment */ + /* + * PAGE_SIZE alignment + * it is the callers responsibility to align the addresses if the + * page size is not a power of 2 + */ - if (start & (PAGE_SIZE - 1)) { - start &= PAGE_MASK; - start += PAGE_SIZE; - } + if (is_power_of_2(page_size)) { + if (start & (PAGE_SIZE - 1)) { + start &= PAGE_MASK; + start += PAGE_SIZE; + } - if (end & (PAGE_SIZE - 1)) - end &= PAGE_MASK; + if (end & (PAGE_SIZE - 1)) + end &= PAGE_MASK; + } if (start >= end) { dev_err(hdev->dev, "too small vm range for va list\n"); @@ -1664,13 +1784,13 @@ static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range, return 0; } -/* - * va_range_fini() - clear a virtual addresses range - * @hdev: pointer to the habanalabs structure - * va_range: pointer to virtual addresses range +/** + * va_range_fini() - clear a virtual addresses range. + * @hdev: pointer to the habanalabs structure. + * va_range: pointer to virtual addresses rang.e * * This function does the following: - * - Frees the virtual addresses block list and its lock + * - Frees the virtual addresses block list and its lock. */ static void va_range_fini(struct hl_device *hdev, struct hl_va_range *va_range) { @@ -1682,22 +1802,22 @@ static void va_range_fini(struct hl_device *hdev, struct hl_va_range *va_range) kfree(va_range); } -/* - * vm_ctx_init_with_ranges() - initialize virtual memory for context - * @ctx: pointer to the habanalabs context structure +/** + * vm_ctx_init_with_ranges() - initialize virtual memory for context. + * @ctx: pointer to the habanalabs context structure. * @host_range_start: host virtual addresses range start. * @host_range_end: host virtual addresses range end. * @host_huge_range_start: host virtual addresses range start for memory - * allocated with huge pages. + * allocated with huge pages. * @host_huge_range_end: host virtual addresses range end for memory allocated * with huge pages. * @dram_range_start: dram virtual addresses range start. * @dram_range_end: dram virtual addresses range end. * * This function initializes the following: - * - MMU for context - * - Virtual address to area descriptor hashtable - * - Virtual block list of available virtual memory + * - MMU for context. + * - Virtual address to area descriptor hashtable. + * - Virtual block list of available virtual memory. */ static int vm_ctx_init_with_ranges(struct hl_ctx *ctx, u64 host_range_start, @@ -1818,7 +1938,8 @@ int hl_vm_ctx_init(struct hl_ctx *ctx) dram_range_start = prop->dmmu.start_addr; dram_range_end = prop->dmmu.end_addr; - dram_page_size = prop->dmmu.page_size; + dram_page_size = prop->dram_page_size ? + prop->dram_page_size : prop->dmmu.page_size; host_range_start = prop->pmmu.start_addr; host_range_end = prop->pmmu.end_addr; host_page_size = prop->pmmu.page_size; @@ -1832,15 +1953,14 @@ int hl_vm_ctx_init(struct hl_ctx *ctx) dram_range_start, dram_range_end, dram_page_size); } -/* - * hl_vm_ctx_fini - virtual memory teardown of context - * - * @ctx : pointer to the habanalabs context structure +/** + * hl_vm_ctx_fini() - virtual memory teardown of context. + * @ctx: pointer to the habanalabs context structure. * * This function perform teardown the following: - * - Virtual block list of available virtual memory - * - Virtual address to area descriptor hashtable - * - MMU for context + * - Virtual block list of available virtual memory. + * - Virtual address to area descriptor hashtable. + * - MMU for context. * * In addition this function does the following: * - Unmaps the existing hashtable nodes if the hashtable is not empty. The @@ -1859,9 +1979,10 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx) struct hl_vm_phys_pg_pack *phys_pg_list; struct hl_vm_hash_node *hnode; struct hlist_node *tmp_node; + struct hl_mem_in args; int i; - if (!ctx->hdev->mmu_enable) + if (!hdev->mmu_enable) return; hl_debugfs_remove_ctx_mem_hash(hdev, ctx); @@ -1878,13 +1999,18 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx) dev_dbg(hdev->dev, "hl_mem_hash_node of vaddr 0x%llx of asid %d is still alive\n", hnode->vaddr, ctx->asid); - unmap_device_va(ctx, hnode->vaddr, true); + args.unmap.device_virt_addr = hnode->vaddr; + unmap_device_va(ctx, &args, true); } + mutex_lock(&ctx->mmu_lock); + /* invalidate the cache once after the unmapping loop */ hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR); hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_PHYS_PACK); + mutex_unlock(&ctx->mmu_lock); + spin_lock(&vm->idr_lock); idr_for_each_entry(&vm->phys_pg_pack_handles, phys_pg_list, i) if (phys_pg_list->asid == ctx->asid) { @@ -1911,19 +2037,19 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx) * because the user notifies us on allocations. If the user is no more, * all DRAM is available */ - if (!ctx->hdev->asic_prop.dram_supports_virtual_memory) - atomic64_set(&ctx->hdev->dram_used_mem, 0); + if (ctx->asid != HL_KERNEL_ASID_ID && + !hdev->asic_prop.dram_supports_virtual_memory) + atomic64_set(&hdev->dram_used_mem, 0); } -/* - * hl_vm_init - initialize virtual memory module - * - * @hdev : pointer to the habanalabs device structure +/** + * hl_vm_init() - initialize virtual memory module. + * @hdev: pointer to the habanalabs device structure. * * This function initializes the following: - * - MMU module - * - DRAM physical pages pool of 2MB - * - Idr for device memory allocation handles + * - MMU module. + * - DRAM physical pages pool of 2MB. + * - Idr for device memory allocation handles. */ int hl_vm_init(struct hl_device *hdev) { @@ -1931,7 +2057,13 @@ int hl_vm_init(struct hl_device *hdev) struct hl_vm *vm = &hdev->vm; int rc; - vm->dram_pg_pool = gen_pool_create(__ffs(prop->dram_page_size), -1); + if (is_power_of_2(prop->dram_page_size)) + vm->dram_pg_pool = + gen_pool_create(__ffs(prop->dram_page_size), -1); + else + vm->dram_pg_pool = + gen_pool_create(__ffs(DRAM_POOL_PAGE_SIZE), -1); + if (!vm->dram_pg_pool) { dev_err(hdev->dev, "Failed to create dram page pool\n"); return -ENOMEM; @@ -1964,15 +2096,14 @@ pool_add_err: return rc; } -/* - * hl_vm_fini - virtual memory module teardown - * - * @hdev : pointer to the habanalabs device structure +/** + * hl_vm_fini() - virtual memory module teardown. + * @hdev: pointer to the habanalabs device structure. * * This function perform teardown to the following: - * - Idr for device memory allocation handles - * - DRAM physical pages pool of 2MB - * - MMU module + * - Idr for device memory allocation handles. + * - DRAM physical pages pool of 2MB. + * - MMU module. */ void hl_vm_fini(struct hl_device *hdev) { diff --git a/drivers/misc/habanalabs/common/mmu/Makefile b/drivers/misc/habanalabs/common/mmu/Makefile new file mode 100644 index 000000000000..d852c3874658 --- /dev/null +++ b/drivers/misc/habanalabs/common/mmu/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +HL_COMMON_MMU_FILES := common/mmu/mmu.o common/mmu/mmu_v1.o diff --git a/drivers/misc/habanalabs/common/mmu.c b/drivers/misc/habanalabs/common/mmu/mmu.c index 28a4638741d8..71703a32350f 100644 --- a/drivers/misc/habanalabs/common/mmu.c +++ b/drivers/misc/habanalabs/common/mmu/mmu.c @@ -7,7 +7,7 @@ #include <linux/slab.h> -#include "habanalabs.h" +#include "../habanalabs.h" bool hl_is_dram_va(struct hl_device *hdev, u64 virt_addr) { @@ -166,7 +166,6 @@ int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size, mmu_prop = &prop->pmmu; pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT; - /* * The H/W handles mapping of specific page sizes. Hence if the page * size is bigger, we break it to sub-pages and unmap them separately. @@ -174,11 +173,21 @@ int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size, if ((page_size % mmu_prop->page_size) == 0) { real_page_size = mmu_prop->page_size; } else { - dev_err(hdev->dev, - "page size of %u is not %uKB aligned, can't unmap\n", - page_size, mmu_prop->page_size >> 10); + /* + * MMU page size may differ from DRAM page size. + * In such case work with the DRAM page size and let the MMU + * scrambling routine to handle this mismatch when + * calculating the address to remove from the MMU page table + */ + if (is_dram_addr && ((page_size % prop->dram_page_size) == 0)) { + real_page_size = prop->dram_page_size; + } else { + dev_err(hdev->dev, + "page size of %u is not %uKB aligned, can't unmap\n", + page_size, mmu_prop->page_size >> 10); - return -EFAULT; + return -EFAULT; + } } npages = page_size / real_page_size; @@ -253,6 +262,17 @@ int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, */ if ((page_size % mmu_prop->page_size) == 0) { real_page_size = mmu_prop->page_size; + } else if (is_dram_addr && ((page_size % prop->dram_page_size) == 0) && + (prop->dram_page_size < mmu_prop->page_size)) { + /* + * MMU page size may differ from DRAM page size. + * In such case work with the DRAM page size and let the MMU + * scrambling routine handle this mismatch when calculating + * the address to place in the MMU page table. (in that case + * also make sure that the dram_page_size smaller than the + * mmu page size) + */ + real_page_size = prop->dram_page_size; } else { dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n", @@ -261,9 +281,21 @@ int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, return -EFAULT; } - WARN_ONCE((phys_addr & (real_page_size - 1)), - "Mapping 0x%llx with page size of 0x%x is erroneous! Address must be divisible by page size", - phys_addr, real_page_size); + /* + * Verify that the phys and virt addresses are aligned with the + * MMU page size (in dram this means checking the address and MMU + * after scrambling) + */ + if ((is_dram_addr && + ((hdev->asic_funcs->scramble_addr(hdev, phys_addr) & + (mmu_prop->page_size - 1)) || + (hdev->asic_funcs->scramble_addr(hdev, virt_addr) & + (mmu_prop->page_size - 1)))) || + (!is_dram_addr && ((phys_addr & (real_page_size - 1)) || + (virt_addr & (real_page_size - 1))))) + dev_crit(hdev->dev, + "Mapping address 0x%llx with virtual address 0x%llx and page size of 0x%x is erroneous! Addresses must be divisible by page size", + phys_addr, virt_addr, real_page_size); npages = page_size / real_page_size; real_virt_addr = virt_addr; @@ -444,19 +476,53 @@ void hl_mmu_swap_in(struct hl_ctx *ctx) hdev->mmu_func[MMU_HR_PGT].swap_in(ctx); } +static void hl_mmu_pa_page_with_offset(struct hl_ctx *ctx, u64 virt_addr, + struct hl_mmu_hop_info *hops, + u64 *phys_addr) +{ + struct hl_device *hdev = ctx->hdev; + struct asic_fixed_properties *prop = &hdev->asic_prop; + u64 offset_mask, addr_mask, hop_shift, tmp_phys_addr; + u32 hop0_shift_off; + void *p; + + /* last hop holds the phys address and flags */ + if (hops->unscrambled_paddr) + tmp_phys_addr = hops->unscrambled_paddr; + else + tmp_phys_addr = hops->hop_info[hops->used_hops - 1].hop_pte_val; + + if (hops->range_type == HL_VA_RANGE_TYPE_HOST_HUGE) + p = &prop->pmmu_huge; + else if (hops->range_type == HL_VA_RANGE_TYPE_HOST) + p = &prop->pmmu; + else /* HL_VA_RANGE_TYPE_DRAM */ + p = &prop->dmmu; + + /* + * find the correct hop shift field in hl_mmu_properties structure + * in order to determine the right maks for the page offset. + */ + hop0_shift_off = offsetof(struct hl_mmu_properties, hop0_shift); + p = (char *)p + hop0_shift_off; + p = (char *)p + ((hops->used_hops - 1) * sizeof(u64)); + hop_shift = *(u64 *)p; + offset_mask = (1ull << hop_shift) - 1; + addr_mask = ~(offset_mask); + *phys_addr = (tmp_phys_addr & addr_mask) | + (virt_addr & offset_mask); +} + int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr) { struct hl_mmu_hop_info hops; - u64 tmp_addr; int rc; rc = hl_mmu_get_tlb_info(ctx, virt_addr, &hops); if (rc) return rc; - /* last hop holds the phys address and flags */ - tmp_addr = hops.hop_info[hops.used_hops - 1].hop_pte_val; - *phys_addr = (tmp_addr & HOP_PHYS_ADDR_MASK) | (virt_addr & FLAGS_MASK); + hl_mmu_pa_page_with_offset(ctx, virt_addr, &hops, phys_addr); return 0; } @@ -473,6 +539,8 @@ int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, if (!hdev->mmu_enable) return -EOPNOTSUPP; + hops->scrambled_vaddr = virt_addr; /* assume no scrambling */ + is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size, prop->dmmu.start_addr, prop->dmmu.end_addr); @@ -491,6 +559,11 @@ int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, mutex_unlock(&ctx->mmu_lock); + /* add page offset to physical address */ + if (hops->unscrambled_paddr) + hl_mmu_pa_page_with_offset(ctx, virt_addr, hops, + &hops->unscrambled_paddr); + return rc; } @@ -512,3 +585,28 @@ int hl_mmu_if_set_funcs(struct hl_device *hdev) return 0; } + +/** + * hl_mmu_scramble_addr() - The generic mmu address scrambling routine. + * @hdev: pointer to device data. + * @addr: The address to scramble. + * + * Return: The scrambled address. + */ +u64 hl_mmu_scramble_addr(struct hl_device *hdev, u64 addr) +{ + return addr; +} + +/** + * hl_mmu_descramble_addr() - The generic mmu address descrambling + * routine. + * @hdev: pointer to device data. + * @addr: The address to descramble. + * + * Return: The un-scrambled address. + */ +u64 hl_mmu_descramble_addr(struct hl_device *hdev, u64 addr) +{ + return addr; +} diff --git a/drivers/misc/habanalabs/common/mmu_v1.c b/drivers/misc/habanalabs/common/mmu/mmu_v1.c index 06d8a44dd5d4..c5e93ff32586 100644 --- a/drivers/misc/habanalabs/common/mmu_v1.c +++ b/drivers/misc/habanalabs/common/mmu/mmu_v1.c @@ -5,8 +5,8 @@ * All Rights Reserved. */ -#include "habanalabs.h" -#include "../include/hw_ip/mmu/mmu_general.h" +#include "../habanalabs.h" +#include "../../include/hw_ip/mmu/mmu_general.h" #include <linux/slab.h> diff --git a/drivers/misc/habanalabs/common/pci/Makefile b/drivers/misc/habanalabs/common/pci/Makefile new file mode 100644 index 000000000000..dc922a686683 --- /dev/null +++ b/drivers/misc/habanalabs/common/pci/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +HL_COMMON_PCI_FILES := common/pci/pci.o diff --git a/drivers/misc/habanalabs/common/pci.c b/drivers/misc/habanalabs/common/pci/pci.c index b4725e6101f6..b799f9258fb0 100644 --- a/drivers/misc/habanalabs/common/pci.c +++ b/drivers/misc/habanalabs/common/pci/pci.c @@ -5,8 +5,8 @@ * All Rights Reserved. */ -#include "habanalabs.h" -#include "../include/hw_ip/pci/pci_general.h" +#include "../habanalabs.h" +#include "../../include/hw_ip/pci/pci_general.h" #include <linux/pci.h> @@ -308,40 +308,6 @@ int hl_pci_set_outbound_region(struct hl_device *hdev, } /** - * hl_pci_set_dma_mask() - Set DMA masks for the device. - * @hdev: Pointer to hl_device structure. - * - * This function sets the DMA masks (regular and consistent) for a specified - * value. If it doesn't succeed, it tries to set it to a fall-back value - * - * Return: 0 on success, non-zero for failure. - */ -static int hl_pci_set_dma_mask(struct hl_device *hdev) -{ - struct pci_dev *pdev = hdev->pdev; - int rc; - - /* set DMA mask */ - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(hdev->dma_mask)); - if (rc) { - dev_err(hdev->dev, - "Failed to set pci dma mask to %d bits, error %d\n", - hdev->dma_mask, rc); - return rc; - } - - rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(hdev->dma_mask)); - if (rc) { - dev_err(hdev->dev, - "Failed to set pci consistent dma mask to %d bits, error %d\n", - hdev->dma_mask, rc); - return rc; - } - - return 0; -} - -/** * hl_pci_init() - PCI initialization code. * @hdev: Pointer to hl_device structure. * @@ -377,9 +343,14 @@ int hl_pci_init(struct hl_device *hdev) goto unmap_pci_bars; } - rc = hl_pci_set_dma_mask(hdev); - if (rc) + rc = dma_set_mask_and_coherent(&pdev->dev, + DMA_BIT_MASK(hdev->dma_mask)); + if (rc) { + dev_err(hdev->dev, + "Failed to set dma mask to %d bits, error %d\n", + hdev->dma_mask, rc); goto unmap_pci_bars; + } return 0; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index b328ddaa64ee..9152242778f5 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -225,6 +225,12 @@ gaudi_qman_arb_error_cause[GAUDI_NUM_OF_QM_ARB_ERR_CAUSE] = { "MSG AXI LBW returned with error" }; +enum gaudi_sm_sei_cause { + GAUDI_SM_SEI_SO_OVERFLOW, + GAUDI_SM_SEI_LBW_4B_UNALIGNED, + GAUDI_SM_SEI_AXI_RESPONSE_ERR +}; + static enum hl_queue_type gaudi_queue_type[GAUDI_QUEUE_ID_SIZE] = { QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_0 */ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_1 */ @@ -354,6 +360,10 @@ static int gaudi_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job); static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr, u32 size, u64 val); +static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base, + u32 num_regs, u32 val); +static int gaudi_schedule_register_memset(struct hl_device *hdev, + u32 hw_queue_id, u64 reg_base, u32 num_regs, u32 val); static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel, u32 tpc_id); static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev); @@ -517,6 +527,11 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev) prop->sync_stream_first_mon + (num_sync_stream_queues * HL_RSVD_MONS); + prop->first_available_user_msix_interrupt = USHRT_MAX; + + for (i = 0 ; i < HL_MAX_DCORES ; i++) + prop->first_available_cq[i] = USHRT_MAX; + /* disable fw security for now, set it in a later stage */ prop->fw_security_disabled = true; prop->fw_security_status_valid = false; @@ -913,11 +928,17 @@ static void gaudi_sob_group_hw_reset(struct kref *ref) struct gaudi_hw_sob_group *hw_sob_group = container_of(ref, struct gaudi_hw_sob_group, kref); struct hl_device *hdev = hw_sob_group->hdev; - int i; + u64 base_addr; + int rc; - for (i = 0 ; i < NUMBER_OF_SOBS_IN_GRP ; i++) - WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + - (hw_sob_group->base_sob_id + i) * 4, 0); + base_addr = CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + + hw_sob_group->base_sob_id * 4; + rc = gaudi_schedule_register_memset(hdev, hw_sob_group->queue_id, + base_addr, NUMBER_OF_SOBS_IN_GRP, 0); + if (rc) + dev_err(hdev->dev, + "failed resetting sob group - sob base %u, count %u", + hw_sob_group->base_sob_id, NUMBER_OF_SOBS_IN_GRP); kref_init(&hw_sob_group->kref); } @@ -1008,6 +1029,8 @@ static void gaudi_collective_master_init_job(struct hl_device *hdev, cprop->hw_sob_group[sob_group_offset].base_sob_id; master_monitor = prop->collective_mstr_mon_id[0]; + cprop->hw_sob_group[sob_group_offset].queue_id = queue_id; + dev_dbg(hdev->dev, "Generate master wait CBs, sob %d (mask %#x), val:0x%x, mon %u, q %d\n", master_sob_base, cprop->mstr_sob_mask[0], @@ -1248,7 +1271,7 @@ static int gaudi_collective_wait_create_jobs(struct hl_device *hdev, u32 queue_id, collective_queue, num_jobs; u32 stream, nic_queue, nic_idx = 0; bool skip; - int i, rc; + int i, rc = 0; /* Verify wait queue id is configured as master */ hw_queue_prop = &hdev->asic_prop.hw_queues_props[wait_queue_id]; @@ -1359,8 +1382,6 @@ static int gaudi_late_init(struct hl_device *hdev) return rc; } - WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_INTS_REGISTER); - rc = gaudi_fetch_psoc_frequency(hdev); if (rc) { dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); @@ -1607,6 +1628,7 @@ static int gaudi_sw_init(struct hl_device *hdev) hdev->supports_sync_stream = true; hdev->supports_coresight = true; + hdev->supports_staged_submission = true; return 0; @@ -3438,6 +3460,12 @@ static void gaudi_set_clock_gating(struct hl_device *hdev) enable = !!(hdev->clock_gating_mask & (BIT_ULL(gaudi_dma_assignment[i]))); + /* GC sends work to DMA engine through Upper CP in DMA5 so + * we need to not enable clock gating in that DMA + */ + if (i == GAUDI_HBM_DMA_4) + enable = 0; + qman_offset = gaudi_dma_assignment[i] * DMA_QMAN_OFFSET; WREG32(mmDMA0_QM_CGM_CFG1 + qman_offset, enable ? QMAN_CGM1_PWR_GATE_EN : 0); @@ -3704,6 +3732,7 @@ static int gaudi_init_cpu(struct hl_device *hdev) static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) { struct gaudi_device *gaudi = hdev->asic_specific; + struct asic_fixed_properties *prop = &hdev->asic_prop; struct hl_eq *eq; u32 status; struct hl_hw_queue *cpu_pq = @@ -3760,6 +3789,10 @@ static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) return -EIO; } + /* update FW application security bits */ + if (prop->fw_security_status_valid) + prop->fw_app_security_map = RREG32(mmCPU_BOOT_DEV_STS0); + gaudi->hw_cap_initialized |= HW_CAP_CPU_Q; return 0; } @@ -4417,9 +4450,12 @@ static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) /* ring the doorbell */ WREG32(db_reg_offset, db_value); - if (hw_queue_id == GAUDI_QUEUE_ID_CPU_PQ) + if (hw_queue_id == GAUDI_QUEUE_ID_CPU_PQ) { + /* make sure device CPU will read latest data from host */ + mb(); WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_PI_UPDATE); + } } static void gaudi_pqe_write(struct hl_device *hdev, __le64 *pqe, @@ -4518,7 +4554,6 @@ static int gaudi_scrub_device_mem(struct hl_device *hdev, u64 addr, u64 size) { struct asic_fixed_properties *prop = &hdev->asic_prop; struct gaudi_device *gaudi = hdev->asic_specific; - u64 idle_mask = 0; int rc = 0; u64 val = 0; @@ -4531,8 +4566,8 @@ static int gaudi_scrub_device_mem(struct hl_device *hdev, u64 addr, u64 size) hdev, mmDMA0_CORE_STS0/* dummy */, val/* dummy */, - (hdev->asic_funcs->is_device_idle(hdev, - &idle_mask, NULL)), + (hdev->asic_funcs->is_device_idle(hdev, NULL, + 0, NULL)), 1000, HBM_SCRUBBING_TIMEOUT_US); if (rc) { @@ -5060,7 +5095,8 @@ static int gaudi_validate_cb(struct hl_device *hdev, * 1. A packet that will act as a completion packet * 2. A packet that will generate MSI-X interrupt */ - parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2; + if (parser->completion) + parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2; return rc; } @@ -5287,8 +5323,11 @@ static int gaudi_parse_cb_mmu(struct hl_device *hdev, * 1. A packet that will act as a completion packet * 2. A packet that will generate MSI interrupt */ - parser->patched_cb_size = parser->user_cb_size + - sizeof(struct packet_msg_prot) * 2; + if (parser->completion) + parser->patched_cb_size = parser->user_cb_size + + sizeof(struct packet_msg_prot) * 2; + else + parser->patched_cb_size = parser->user_cb_size; rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx, parser->patched_cb_size, false, false, @@ -5304,10 +5343,10 @@ static int gaudi_parse_cb_mmu(struct hl_device *hdev, patched_cb_handle >>= PAGE_SHIFT; parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) patched_cb_handle); - /* hl_cb_get should never fail here so use kernel WARN */ - WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n", - (u32) patched_cb_handle); + /* hl_cb_get should never fail */ if (!parser->patched_cb) { + dev_crit(hdev->dev, "DMA CB handle invalid 0x%x\n", + (u32) patched_cb_handle); rc = -EFAULT; goto out; } @@ -5376,10 +5415,10 @@ static int gaudi_parse_cb_no_mmu(struct hl_device *hdev, patched_cb_handle >>= PAGE_SHIFT; parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) patched_cb_handle); - /* hl_cb_get should never fail here so use kernel WARN */ - WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n", - (u32) patched_cb_handle); + /* hl_cb_get should never fail here */ if (!parser->patched_cb) { + dev_crit(hdev->dev, "DMA CB handle invalid 0x%x\n", + (u32) patched_cb_handle); rc = -EFAULT; goto out; } @@ -5579,31 +5618,206 @@ release_cb: return rc; } -static void gaudi_restore_sm_registers(struct hl_device *hdev) +static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base, + u32 num_regs, u32 val) +{ + struct packet_msg_long *pkt; + struct hl_cs_job *job; + u32 cb_size, ctl; + struct hl_cb *cb; + int i, rc; + + cb_size = (sizeof(*pkt) * num_regs) + sizeof(struct packet_msg_prot); + + if (cb_size > SZ_2M) { + dev_err(hdev->dev, "CB size must be smaller than %uMB", SZ_2M); + return -ENOMEM; + } + + cb = hl_cb_kernel_create(hdev, cb_size, false); + if (!cb) + return -EFAULT; + + pkt = cb->kernel_address; + + ctl = FIELD_PREP(GAUDI_PKT_LONG_CTL_OP_MASK, 0); /* write the value */ + ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_LONG); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); + + for (i = 0; i < num_regs ; i++, pkt++) { + pkt->ctl = cpu_to_le32(ctl); + pkt->value = cpu_to_le32(val); + pkt->addr = cpu_to_le64(reg_base + (i * 4)); + } + + job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true); + if (!job) { + dev_err(hdev->dev, "Failed to allocate a new job\n"); + rc = -ENOMEM; + goto release_cb; + } + + job->id = 0; + job->user_cb = cb; + atomic_inc(&job->user_cb->cs_cnt); + job->user_cb_size = cb_size; + job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0; + job->patched_cb = job->user_cb; + job->job_cb_size = cb_size; + + hl_debugfs_add_job(hdev, job); + + rc = gaudi_send_job_on_qman0(hdev, job); + hl_debugfs_remove_job(hdev, job); + kfree(job); + atomic_dec(&cb->cs_cnt); + +release_cb: + hl_cb_put(cb); + hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT); + + return rc; +} + +static int gaudi_schedule_register_memset(struct hl_device *hdev, + u32 hw_queue_id, u64 reg_base, u32 num_regs, u32 val) { + struct hl_ctx *ctx = hdev->compute_ctx; + struct hl_pending_cb *pending_cb; + struct packet_msg_long *pkt; + u32 cb_size, ctl; + struct hl_cb *cb; int i; - for (i = 0 ; i < NUM_OF_SOB_IN_BLOCK << 2 ; i += 4) { - WREG32(mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0); - WREG32(mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0); - WREG32(mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0); + /* If no compute context available or context is going down + * memset registers directly + */ + if (!ctx || kref_read(&ctx->refcount) == 0) + return gaudi_memset_registers(hdev, reg_base, num_regs, val); + + cb_size = (sizeof(*pkt) * num_regs) + + sizeof(struct packet_msg_prot) * 2; + + if (cb_size > SZ_2M) { + dev_err(hdev->dev, "CB size must be smaller than %uMB", SZ_2M); + return -ENOMEM; + } + + pending_cb = kzalloc(sizeof(*pending_cb), GFP_KERNEL); + if (!pending_cb) + return -ENOMEM; + + cb = hl_cb_kernel_create(hdev, cb_size, false); + if (!cb) { + kfree(pending_cb); + return -EFAULT; } - for (i = 0 ; i < NUM_OF_MONITORS_IN_BLOCK << 2 ; i += 4) { - WREG32(mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0); - WREG32(mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0); - WREG32(mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0); + pkt = cb->kernel_address; + + ctl = FIELD_PREP(GAUDI_PKT_LONG_CTL_OP_MASK, 0); /* write the value */ + ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_LONG); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); + + for (i = 0; i < num_regs ; i++, pkt++) { + pkt->ctl = cpu_to_le32(ctl); + pkt->value = cpu_to_le32(val); + pkt->addr = cpu_to_le64(reg_base + (i * 4)); } - i = GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT * 4; + hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT); - for (; i < NUM_OF_SOB_IN_BLOCK << 2 ; i += 4) - WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0); + pending_cb->cb = cb; + pending_cb->cb_size = cb_size; + /* The queue ID MUST be an external queue ID. Otherwise, we will + * have undefined behavior + */ + pending_cb->hw_queue_id = hw_queue_id; - i = GAUDI_FIRST_AVAILABLE_W_S_MONITOR * 4; + spin_lock(&ctx->pending_cb_lock); + list_add_tail(&pending_cb->cb_node, &ctx->pending_cb_list); + spin_unlock(&ctx->pending_cb_lock); - for (; i < NUM_OF_MONITORS_IN_BLOCK << 2 ; i += 4) - WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0); + return 0; +} + +static int gaudi_restore_sm_registers(struct hl_device *hdev) +{ + u64 base_addr; + u32 num_regs; + int rc; + + base_addr = CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0; + num_regs = NUM_OF_SOB_IN_BLOCK; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_SOB_OBJ_0; + num_regs = NUM_OF_SOB_IN_BLOCK; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0; + num_regs = NUM_OF_SOB_IN_BLOCK; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0; + num_regs = NUM_OF_MONITORS_IN_BLOCK; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_MON_STATUS_0; + num_regs = NUM_OF_MONITORS_IN_BLOCK; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_MON_STATUS_0; + num_regs = NUM_OF_MONITORS_IN_BLOCK; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + + (GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT * 4); + num_regs = NUM_OF_SOB_IN_BLOCK - GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0 + + (GAUDI_FIRST_AVAILABLE_W_S_MONITOR * 4); + num_regs = NUM_OF_MONITORS_IN_BLOCK - GAUDI_FIRST_AVAILABLE_W_S_MONITOR; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + return 0; } static void gaudi_restore_dma_registers(struct hl_device *hdev) @@ -5660,18 +5874,23 @@ static void gaudi_restore_qm_registers(struct hl_device *hdev) } } -static void gaudi_restore_user_registers(struct hl_device *hdev) +static int gaudi_restore_user_registers(struct hl_device *hdev) { - gaudi_restore_sm_registers(hdev); + int rc; + + rc = gaudi_restore_sm_registers(hdev); + if (rc) + return rc; + gaudi_restore_dma_registers(hdev); gaudi_restore_qm_registers(hdev); + + return 0; } static int gaudi_context_switch(struct hl_device *hdev, u32 asid) { - gaudi_restore_user_registers(hdev); - - return 0; + return gaudi_restore_user_registers(hdev); } static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev) @@ -5730,8 +5949,6 @@ static int gaudi_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val) } if (hbm_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *val = *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE); } else { rc = -EFAULT; } @@ -5777,8 +5994,6 @@ static int gaudi_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val) } if (hbm_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; } else { rc = -EFAULT; } @@ -5828,8 +6043,6 @@ static int gaudi_debugfs_read64(struct hl_device *hdev, u64 addr, u64 *val) } if (hbm_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *val = *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE); } else { rc = -EFAULT; } @@ -5878,8 +6091,6 @@ static int gaudi_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val) } if (hbm_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; } else { rc = -EFAULT; } @@ -5924,7 +6135,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) return; if (asid & ~DMA0_QM_GLBL_NON_SECURE_PROPS_0_ASID_MASK) { - WARN(1, "asid %u is too big\n", asid); + dev_crit(hdev->dev, "asid %u is too big\n", asid); return; } @@ -6227,7 +6438,7 @@ static int gaudi_send_job_on_qman0(struct hl_device *hdev, else timeout = HL_DEVICE_TIMEOUT_USEC; - if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) { + if (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) { dev_err_ratelimited(hdev->dev, "Can't send driver job on QMAN0 because the device is not idle\n"); return -EBUSY; @@ -6658,6 +6869,34 @@ static void gaudi_handle_qman_err_generic(struct hl_device *hdev, } } +static void gaudi_print_sm_sei_info(struct hl_device *hdev, u16 event_type, + struct hl_eq_sm_sei_data *sei_data) +{ + u32 index = event_type - GAUDI_EVENT_DMA_IF_SEI_0; + + switch (sei_data->sei_cause) { + case SM_SEI_SO_OVERFLOW: + dev_err(hdev->dev, + "SM %u SEI Error: SO %u overflow/underflow", + index, le32_to_cpu(sei_data->sei_log)); + break; + case SM_SEI_LBW_4B_UNALIGNED: + dev_err(hdev->dev, + "SM %u SEI Error: Unaligned 4B LBW access, monitor agent address low - %#x", + index, le32_to_cpu(sei_data->sei_log)); + break; + case SM_SEI_AXI_RESPONSE_ERR: + dev_err(hdev->dev, + "SM %u SEI Error: AXI ID %u response error", + index, le32_to_cpu(sei_data->sei_log)); + break; + default: + dev_err(hdev->dev, "Unknown SM SEI cause %u", + le32_to_cpu(sei_data->sei_log)); + break; + } +} + static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type, struct hl_eq_ecc_data *ecc_data) { @@ -6874,7 +7113,9 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, u32 base, val, val2, wr_par, rd_par, ca_par, derr, serr, type, ch; int err = 0; - if (!hdev->asic_prop.fw_security_disabled) { + if (hdev->asic_prop.fw_security_status_valid && + (hdev->asic_prop.fw_app_security_map & + CPU_BOOT_DEV_STS0_HBM_ECC_EN)) { if (!hbm_ecc_data) { dev_err(hdev->dev, "No FW ECC data"); return 0; @@ -6896,14 +7137,24 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); dev_err(hdev->dev, - "HBM%d pc%d ECC: TYPE=%d, WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n", - device, ch, type, wr_par, rd_par, ca_par, serr, derr); + "HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n", + device, ch, wr_par, rd_par, ca_par, serr, derr); + dev_err(hdev->dev, + "HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%u, SEC_CNT=%d, DEC_CNT=%d\n", + device, ch, hbm_ecc_data->first_addr, type, + hbm_ecc_data->sec_cont_cnt, hbm_ecc_data->sec_cnt, + hbm_ecc_data->dec_cnt); err = 1; return 0; } + if (!hdev->asic_prop.fw_security_disabled) { + dev_info(hdev->dev, "Cannot access MC regs for ECC data while security is enabled\n"); + return 0; + } + base = GAUDI_HBM_CFG_BASE + device * GAUDI_HBM_CFG_OFFSET; for (ch = 0 ; ch < GAUDI_HBM_CHANNELS ; ch++) { val = RREG32_MASK(base + ch * 0x1000 + 0x06C, 0x0000FFFF); @@ -7153,6 +7404,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, gaudi_hbm_read_interrupts(hdev, gaudi_hbm_event_to_dev(event_type), &eq_entry->hbm_ecc_data); + hl_fw_unmask_irq(hdev, event_type); break; case GAUDI_EVENT_TPC0_DEC: @@ -7281,6 +7533,13 @@ static void gaudi_handle_eqe(struct hl_device *hdev, hl_fw_unmask_irq(hdev, event_type); break; + case GAUDI_EVENT_DMA_IF_SEI_0 ... GAUDI_EVENT_DMA_IF_SEI_3: + gaudi_print_irq_info(hdev, event_type, false); + gaudi_print_sm_sei_info(hdev, event_type, + &eq_entry->sm_sei_data); + hl_fw_unmask_irq(hdev, event_type); + break; + case GAUDI_EVENT_FIX_POWER_ENV_S ... GAUDI_EVENT_FIX_THERMAL_ENV_E: gaudi_print_clk_change_info(hdev, event_type); hl_fw_unmask_irq(hdev, event_type); @@ -7330,8 +7589,6 @@ static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, else timeout_usec = MMU_CONFIG_TIMEOUT_USEC; - mutex_lock(&hdev->mmu_cache_lock); - /* L0 & L1 invalidation */ WREG32(mmSTLB_INV_PS, 3); WREG32(mmSTLB_CACHE_INV, gaudi->mmu_cache_inv_pi++); @@ -7347,8 +7604,6 @@ static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, WREG32(mmSTLB_INV_SET, 0); - mutex_unlock(&hdev->mmu_cache_lock); - if (rc) { dev_err_ratelimited(hdev->dev, "MMU cache invalidation timeout\n"); @@ -7371,8 +7626,6 @@ static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev, hdev->hard_reset_pending) return 0; - mutex_lock(&hdev->mmu_cache_lock); - if (hdev->pldm) timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC; else @@ -7400,8 +7653,6 @@ static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev, 1000, timeout_usec); - mutex_unlock(&hdev->mmu_cache_lock); - if (rc) { dev_err_ratelimited(hdev->dev, "MMU cache invalidation timeout\n"); @@ -7463,7 +7714,7 @@ static int gaudi_cpucp_info_get(struct hl_device *hdev) if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) return 0; - rc = hl_fw_cpucp_info_get(hdev, mmCPU_BOOT_DEV_STS0); + rc = hl_fw_cpucp_info_get(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0); if (rc) return rc; @@ -7483,13 +7734,14 @@ static int gaudi_cpucp_info_get(struct hl_device *hdev) return 0; } -static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask, - struct seq_file *s) +static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, + u8 mask_len, struct seq_file *s) { struct gaudi_device *gaudi = hdev->asic_specific; const char *fmt = "%-5d%-9s%#-14x%#-12x%#x\n"; const char *mme_slave_fmt = "%-5d%-9s%-14s%-12s%#x\n"; const char *nic_fmt = "%-5d%-9s%#-14x%#x\n"; + unsigned long *mask = (unsigned long *)mask_arr; u32 qm_glbl_sts0, qm_cgm_sts, dma_core_sts0, tpc_cfg_sts, mme_arch_sts; bool is_idle = true, is_eng_idle, is_slave; u64 offset; @@ -7515,9 +7767,8 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask, IS_DMA_IDLE(dma_core_sts0); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GAUDI_ENGINE_ID_DMA_0 + dma_id); + if (mask && !is_eng_idle) + set_bit(GAUDI_ENGINE_ID_DMA_0 + dma_id, mask); if (s) seq_printf(s, fmt, dma_id, is_eng_idle ? "Y" : "N", qm_glbl_sts0, @@ -7538,9 +7789,8 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask, IS_TPC_IDLE(tpc_cfg_sts); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GAUDI_ENGINE_ID_TPC_0 + i); + if (mask && !is_eng_idle) + set_bit(GAUDI_ENGINE_ID_TPC_0 + i, mask); if (s) seq_printf(s, fmt, i, is_eng_idle ? "Y" : "N", @@ -7567,9 +7817,8 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask, is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GAUDI_ENGINE_ID_MME_0 + i); + if (mask && !is_eng_idle) + set_bit(GAUDI_ENGINE_ID_MME_0 + i, mask); if (s) { if (!is_slave) seq_printf(s, fmt, i, @@ -7595,9 +7844,8 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask, is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GAUDI_ENGINE_ID_NIC_0 + port); + if (mask && !is_eng_idle) + set_bit(GAUDI_ENGINE_ID_NIC_0 + port, mask); if (s) seq_printf(s, nic_fmt, port, is_eng_idle ? "Y" : "N", @@ -7611,9 +7859,8 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask, is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GAUDI_ENGINE_ID_NIC_0 + port); + if (mask && !is_eng_idle) + set_bit(GAUDI_ENGINE_ID_NIC_0 + port, mask); if (s) seq_printf(s, nic_fmt, port, is_eng_idle ? "Y" : "N", @@ -7876,18 +8123,16 @@ static void gaudi_internal_cb_pool_fini(struct hl_device *hdev, static int gaudi_ctx_init(struct hl_ctx *ctx) { + if (ctx->asid == HL_KERNEL_ASID_ID) + return 0; + gaudi_mmu_prepare(ctx->hdev, ctx->asid); return gaudi_internal_cb_pool_init(ctx->hdev, ctx); } static void gaudi_ctx_fini(struct hl_ctx *ctx) { - struct hl_device *hdev = ctx->hdev; - - /* Gaudi will NEVER support more then a single compute context. - * Therefore, don't clear anything unless it is the compute context - */ - if (hdev->compute_ctx != ctx) + if (ctx->asid == HL_KERNEL_ASID_ID) return; gaudi_internal_cb_pool_fini(ctx->hdev, ctx); @@ -7928,10 +8173,10 @@ static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, ctl = FIELD_PREP(GAUDI_PKT_SHORT_CTL_ADDR_MASK, sob_id * 4); ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OP_MASK, 0); /* write the value */ ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_BASE_MASK, 3); /* W_S SOB base */ - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_EB_MASK, eb); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_RB_MASK, 1); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_MB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, eb); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); pkt->value = cpu_to_le32(value); pkt->ctl = cpu_to_le32(ctl); @@ -7948,10 +8193,10 @@ static u32 gaudi_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, ctl = FIELD_PREP(GAUDI_PKT_SHORT_CTL_ADDR_MASK, addr); ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_BASE_MASK, 2); /* W_S MON base */ - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_EB_MASK, 0); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_RB_MASK, 1); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_MB_MASK, 0); /* last pkt MB */ + ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 0); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 0); /* last pkt MB */ pkt->value = cpu_to_le32(value); pkt->ctl = cpu_to_le32(ctl); @@ -7997,10 +8242,10 @@ static u32 gaudi_add_arm_monitor_pkt(struct hl_device *hdev, ctl = FIELD_PREP(GAUDI_PKT_SHORT_CTL_ADDR_MASK, msg_addr_offset); ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OP_MASK, 0); /* write the value */ ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_BASE_MASK, 2); /* W_S MON base */ - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_EB_MASK, 0); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_RB_MASK, 1); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_MB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 0); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); pkt->value = cpu_to_le32(value); pkt->ctl = cpu_to_le32(ctl); @@ -8018,10 +8263,10 @@ static u32 gaudi_add_fence_pkt(struct packet_fence *pkt) cfg |= FIELD_PREP(GAUDI_PKT_FENCE_CFG_TARGET_VAL_MASK, 1); cfg |= FIELD_PREP(GAUDI_PKT_FENCE_CFG_ID_MASK, 2); - ctl = FIELD_PREP(GAUDI_PKT_FENCE_CTL_OPCODE_MASK, PACKET_FENCE); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_EB_MASK, 0); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_RB_MASK, 1); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_MB_MASK, 1); + ctl = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_FENCE); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 0); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); pkt->cfg = cpu_to_le32(cfg); pkt->ctl = cpu_to_le32(ctl); @@ -8217,12 +8462,16 @@ static u32 gaudi_gen_wait_cb(struct hl_device *hdev, static void gaudi_reset_sob(struct hl_device *hdev, void *data) { struct hl_hw_sob *hw_sob = (struct hl_hw_sob *) data; + int rc; dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id); - WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, - 0); + rc = gaudi_schedule_register_memset(hdev, hw_sob->q_idx, + CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + + hw_sob->sob_id * 4, 1, 0); + if (rc) + dev_err(hdev->dev, "failed resetting sob %u", hw_sob->sob_id); kref_init(&hw_sob->kref); } @@ -8246,6 +8495,24 @@ static u64 gaudi_get_device_time(struct hl_device *hdev) return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL); } +static int gaudi_get_hw_block_id(struct hl_device *hdev, u64 block_addr, + u32 *block_size, u32 *block_id) +{ + return -EPERM; +} + +static int gaudi_block_mmap(struct hl_device *hdev, + struct vm_area_struct *vma, + u32 block_id, u32 block_size) +{ + return -EPERM; +} + +static void gaudi_enable_events_from_fw(struct hl_device *hdev) +{ + WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_INTS_REGISTER); +} + static const struct hl_asic_funcs gaudi_funcs = { .early_init = gaudi_early_init, .early_fini = gaudi_early_fini, @@ -8322,7 +8589,13 @@ static const struct hl_asic_funcs gaudi_funcs = { .set_dma_mask_from_fw = gaudi_set_dma_mask_from_fw, .get_device_time = gaudi_get_device_time, .collective_wait_init_cs = gaudi_collective_wait_init_cs, - .collective_wait_create_jobs = gaudi_collective_wait_create_jobs + .collective_wait_create_jobs = gaudi_collective_wait_create_jobs, + .scramble_addr = hl_mmu_scramble_addr, + .descramble_addr = hl_mmu_descramble_addr, + .ack_protection_bits_errors = gaudi_ack_protection_bits_errors, + .get_hw_block_id = gaudi_get_hw_block_id, + .hw_block_mmap = gaudi_block_mmap, + .enable_events_from_fw = gaudi_enable_events_from_fw }; /** diff --git a/drivers/misc/habanalabs/gaudi/gaudiP.h b/drivers/misc/habanalabs/gaudi/gaudiP.h index a7ab2d7e57d4..50bb4ad570fd 100644 --- a/drivers/misc/habanalabs/gaudi/gaudiP.h +++ b/drivers/misc/habanalabs/gaudi/gaudiP.h @@ -251,11 +251,13 @@ enum gaudi_nic_mask { * @hdev: habanalabs device structure. * @kref: refcount of this SOB group. group will reset once refcount is zero. * @base_sob_id: base sob id of this SOB group. + * @queue_id: id of the queue that waits on this sob group */ struct gaudi_hw_sob_group { struct hl_device *hdev; struct kref kref; u32 base_sob_id; + u32 queue_id; }; #define NUM_SOB_GROUPS (HL_RSVD_SOBS * QMAN_STREAMS) @@ -333,6 +335,7 @@ struct gaudi_device { }; void gaudi_init_security(struct hl_device *hdev); +void gaudi_ack_protection_bits_errors(struct hl_device *hdev); void gaudi_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_attr_grp); void gaudi_set_pll_profile(struct hl_device *hdev, enum hl_pll_frequency freq); diff --git a/drivers/misc/habanalabs/gaudi/gaudi_coresight.c b/drivers/misc/habanalabs/gaudi/gaudi_coresight.c index 88a09d42e111..6e56fa1c6c69 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi_coresight.c +++ b/drivers/misc/habanalabs/gaudi/gaudi_coresight.c @@ -634,9 +634,21 @@ static int gaudi_config_etr(struct hl_device *hdev, WREG32(mmPSOC_ETR_BUFWM, 0x3FFC); WREG32(mmPSOC_ETR_RSZ, input->buffer_size); WREG32(mmPSOC_ETR_MODE, input->sink_mode); - /* Workaround for H3 #HW-2075 bug: use small data chunks */ - WREG32(mmPSOC_ETR_AXICTL, (is_host ? 0 : 0x700) | - PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT); + if (hdev->asic_prop.fw_security_disabled) { + /* make ETR not privileged */ + val = FIELD_PREP( + PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK, 0); + /* make ETR non-secured (inverted logic) */ + val |= FIELD_PREP( + PSOC_ETR_AXICTL_PROTCTRLBIT1_MASK, 1); + /* + * Workaround for H3 #HW-2075 bug: use small data + * chunks + */ + val |= FIELD_PREP(PSOC_ETR_AXICTL_WRBURSTLEN_MASK, + is_host ? 0 : 7); + WREG32(mmPSOC_ETR_AXICTL, val); + } WREG32(mmPSOC_ETR_DBALO, lower_32_bits(input->buffer_address)); WREG32(mmPSOC_ETR_DBAHI, diff --git a/drivers/misc/habanalabs/gaudi/gaudi_security.c b/drivers/misc/habanalabs/gaudi/gaudi_security.c index e10181692d0b..7085f45814ae 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi_security.c +++ b/drivers/misc/habanalabs/gaudi/gaudi_security.c @@ -13052,3 +13052,8 @@ void gaudi_init_security(struct hl_device *hdev) gaudi_init_protection_bits(hdev); } + +void gaudi_ack_protection_bits_errors(struct hl_device *hdev) +{ + +} diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 63679a747d2c..ed566c52ccaa 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -455,6 +455,11 @@ int goya_get_fixed_properties(struct hl_device *hdev) prop->max_pending_cs = GOYA_MAX_PENDING_CS; + prop->first_available_user_msix_interrupt = USHRT_MAX; + + for (i = 0 ; i < HL_MAX_DCORES ; i++) + prop->first_available_cq[i] = USHRT_MAX; + /* disable fw security for now, set it in a later stage */ prop->fw_security_disabled = true; prop->fw_security_status_valid = false; @@ -792,9 +797,6 @@ int goya_late_init(struct hl_device *hdev) return rc; } - WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, - GOYA_ASYNC_EVENT_ID_INTS_REGISTER); - return 0; } @@ -1186,6 +1188,7 @@ static int goya_stop_external_queues(struct hl_device *hdev) int goya_init_cpu_queues(struct hl_device *hdev) { struct goya_device *goya = hdev->asic_specific; + struct asic_fixed_properties *prop = &hdev->asic_prop; struct hl_eq *eq; u32 status; struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; @@ -1238,6 +1241,10 @@ int goya_init_cpu_queues(struct hl_device *hdev) return -EIO; } + /* update FW application security bits */ + if (prop->fw_security_status_valid) + prop->fw_app_security_map = RREG32(mmCPU_BOOT_DEV_STS0); + goya->hw_cap_initialized |= HW_CAP_CPU_Q; return 0; } @@ -2804,9 +2811,12 @@ void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) /* ring the doorbell */ WREG32(db_reg_offset, db_value); - if (hw_queue_id == GOYA_QUEUE_ID_CPU_PQ) + if (hw_queue_id == GOYA_QUEUE_ID_CPU_PQ) { + /* make sure device CPU will read latest data from host */ + mb(); WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GOYA_ASYNC_EVENT_ID_PI_UPDATE); + } } void goya_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) @@ -2914,7 +2924,7 @@ static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job) else timeout = HL_DEVICE_TIMEOUT_USEC; - if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) { + if (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) { dev_err_ratelimited(hdev->dev, "Can't send driver job on QMAN0 because the device is not idle\n"); return -EBUSY; @@ -3876,10 +3886,10 @@ static int goya_parse_cb_mmu(struct hl_device *hdev, patched_cb_handle >>= PAGE_SHIFT; parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) patched_cb_handle); - /* hl_cb_get should never fail here so use kernel WARN */ - WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n", - (u32) patched_cb_handle); + /* hl_cb_get should never fail here */ if (!parser->patched_cb) { + dev_crit(hdev->dev, "DMA CB handle invalid 0x%x\n", + (u32) patched_cb_handle); rc = -EFAULT; goto out; } @@ -3948,10 +3958,10 @@ static int goya_parse_cb_no_mmu(struct hl_device *hdev, patched_cb_handle >>= PAGE_SHIFT; parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) patched_cb_handle); - /* hl_cb_get should never fail here so use kernel WARN */ - WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n", - (u32) patched_cb_handle); + /* hl_cb_get should never fail here */ if (!parser->patched_cb) { + dev_crit(hdev->dev, "DMA CB handle invalid 0x%x\n", + (u32) patched_cb_handle); rc = -EFAULT; goto out; } @@ -4122,9 +4132,6 @@ static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val) if (ddr_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *val = *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE); - } else { rc = -EFAULT; } @@ -4178,9 +4185,6 @@ static int goya_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val) if (ddr_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; - } else { rc = -EFAULT; } @@ -4223,9 +4227,6 @@ static int goya_debugfs_read64(struct hl_device *hdev, u64 addr, u64 *val) if (ddr_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *val = *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE); - } else { rc = -EFAULT; } @@ -4266,9 +4267,6 @@ static int goya_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val) if (ddr_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; - } else { rc = -EFAULT; } @@ -4877,8 +4875,6 @@ int goya_context_switch(struct hl_device *hdev, u32 asid) WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020); - goya_mmu_prepare(hdev, asid); - goya_clear_sm_regs(hdev); return 0; @@ -5044,7 +5040,7 @@ static void goya_mmu_prepare(struct hl_device *hdev, u32 asid) return; if (asid & ~MME_QM_GLBL_SECURE_PROPS_ASID_MASK) { - WARN(1, "asid %u is too big\n", asid); + dev_crit(hdev->dev, "asid %u is too big\n", asid); return; } @@ -5073,8 +5069,6 @@ static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, else timeout_usec = MMU_CONFIG_TIMEOUT_USEC; - mutex_lock(&hdev->mmu_cache_lock); - /* L0 & L1 invalidation */ WREG32(mmSTLB_INV_ALL_START, 1); @@ -5086,8 +5080,6 @@ static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, 1000, timeout_usec); - mutex_unlock(&hdev->mmu_cache_lock); - if (rc) { dev_err_ratelimited(hdev->dev, "MMU cache invalidation timeout\n"); @@ -5117,8 +5109,6 @@ static int goya_mmu_invalidate_cache_range(struct hl_device *hdev, else timeout_usec = MMU_CONFIG_TIMEOUT_USEC; - mutex_lock(&hdev->mmu_cache_lock); - /* * TODO: currently invalidate entire L0 & L1 as in regular hard * invalidation. Need to apply invalidation of specific cache lines with @@ -5141,8 +5131,6 @@ static int goya_mmu_invalidate_cache_range(struct hl_device *hdev, 1000, timeout_usec); - mutex_unlock(&hdev->mmu_cache_lock); - if (rc) { dev_err_ratelimited(hdev->dev, "MMU cache invalidation timeout\n"); @@ -5172,7 +5160,7 @@ int goya_cpucp_info_get(struct hl_device *hdev) if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) return 0; - rc = hl_fw_cpucp_info_get(hdev, mmCPU_BOOT_DEV_STS0); + rc = hl_fw_cpucp_info_get(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0); if (rc) return rc; @@ -5207,11 +5195,12 @@ static void goya_disable_clock_gating(struct hl_device *hdev) /* clock gating not supported in Goya */ } -static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask, - struct seq_file *s) +static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask_arr, + u8 mask_len, struct seq_file *s) { const char *fmt = "%-5d%-9s%#-14x%#-16x%#x\n"; const char *dma_fmt = "%-5d%-9s%#-14x%#x\n"; + unsigned long *mask = (unsigned long *)mask_arr; u32 qm_glbl_sts0, cmdq_glbl_sts0, dma_core_sts0, tpc_cfg_sts, mme_arch_sts; bool is_idle = true, is_eng_idle; @@ -5231,9 +5220,8 @@ static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask, IS_DMA_IDLE(dma_core_sts0); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GOYA_ENGINE_ID_DMA_0 + i); + if (mask && !is_eng_idle) + set_bit(GOYA_ENGINE_ID_DMA_0 + i, mask); if (s) seq_printf(s, dma_fmt, i, is_eng_idle ? "Y" : "N", qm_glbl_sts0, dma_core_sts0); @@ -5255,9 +5243,8 @@ static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask, IS_TPC_IDLE(tpc_cfg_sts); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GOYA_ENGINE_ID_TPC_0 + i); + if (mask && !is_eng_idle) + set_bit(GOYA_ENGINE_ID_TPC_0 + i, mask); if (s) seq_printf(s, fmt, i, is_eng_idle ? "Y" : "N", qm_glbl_sts0, cmdq_glbl_sts0, tpc_cfg_sts); @@ -5276,8 +5263,8 @@ static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask, IS_MME_IDLE(mme_arch_sts); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << GOYA_ENGINE_ID_MME_0; + if (mask && !is_eng_idle) + set_bit(GOYA_ENGINE_ID_MME_0, mask); if (s) { seq_printf(s, fmt, 0, is_eng_idle ? "Y" : "N", qm_glbl_sts0, cmdq_glbl_sts0, mme_arch_sts); @@ -5321,6 +5308,9 @@ static int goya_get_eeprom_data(struct hl_device *hdev, void *data, static int goya_ctx_init(struct hl_ctx *ctx) { + if (ctx->asid != HL_KERNEL_ASID_ID) + goya_mmu_prepare(ctx->hdev, ctx->asid); + return 0; } @@ -5399,6 +5389,24 @@ static void goya_ctx_fini(struct hl_ctx *ctx) } +static int goya_get_hw_block_id(struct hl_device *hdev, u64 block_addr, + u32 *block_size, u32 *block_id) +{ + return -EPERM; +} + +static int goya_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, + u32 block_id, u32 block_size) +{ + return -EPERM; +} + +static void goya_enable_events_from_fw(struct hl_device *hdev) +{ + WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, + GOYA_ASYNC_EVENT_ID_INTS_REGISTER); +} + static const struct hl_asic_funcs goya_funcs = { .early_init = goya_early_init, .early_fini = goya_early_fini, @@ -5475,7 +5483,13 @@ static const struct hl_asic_funcs goya_funcs = { .set_dma_mask_from_fw = goya_set_dma_mask_from_fw, .get_device_time = goya_get_device_time, .collective_wait_init_cs = goya_collective_wait_init_cs, - .collective_wait_create_jobs = goya_collective_wait_create_jobs + .collective_wait_create_jobs = goya_collective_wait_create_jobs, + .scramble_addr = hl_mmu_scramble_addr, + .descramble_addr = hl_mmu_descramble_addr, + .ack_protection_bits_errors = goya_ack_protection_bits_errors, + .get_hw_block_id = goya_get_hw_block_id, + .hw_block_mmap = goya_block_mmap, + .enable_events_from_fw = goya_enable_events_from_fw }; /* diff --git a/drivers/misc/habanalabs/goya/goyaP.h b/drivers/misc/habanalabs/goya/goyaP.h index 8b3408211af6..23fe099ed218 100644 --- a/drivers/misc/habanalabs/goya/goyaP.h +++ b/drivers/misc/habanalabs/goya/goyaP.h @@ -173,6 +173,7 @@ void goya_init_mme_qmans(struct hl_device *hdev); void goya_init_tpc_qmans(struct hl_device *hdev); int goya_init_cpu_queues(struct hl_device *hdev); void goya_init_security(struct hl_device *hdev); +void goya_ack_protection_bits_errors(struct hl_device *hdev); int goya_late_init(struct hl_device *hdev); void goya_late_fini(struct hl_device *hdev); diff --git a/drivers/misc/habanalabs/goya/goya_coresight.c b/drivers/misc/habanalabs/goya/goya_coresight.c index 6fa03933b438..6b7445cca580 100644 --- a/drivers/misc/habanalabs/goya/goya_coresight.c +++ b/drivers/misc/habanalabs/goya/goya_coresight.c @@ -434,8 +434,15 @@ static int goya_config_etr(struct hl_device *hdev, WREG32(mmPSOC_ETR_BUFWM, 0x3FFC); WREG32(mmPSOC_ETR_RSZ, input->buffer_size); WREG32(mmPSOC_ETR_MODE, input->sink_mode); - WREG32(mmPSOC_ETR_AXICTL, - 0x700 | PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT); + if (hdev->asic_prop.fw_security_disabled) { + /* make ETR not privileged */ + val = FIELD_PREP(PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK, 0); + /* make ETR non-secured (inverted logic) */ + val |= FIELD_PREP(PSOC_ETR_AXICTL_PROTCTRLBIT1_MASK, 1); + /* burst size 8 */ + val |= FIELD_PREP(PSOC_ETR_AXICTL_WRBURSTLEN_MASK, 7); + WREG32(mmPSOC_ETR_AXICTL, val); + } WREG32(mmPSOC_ETR_DBALO, lower_32_bits(input->buffer_address)); WREG32(mmPSOC_ETR_DBAHI, diff --git a/drivers/misc/habanalabs/goya/goya_security.c b/drivers/misc/habanalabs/goya/goya_security.c index 14701836f92b..14c3bae3ccdc 100644 --- a/drivers/misc/habanalabs/goya/goya_security.c +++ b/drivers/misc/habanalabs/goya/goya_security.c @@ -3120,3 +3120,8 @@ void goya_init_security(struct hl_device *hdev) goya_init_protection_bits(hdev); } + +void goya_ack_protection_bits_errors(struct hl_device *hdev) +{ + +} diff --git a/drivers/misc/habanalabs/include/common/cpucp_if.h b/drivers/misc/habanalabs/include/common/cpucp_if.h index 00bd9b392f93..b77c1c16c32c 100644 --- a/drivers/misc/habanalabs/include/common/cpucp_if.h +++ b/drivers/misc/habanalabs/include/common/cpucp_if.h @@ -58,11 +58,25 @@ struct hl_eq_ecc_data { __u8 pad[7]; }; +enum hl_sm_sei_cause { + SM_SEI_SO_OVERFLOW, + SM_SEI_LBW_4B_UNALIGNED, + SM_SEI_AXI_RESPONSE_ERR +}; + +struct hl_eq_sm_sei_data { + __le32 sei_log; + /* enum hl_sm_sei_cause */ + __u8 sei_cause; + __u8 pad[3]; +}; + struct hl_eq_entry { struct hl_eq_header hdr; union { struct hl_eq_ecc_data ecc_data; struct hl_eq_hbm_ecc_data hbm_ecc_data; + struct hl_eq_sm_sei_data sm_sei_data; __le64 data[7]; }; }; diff --git a/drivers/misc/habanalabs/include/common/hl_boot_if.h b/drivers/misc/habanalabs/include/common/hl_boot_if.h index b637dfd69f6e..e87f5a98e193 100644 --- a/drivers/misc/habanalabs/include/common/hl_boot_if.h +++ b/drivers/misc/habanalabs/include/common/hl_boot_if.h @@ -70,6 +70,9 @@ * checksum. Trying to program image again * might solve this. * + * CPU_BOOT_ERR0_PLL_FAIL PLL settings failed, meaning that one + * of the PLLs remains in REF_CLK + * * CPU_BOOT_ERR0_ENABLED Error registers enabled. * This is a main indication that the * running FW populates the error @@ -88,6 +91,7 @@ #define CPU_BOOT_ERR0_EFUSE_FAIL (1 << 9) #define CPU_BOOT_ERR0_PRI_IMG_VER_FAIL (1 << 10) #define CPU_BOOT_ERR0_SEC_IMG_VER_FAIL (1 << 11) +#define CPU_BOOT_ERR0_PLL_FAIL (1 << 12) #define CPU_BOOT_ERR0_ENABLED (1 << 31) /* @@ -150,10 +154,22 @@ * CPU_BOOT_DEV_STS0_PLL_INFO_EN FW retrieval of PLL info is enabled. * Initialized in: linux * + * CPU_BOOT_DEV_STS0_SP_SRAM_EN SP SRAM is initialized and available + * for use. + * Initialized in: preboot + * * CPU_BOOT_DEV_STS0_CLK_GATE_EN Clock Gating enabled. * FW initialized Clock Gating. * Initialized in: preboot * + * CPU_BOOT_DEV_STS0_HBM_ECC_EN HBM ECC handling Enabled. + * FW handles HBM ECC indications. + * Initialized in: linux + * + * CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN Packets ack value used in the armcpd + * is set to the PI counter. + * Initialized in: linux + * * CPU_BOOT_DEV_STS0_ENABLED Device status register enabled. * This is a main indication that the * running FW populates the device status @@ -175,7 +191,10 @@ #define CPU_BOOT_DEV_STS0_DRAM_SCR_EN (1 << 9) #define CPU_BOOT_DEV_STS0_FW_HARD_RST_EN (1 << 10) #define CPU_BOOT_DEV_STS0_PLL_INFO_EN (1 << 11) +#define CPU_BOOT_DEV_STS0_SP_SRAM_EN (1 << 12) #define CPU_BOOT_DEV_STS0_CLK_GATE_EN (1 << 13) +#define CPU_BOOT_DEV_STS0_HBM_ECC_EN (1 << 14) +#define CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN (1 << 15) #define CPU_BOOT_DEV_STS0_ENABLED (1 << 31) enum cpu_boot_status { diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h b/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h index 9ccba8437ec9..49335e8334b4 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h @@ -212,6 +212,10 @@ enum gaudi_async_event_id { GAUDI_EVENT_NIC_SEI_2 = 266, GAUDI_EVENT_NIC_SEI_3 = 267, GAUDI_EVENT_NIC_SEI_4 = 268, + GAUDI_EVENT_DMA_IF_SEI_0 = 277, + GAUDI_EVENT_DMA_IF_SEI_1 = 278, + GAUDI_EVENT_DMA_IF_SEI_2 = 279, + GAUDI_EVENT_DMA_IF_SEI_3 = 280, GAUDI_EVENT_PCIE_FLR = 290, GAUDI_EVENT_TPC0_BMON_SPMU = 300, GAUDI_EVENT_TPC0_KRN_ERR = 301, diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h b/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h index b9b90d079e23..b53aeda9a982 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h @@ -388,7 +388,10 @@ enum axi_id { #define RAZWI_INITIATOR_ID_X_Y_TPC6 RAZWI_INITIATOR_ID_X_Y(7, 6) #define RAZWI_INITIATOR_ID_X_Y_TPC7_NIC4_NIC5 RAZWI_INITIATOR_ID_X_Y(8, 6) -#define PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT 1 +#define PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT 1 +#define PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK 0x1 +#define PSOC_ETR_AXICTL_PROTCTRLBIT1_MASK 0x2 +#define PSOC_ETR_AXICTL_WRBURSTLEN_MASK 0xF00 /* STLB_CACHE_INV */ #define STLB_CACHE_INV_PRODUCER_INDEX_SHIFT 0 diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h b/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h index f30f2c0458d7..6e097ace2e96 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h @@ -78,6 +78,9 @@ struct packet_wreg_bulk { __le64 values[0]; /* data starts here */ }; +#define GAUDI_PKT_LONG_CTL_OP_SHIFT 20 +#define GAUDI_PKT_LONG_CTL_OP_MASK 0x00300000 + struct packet_msg_long { __le32 value; __le32 ctl; @@ -111,18 +114,6 @@ struct packet_msg_long { #define GAUDI_PKT_SHORT_CTL_BASE_SHIFT 22 #define GAUDI_PKT_SHORT_CTL_BASE_MASK 0x00C00000 -#define GAUDI_PKT_SHORT_CTL_OPCODE_SHIFT 24 -#define GAUDI_PKT_SHORT_CTL_OPCODE_MASK 0x1F000000 - -#define GAUDI_PKT_SHORT_CTL_EB_SHIFT 29 -#define GAUDI_PKT_SHORT_CTL_EB_MASK 0x20000000 - -#define GAUDI_PKT_SHORT_CTL_RB_SHIFT 30 -#define GAUDI_PKT_SHORT_CTL_RB_MASK 0x40000000 - -#define GAUDI_PKT_SHORT_CTL_MB_SHIFT 31 -#define GAUDI_PKT_SHORT_CTL_MB_MASK 0x80000000 - struct packet_msg_short { __le32 value; __le32 ctl; @@ -146,18 +137,6 @@ struct packet_msg_prot { #define GAUDI_PKT_FENCE_CTL_PRED_SHIFT 0 #define GAUDI_PKT_FENCE_CTL_PRED_MASK 0x0000001F -#define GAUDI_PKT_FENCE_CTL_OPCODE_SHIFT 24 -#define GAUDI_PKT_FENCE_CTL_OPCODE_MASK 0x1F000000 - -#define GAUDI_PKT_FENCE_CTL_EB_SHIFT 29 -#define GAUDI_PKT_FENCE_CTL_EB_MASK 0x20000000 - -#define GAUDI_PKT_FENCE_CTL_RB_SHIFT 30 -#define GAUDI_PKT_FENCE_CTL_RB_MASK 0x40000000 - -#define GAUDI_PKT_FENCE_CTL_MB_SHIFT 31 -#define GAUDI_PKT_FENCE_CTL_MB_MASK 0x80000000 - struct packet_fence { __le32 cfg; __le32 ctl; diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h index 067489bd048e..9ff3cb245580 100644 --- a/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h +++ b/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h @@ -259,6 +259,9 @@ #define DMA_QM_3_GLBL_CFG1_DMA_STOP_SHIFT DMA_QM_0_GLBL_CFG1_DMA_STOP_SHIFT #define DMA_QM_4_GLBL_CFG1_DMA_STOP_SHIFT DMA_QM_0_GLBL_CFG1_DMA_STOP_SHIFT -#define PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT 1 +#define PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT 1 +#define PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK 0x1 +#define PSOC_ETR_AXICTL_PROTCTRLBIT1_MASK 0x2 +#define PSOC_ETR_AXICTL_WRBURSTLEN_MASK 0xF00 #endif /* ASIC_REG_GOYA_MASKS_H_ */ diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c index 2907db260fba..935acc6bbf3c 100644 --- a/drivers/misc/mei/bus.c +++ b/drivers/misc/mei/bus.c @@ -44,7 +44,8 @@ ssize_t __mei_cl_send(struct mei_cl *cl, u8 *buf, size_t length, u8 vtag, bus = cl->dev; mutex_lock(&bus->device_lock); - if (bus->dev_state != MEI_DEV_ENABLED) { + if (bus->dev_state != MEI_DEV_ENABLED && + bus->dev_state != MEI_DEV_POWERING_DOWN) { rets = -ENODEV; goto out; } @@ -60,6 +61,13 @@ ssize_t __mei_cl_send(struct mei_cl *cl, u8 *buf, size_t length, u8 vtag, goto out; } + if (vtag) { + /* Check if vtag is supported by client */ + rets = mei_cl_vt_support_check(cl); + if (rets) + goto out; + } + if (length > mei_cl_mtu(cl)) { rets = -EFBIG; goto out; @@ -128,7 +136,8 @@ ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length, u8 *vtag, bus = cl->dev; mutex_lock(&bus->device_lock); - if (bus->dev_state != MEI_DEV_ENABLED) { + if (bus->dev_state != MEI_DEV_ENABLED && + bus->dev_state != MEI_DEV_POWERING_DOWN) { rets = -ENODEV; goto out; } @@ -878,22 +887,17 @@ static int mei_cl_device_probe(struct device *dev) static int mei_cl_device_remove(struct device *dev) { struct mei_cl_device *cldev = to_mei_cl_device(dev); - struct mei_cl_driver *cldrv; - int ret = 0; + struct mei_cl_driver *cldrv = to_mei_cl_driver(dev->driver); - if (!cldev || !dev->driver) - return 0; - - cldrv = to_mei_cl_driver(dev->driver); if (cldrv->remove) - ret = cldrv->remove(cldev); + cldrv->remove(cldev); mei_cldev_unregister_callbacks(cldev); mei_cl_bus_module_put(cldev); module_put(THIS_MODULE); - return ret; + return 0; } static ssize_t name_show(struct device *dev, struct device_attribute *a, diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c index a56d41321f32..4378a9b25848 100644 --- a/drivers/misc/mei/client.c +++ b/drivers/misc/mei/client.c @@ -9,6 +9,7 @@ #include <linux/delay.h> #include <linux/slab.h> #include <linux/pm_runtime.h> +#include <linux/dma-mapping.h> #include <linux/mei.h> @@ -990,7 +991,8 @@ int mei_cl_disconnect(struct mei_cl *cl) return 0; } - if (dev->dev_state == MEI_DEV_POWER_DOWN) { + if (dev->dev_state == MEI_DEV_POWERING_DOWN || + dev->dev_state == MEI_DEV_POWER_DOWN) { cl_dbg(dev, cl, "Device is powering down, don't bother with disconnection\n"); mei_cl_set_disconnected(cl); return 0; @@ -1737,7 +1739,7 @@ static inline u8 mei_ext_hdr_set_vtag(struct mei_ext_hdr *ext, u8 vtag) * * @cb: message callback structure * - * Return: a pointer to initialized header + * Return: a pointer to initialized header or ERR_PTR on failure */ static struct mei_msg_hdr *mei_msg_hdr_init(const struct mei_cl_cb *cb) { @@ -2113,6 +2115,8 @@ void mei_cl_complete(struct mei_cl *cl, struct mei_cl_cb *cb) case MEI_FOP_DISCONNECT: case MEI_FOP_NOTIFY_STOP: case MEI_FOP_NOTIFY_START: + case MEI_FOP_DMA_MAP: + case MEI_FOP_DMA_UNMAP: if (waitqueue_active(&cl->wait)) wake_up(&cl->wait); @@ -2139,3 +2143,286 @@ void mei_cl_all_disconnect(struct mei_device *dev) list_for_each_entry(cl, &dev->file_list, link) mei_cl_set_disconnected(cl); } + +static struct mei_cl *mei_cl_dma_map_find(struct mei_device *dev, u8 buffer_id) +{ + struct mei_cl *cl; + + list_for_each_entry(cl, &dev->file_list, link) + if (cl->dma.buffer_id == buffer_id) + return cl; + return NULL; +} + +/** + * mei_cl_irq_dma_map - send client dma map request in irq_thread context + * + * @cl: client + * @cb: callback block. + * @cmpl_list: complete list. + * + * Return: 0 on such and error otherwise. + */ +int mei_cl_irq_dma_map(struct mei_cl *cl, struct mei_cl_cb *cb, + struct list_head *cmpl_list) +{ + struct mei_device *dev = cl->dev; + u32 msg_slots; + int slots; + int ret; + + msg_slots = mei_hbm2slots(sizeof(struct hbm_client_dma_map_request)); + slots = mei_hbuf_empty_slots(dev); + if (slots < 0) + return -EOVERFLOW; + + if ((u32)slots < msg_slots) + return -EMSGSIZE; + + ret = mei_hbm_cl_dma_map_req(dev, cl); + if (ret) { + cl->status = ret; + list_move_tail(&cb->list, cmpl_list); + return ret; + } + + list_move_tail(&cb->list, &dev->ctrl_rd_list); + return 0; +} + +/** + * mei_cl_irq_dma_unmap - send client dma unmap request in irq_thread context + * + * @cl: client + * @cb: callback block. + * @cmpl_list: complete list. + * + * Return: 0 on such and error otherwise. + */ +int mei_cl_irq_dma_unmap(struct mei_cl *cl, struct mei_cl_cb *cb, + struct list_head *cmpl_list) +{ + struct mei_device *dev = cl->dev; + u32 msg_slots; + int slots; + int ret; + + msg_slots = mei_hbm2slots(sizeof(struct hbm_client_dma_unmap_request)); + slots = mei_hbuf_empty_slots(dev); + if (slots < 0) + return -EOVERFLOW; + + if ((u32)slots < msg_slots) + return -EMSGSIZE; + + ret = mei_hbm_cl_dma_unmap_req(dev, cl); + if (ret) { + cl->status = ret; + list_move_tail(&cb->list, cmpl_list); + return ret; + } + + list_move_tail(&cb->list, &dev->ctrl_rd_list); + return 0; +} + +static int mei_cl_dma_alloc(struct mei_cl *cl, u8 buf_id, size_t size) +{ + cl->dma.vaddr = dmam_alloc_coherent(cl->dev->dev, size, + &cl->dma.daddr, GFP_KERNEL); + if (!cl->dma.vaddr) + return -ENOMEM; + + cl->dma.buffer_id = buf_id; + cl->dma.size = size; + + return 0; +} + +static void mei_cl_dma_free(struct mei_cl *cl) +{ + cl->dma.buffer_id = 0; + dmam_free_coherent(cl->dev->dev, + cl->dma.size, cl->dma.vaddr, cl->dma.daddr); + cl->dma.size = 0; + cl->dma.vaddr = NULL; + cl->dma.daddr = 0; +} + +/** + * mei_cl_alloc_and_map - send client dma map request + * + * @cl: host client + * @fp: pointer to file structure + * @buffer_id: id of the mapped buffer + * @size: size of the buffer + * + * Locking: called under "dev->device_lock" lock + * + * Return: + * * -ENODEV + * * -EINVAL + * * -EOPNOTSUPP + * * -EPROTO + * * -ENOMEM; + */ +int mei_cl_dma_alloc_and_map(struct mei_cl *cl, const struct file *fp, + u8 buffer_id, size_t size) +{ + struct mei_device *dev; + struct mei_cl_cb *cb; + int rets; + + if (WARN_ON(!cl || !cl->dev)) + return -ENODEV; + + dev = cl->dev; + + if (!dev->hbm_f_cd_supported) { + cl_dbg(dev, cl, "client dma is not supported\n"); + return -EOPNOTSUPP; + } + + if (buffer_id == 0) + return -EINVAL; + + if (!mei_cl_is_connected(cl)) + return -ENODEV; + + if (cl->dma_mapped) + return -EPROTO; + + if (mei_cl_dma_map_find(dev, buffer_id)) { + cl_dbg(dev, cl, "client dma with id %d is already allocated\n", + cl->dma.buffer_id); + return -EPROTO; + } + + rets = pm_runtime_get(dev->dev); + if (rets < 0 && rets != -EINPROGRESS) { + pm_runtime_put_noidle(dev->dev); + cl_err(dev, cl, "rpm: get failed %d\n", rets); + return rets; + } + + rets = mei_cl_dma_alloc(cl, buffer_id, size); + if (rets) { + pm_runtime_put_noidle(dev->dev); + return rets; + } + + cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, MEI_FOP_DMA_MAP, fp); + if (!cb) { + rets = -ENOMEM; + goto out; + } + + if (mei_hbuf_acquire(dev)) { + if (mei_hbm_cl_dma_map_req(dev, cl)) { + rets = -ENODEV; + goto out; + } + list_move_tail(&cb->list, &dev->ctrl_rd_list); + } + + mutex_unlock(&dev->device_lock); + wait_event_timeout(cl->wait, + cl->dma_mapped || + cl->status || + !mei_cl_is_connected(cl), + mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT)); + mutex_lock(&dev->device_lock); + + if (!cl->dma_mapped && !cl->status) + cl->status = -EFAULT; + + rets = cl->status; + +out: + if (rets) + mei_cl_dma_free(cl); + + cl_dbg(dev, cl, "rpm: autosuspend\n"); + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); + + mei_io_cb_free(cb); + return rets; +} + +/** + * mei_cl_unmap_and_free - send client dma unmap request + * + * @cl: host client + * @fp: pointer to file structure + * + * Locking: called under "dev->device_lock" lock + * + * Return: 0 on such and error otherwise. + */ +int mei_cl_dma_unmap(struct mei_cl *cl, const struct file *fp) +{ + struct mei_device *dev; + struct mei_cl_cb *cb; + int rets; + + if (WARN_ON(!cl || !cl->dev)) + return -ENODEV; + + dev = cl->dev; + + if (!dev->hbm_f_cd_supported) { + cl_dbg(dev, cl, "client dma is not supported\n"); + return -EOPNOTSUPP; + } + + if (!mei_cl_is_connected(cl)) + return -ENODEV; + + if (!cl->dma_mapped) + return -EPROTO; + + rets = pm_runtime_get(dev->dev); + if (rets < 0 && rets != -EINPROGRESS) { + pm_runtime_put_noidle(dev->dev); + cl_err(dev, cl, "rpm: get failed %d\n", rets); + return rets; + } + + cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, MEI_FOP_DMA_UNMAP, fp); + if (!cb) { + rets = -ENOMEM; + goto out; + } + + if (mei_hbuf_acquire(dev)) { + if (mei_hbm_cl_dma_unmap_req(dev, cl)) { + rets = -ENODEV; + goto out; + } + list_move_tail(&cb->list, &dev->ctrl_rd_list); + } + + mutex_unlock(&dev->device_lock); + wait_event_timeout(cl->wait, + !cl->dma_mapped || + cl->status || + !mei_cl_is_connected(cl), + mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT)); + mutex_lock(&dev->device_lock); + + if (cl->dma_mapped && !cl->status) + cl->status = -EFAULT; + + rets = cl->status; + + if (!rets) + mei_cl_dma_free(cl); +out: + cl_dbg(dev, cl, "rpm: autosuspend\n"); + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); + + mei_io_cb_free(cb); + return rets; +} diff --git a/drivers/misc/mei/client.h b/drivers/misc/mei/client.h index 9e08a9843bba..b12cdcde9436 100644 --- a/drivers/misc/mei/client.h +++ b/drivers/misc/mei/client.h @@ -265,6 +265,14 @@ void mei_cl_notify(struct mei_cl *cl); void mei_cl_all_disconnect(struct mei_device *dev); +int mei_cl_irq_dma_map(struct mei_cl *cl, struct mei_cl_cb *cb, + struct list_head *cmpl_list); +int mei_cl_irq_dma_unmap(struct mei_cl *cl, struct mei_cl_cb *cb, + struct list_head *cmpl_list); +int mei_cl_dma_alloc_and_map(struct mei_cl *cl, const struct file *fp, + u8 buffer_id, size_t size); +int mei_cl_dma_unmap(struct mei_cl *cl, const struct file *fp); + #define MEI_CL_FMT "cl:host=%02d me=%02d " #define MEI_CL_PRM(cl) (cl)->host_client_id, mei_cl_me_id(cl) diff --git a/drivers/misc/mei/debugfs.c b/drivers/misc/mei/debugfs.c index 3ab1a431d810..1ce61e9e24fc 100644 --- a/drivers/misc/mei/debugfs.c +++ b/drivers/misc/mei/debugfs.c @@ -106,6 +106,7 @@ static int mei_dbgfs_devstate_show(struct seq_file *m, void *unused) seq_printf(m, "\tDR: %01d\n", dev->hbm_f_dr_supported); seq_printf(m, "\tVT: %01d\n", dev->hbm_f_vt_supported); seq_printf(m, "\tCAP: %01d\n", dev->hbm_f_cap_supported); + seq_printf(m, "\tCD: %01d\n", dev->hbm_f_cd_supported); } seq_printf(m, "pg: %s, %s\n", diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c index 686e8b6a4c55..d0277c7fed10 100644 --- a/drivers/misc/mei/hbm.c +++ b/drivers/misc/mei/hbm.c @@ -339,7 +339,9 @@ static int mei_hbm_capabilities_req(struct mei_device *dev) memset(&req, 0, sizeof(req)); req.hbm_cmd = MEI_HBM_CAPABILITIES_REQ_CMD; if (dev->hbm_f_vt_supported) - req.capability_requested[0] = HBM_CAP_VT; + req.capability_requested[0] |= HBM_CAP_VT; + if (dev->hbm_f_cd_supported) + req.capability_requested[0] |= HBM_CAP_CD; ret = mei_hbm_write_message(dev, &mei_hdr, &req); if (ret) { @@ -593,6 +595,117 @@ static void mei_hbm_cl_notify(struct mei_device *dev, } /** + * mei_hbm_cl_dma_map_req - send client dma map request + * + * @dev: the device structure + * @cl: mei host client + * + * Return: 0 on success and -EIO on write failure + */ +int mei_hbm_cl_dma_map_req(struct mei_device *dev, struct mei_cl *cl) +{ + struct mei_msg_hdr mei_hdr; + struct hbm_client_dma_map_request req; + int ret; + + mei_hbm_hdr(&mei_hdr, sizeof(req)); + + memset(&req, 0, sizeof(req)); + + req.hbm_cmd = MEI_HBM_CLIENT_DMA_MAP_REQ_CMD; + req.client_buffer_id = cl->dma.buffer_id; + req.address_lsb = lower_32_bits(cl->dma.daddr); + req.address_msb = upper_32_bits(cl->dma.daddr); + req.size = cl->dma.size; + + ret = mei_hbm_write_message(dev, &mei_hdr, &req); + if (ret) + dev_err(dev->dev, "dma map request failed: ret = %d\n", ret); + + return ret; +} + +/** + * mei_hbm_cl_dma_unmap_req - send client dma unmap request + * + * @dev: the device structure + * @cl: mei host client + * + * Return: 0 on success and -EIO on write failure + */ +int mei_hbm_cl_dma_unmap_req(struct mei_device *dev, struct mei_cl *cl) +{ + struct mei_msg_hdr mei_hdr; + struct hbm_client_dma_unmap_request req; + int ret; + + mei_hbm_hdr(&mei_hdr, sizeof(req)); + + memset(&req, 0, sizeof(req)); + + req.hbm_cmd = MEI_HBM_CLIENT_DMA_UNMAP_REQ_CMD; + req.client_buffer_id = cl->dma.buffer_id; + + ret = mei_hbm_write_message(dev, &mei_hdr, &req); + if (ret) + dev_err(dev->dev, "dma unmap request failed: ret = %d\n", ret); + + return ret; +} + +static void mei_hbm_cl_dma_map_res(struct mei_device *dev, + struct hbm_client_dma_response *res) +{ + struct mei_cl *cl; + struct mei_cl_cb *cb, *next; + + cl = NULL; + list_for_each_entry_safe(cb, next, &dev->ctrl_rd_list, list) { + if (cb->fop_type != MEI_FOP_DMA_MAP) + continue; + if (!cb->cl->dma.buffer_id || cb->cl->dma_mapped) + continue; + + cl = cb->cl; + break; + } + if (!cl) + return; + + dev_dbg(dev->dev, "cl dma map result = %d\n", res->status); + cl->status = res->status; + if (!cl->status) + cl->dma_mapped = 1; + wake_up(&cl->wait); +} + +static void mei_hbm_cl_dma_unmap_res(struct mei_device *dev, + struct hbm_client_dma_response *res) +{ + struct mei_cl *cl; + struct mei_cl_cb *cb, *next; + + cl = NULL; + list_for_each_entry_safe(cb, next, &dev->ctrl_rd_list, list) { + if (cb->fop_type != MEI_FOP_DMA_UNMAP) + continue; + if (!cb->cl->dma.buffer_id || !cb->cl->dma_mapped) + continue; + + cl = cb->cl; + break; + } + if (!cl) + return; + + dev_dbg(dev->dev, "cl dma unmap result = %d\n", res->status); + cl->status = res->status; + if (!cl->status) + cl->dma_mapped = 0; + wake_up(&cl->wait); +} + +/** * mei_hbm_prop_req - request property for a single client * * @dev: the device structure @@ -1085,6 +1198,13 @@ static void mei_hbm_config_features(struct mei_device *dev) (dev->version.major_version == HBM_MAJOR_VERSION_CAP && dev->version.minor_version >= HBM_MINOR_VERSION_CAP)) dev->hbm_f_cap_supported = 1; + + /* Client DMA Support */ + dev->hbm_f_cd_supported = 0; + if (dev->version.major_version > HBM_MAJOR_VERSION_CD || + (dev->version.major_version == HBM_MAJOR_VERSION_CD && + dev->version.minor_version >= HBM_MINOR_VERSION_CD)) + dev->hbm_f_cd_supported = 1; } /** @@ -1124,6 +1244,7 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) struct mei_hbm_cl_cmd *cl_cmd; struct hbm_client_connect_request *disconnect_req; struct hbm_flow_control *fctrl; + struct hbm_client_dma_response *client_dma_res; /* read the message to our buffer */ BUG_ON(hdr->length >= sizeof(dev->rd_msg_buf)); @@ -1177,6 +1298,10 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) if (dev->dev_state != MEI_DEV_INIT_CLIENTS || dev->hbm_state != MEI_HBM_STARTING) { + if (dev->dev_state == MEI_DEV_POWER_DOWN) { + dev_dbg(dev->dev, "hbm: start: on shutdown, ignoring\n"); + return 0; + } dev_err(dev->dev, "hbm: start: state mismatch, [%d, %d]\n", dev->dev_state, dev->hbm_state); return -EPROTO; @@ -1215,7 +1340,12 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) dev->init_clients_timer = 0; - if (dev->hbm_state != MEI_HBM_CAP_SETUP) { + if (dev->dev_state != MEI_DEV_INIT_CLIENTS || + dev->hbm_state != MEI_HBM_CAP_SETUP) { + if (dev->dev_state == MEI_DEV_POWER_DOWN) { + dev_dbg(dev->dev, "hbm: capabilities response: on shutdown, ignoring\n"); + return 0; + } dev_err(dev->dev, "hbm: capabilities response: state mismatch, [%d, %d]\n", dev->dev_state, dev->hbm_state); return -EPROTO; @@ -1224,6 +1354,8 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) capability_res = (struct hbm_capability_response *)mei_msg; if (!(capability_res->capability_granted[0] & HBM_CAP_VT)) dev->hbm_f_vt_supported = 0; + if (!(capability_res->capability_granted[0] & HBM_CAP_CD)) + dev->hbm_f_cd_supported = 0; if (dev->hbm_f_dr_supported) { if (mei_dmam_ring_alloc(dev)) @@ -1247,7 +1379,12 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) dev->init_clients_timer = 0; - if (dev->hbm_state != MEI_HBM_DR_SETUP) { + if (dev->dev_state != MEI_DEV_INIT_CLIENTS || + dev->hbm_state != MEI_HBM_DR_SETUP) { + if (dev->dev_state == MEI_DEV_POWER_DOWN) { + dev_dbg(dev->dev, "hbm: dma setup response: on shutdown, ignoring\n"); + return 0; + } dev_err(dev->dev, "hbm: dma setup response: state mismatch, [%d, %d]\n", dev->dev_state, dev->hbm_state); return -EPROTO; @@ -1311,6 +1448,10 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) if (dev->dev_state != MEI_DEV_INIT_CLIENTS || dev->hbm_state != MEI_HBM_CLIENT_PROPERTIES) { + if (dev->dev_state == MEI_DEV_POWER_DOWN) { + dev_dbg(dev->dev, "hbm: properties response: on shutdown, ignoring\n"); + return 0; + } dev_err(dev->dev, "hbm: properties response: state mismatch, [%d, %d]\n", dev->dev_state, dev->hbm_state); return -EPROTO; @@ -1349,6 +1490,10 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) if (dev->dev_state != MEI_DEV_INIT_CLIENTS || dev->hbm_state != MEI_HBM_ENUM_CLIENTS) { + if (dev->dev_state == MEI_DEV_POWER_DOWN) { + dev_dbg(dev->dev, "hbm: enumeration response: on shutdown, ignoring\n"); + return 0; + } dev_err(dev->dev, "hbm: enumeration response: state mismatch, [%d, %d]\n", dev->dev_state, dev->hbm_state); return -EPROTO; @@ -1373,7 +1518,7 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) return -EPROTO; } - dev->dev_state = MEI_DEV_POWER_DOWN; + mei_set_devstate(dev, MEI_DEV_POWER_DOWN); dev_info(dev->dev, "hbm: stop response: resetting.\n"); /* force the reset */ return -EPROTO; @@ -1426,6 +1571,18 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) mei_hbm_cl_notify(dev, cl_cmd); break; + case MEI_HBM_CLIENT_DMA_MAP_RES_CMD: + dev_dbg(dev->dev, "hbm: client dma map response: message received.\n"); + client_dma_res = (struct hbm_client_dma_response *)mei_msg; + mei_hbm_cl_dma_map_res(dev, client_dma_res); + break; + + case MEI_HBM_CLIENT_DMA_UNMAP_RES_CMD: + dev_dbg(dev->dev, "hbm: client dma unmap response: message received.\n"); + client_dma_res = (struct hbm_client_dma_response *)mei_msg; + mei_hbm_cl_dma_unmap_res(dev, client_dma_res); + break; + default: WARN(1, "hbm: wrong command %d\n", mei_msg->hbm_cmd); return -EPROTO; diff --git a/drivers/misc/mei/hbm.h b/drivers/misc/mei/hbm.h index 4d95e38e4ddf..cd5b08ca34b6 100644 --- a/drivers/misc/mei/hbm.h +++ b/drivers/misc/mei/hbm.h @@ -10,6 +10,7 @@ struct mei_device; struct mei_msg_hdr; struct mei_cl; +struct mei_dma_data; /** * enum mei_hbm_state - host bus message protocol state @@ -51,6 +52,7 @@ int mei_hbm_pg(struct mei_device *dev, u8 pg_cmd); void mei_hbm_pg_resume(struct mei_device *dev); int mei_hbm_cl_notify_req(struct mei_device *dev, struct mei_cl *cl, u8 request); - +int mei_hbm_cl_dma_map_req(struct mei_device *dev, struct mei_cl *cl); +int mei_hbm_cl_dma_unmap_req(struct mei_device *dev, struct mei_cl *cl); #endif /* _MEI_HBM_H_ */ diff --git a/drivers/misc/mei/hdcp/mei_hdcp.c b/drivers/misc/mei/hdcp/mei_hdcp.c index 3506a3534294..ec2a4fce8581 100644 --- a/drivers/misc/mei/hdcp/mei_hdcp.c +++ b/drivers/misc/mei/hdcp/mei_hdcp.c @@ -844,16 +844,19 @@ enable_err_exit: return ret; } -static int mei_hdcp_remove(struct mei_cl_device *cldev) +static void mei_hdcp_remove(struct mei_cl_device *cldev) { struct i915_hdcp_comp_master *comp_master = mei_cldev_get_drvdata(cldev); + int ret; component_master_del(&cldev->dev, &mei_component_master_ops); kfree(comp_master); mei_cldev_set_drvdata(cldev, NULL); - return mei_cldev_disable(cldev); + ret = mei_cldev_disable(cldev); + if (ret) + dev_warn(&cldev->dev, "mei_cldev_disable() failed\n"); } #define MEI_UUID_HDCP GUID_INIT(0xB638AB7E, 0x94E2, 0x4EA2, 0xA5, \ diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h index 9cf8d8f60cfe..14be76d4c2e6 100644 --- a/drivers/misc/mei/hw-me-regs.h +++ b/drivers/misc/mei/hw-me-regs.h @@ -101,6 +101,11 @@ #define MEI_DEV_ID_MCC 0x4B70 /* Mule Creek Canyon (EHL) */ #define MEI_DEV_ID_MCC_4 0x4B75 /* Mule Creek Canyon 4 (EHL) */ +#define MEI_DEV_ID_EBG 0x1BE0 /* Emmitsburg WS */ + +#define MEI_DEV_ID_ADP_S 0x7AE8 /* Alder Lake Point S */ +#define MEI_DEV_ID_ADP_LP 0x7A60 /* Alder Lake Point LP */ + /* * MEI HW Section */ diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h index df2fb9520dd8..b10606550613 100644 --- a/drivers/misc/mei/hw.h +++ b/drivers/misc/mei/hw.h @@ -88,6 +88,12 @@ #define HBM_MINOR_VERSION_CAP 2 #define HBM_MAJOR_VERSION_CAP 2 +/* + * MEI version with client DMA support + */ +#define HBM_MINOR_VERSION_CD 2 +#define HBM_MAJOR_VERSION_CD 2 + /* Host bus message command opcode */ #define MEI_HBM_CMD_OP_MSK 0x7f /* Host bus message command RESPONSE */ @@ -136,6 +142,12 @@ #define MEI_HBM_CAPABILITIES_REQ_CMD 0x13 #define MEI_HBM_CAPABILITIES_RES_CMD 0x93 +#define MEI_HBM_CLIENT_DMA_MAP_REQ_CMD 0x14 +#define MEI_HBM_CLIENT_DMA_MAP_RES_CMD 0x94 + +#define MEI_HBM_CLIENT_DMA_UNMAP_REQ_CMD 0x15 +#define MEI_HBM_CLIENT_DMA_UNMAP_RES_CMD 0x95 + /* * MEI Stop Reason * used by hbm_host_stop_request.reason @@ -648,6 +660,8 @@ struct hbm_dma_ring_ctrl { /* virtual tag supported */ #define HBM_CAP_VT BIT(0) +/* client dma supported */ +#define HBM_CAP_CD BIT(2) /** * struct hbm_capability_request - capability request from host to fw @@ -671,4 +685,51 @@ struct hbm_capability_response { u8 capability_granted[3]; } __packed; +/** + * struct hbm_client_dma_map_request - client dma map request from host to fw + * + * @hbm_cmd: bus message command header + * @client_buffer_id: client buffer id + * @reserved: reserved + * @address_lsb: DMA address LSB + * @address_msb: DMA address MSB + * @size: DMA size + */ +struct hbm_client_dma_map_request { + u8 hbm_cmd; + u8 client_buffer_id; + u8 reserved[2]; + u32 address_lsb; + u32 address_msb; + u32 size; +} __packed; + +/** + * struct hbm_client_dma_unmap_request + * client dma unmap request from the host to the firmware + * + * @hbm_cmd: bus message command header + * @status: unmap status + * @client_buffer_id: client buffer id + * @reserved: reserved + */ +struct hbm_client_dma_unmap_request { + u8 hbm_cmd; + u8 status; + u8 client_buffer_id; + u8 reserved; +} __packed; + +/** + * struct hbm_client_dma_response + * client dma unmap response from the firmware to the host + * + * @hbm_cmd: bus message command header + * @status: command status + */ +struct hbm_client_dma_response { + u8 hbm_cmd; + u8 status; +} __packed; + #endif diff --git a/drivers/misc/mei/init.c b/drivers/misc/mei/init.c index bcee77768b91..5c8cb679b997 100644 --- a/drivers/misc/mei/init.c +++ b/drivers/misc/mei/init.c @@ -303,9 +303,12 @@ void mei_stop(struct mei_device *dev) dev_dbg(dev->dev, "stopping the device.\n"); mutex_lock(&dev->device_lock); - mei_set_devstate(dev, MEI_DEV_POWER_DOWN); + mei_set_devstate(dev, MEI_DEV_POWERING_DOWN); mutex_unlock(&dev->device_lock); mei_cl_bus_remove_devices(dev); + mutex_lock(&dev->device_lock); + mei_set_devstate(dev, MEI_DEV_POWER_DOWN); + mutex_unlock(&dev->device_lock); mei_cancel_work(dev); diff --git a/drivers/misc/mei/interrupt.c b/drivers/misc/mei/interrupt.c index 326955b04fda..a98f6b895af7 100644 --- a/drivers/misc/mei/interrupt.c +++ b/drivers/misc/mei/interrupt.c @@ -295,12 +295,17 @@ static inline bool hdr_is_fixed(struct mei_msg_hdr *mei_hdr) static inline int hdr_is_valid(u32 msg_hdr) { struct mei_msg_hdr *mei_hdr; + u32 expected_len = 0; mei_hdr = (struct mei_msg_hdr *)&msg_hdr; if (!msg_hdr || mei_hdr->reserved) return -EBADMSG; - if (mei_hdr->dma_ring && mei_hdr->length != MEI_SLOT_SIZE) + if (mei_hdr->dma_ring) + expected_len += MEI_SLOT_SIZE; + if (mei_hdr->extended) + expected_len += MEI_SLOT_SIZE; + if (mei_hdr->length < expected_len) return -EBADMSG; return 0; @@ -324,6 +329,8 @@ int mei_irq_read_handler(struct mei_device *dev, struct mei_cl *cl; int ret; u32 ext_meta_hdr_u32; + u32 hdr_size_left; + u32 hdr_size_ext; int i; int ext_hdr_end; @@ -353,6 +360,7 @@ int mei_irq_read_handler(struct mei_device *dev, } ext_hdr_end = 1; + hdr_size_left = mei_hdr->length; if (mei_hdr->extended) { if (!dev->rd_msg_hdr[1]) { @@ -363,8 +371,21 @@ int mei_irq_read_handler(struct mei_device *dev, dev_dbg(dev->dev, "extended header is %08x\n", ext_meta_hdr_u32); } - meta_hdr = ((struct mei_ext_meta_hdr *) - dev->rd_msg_hdr + 1); + meta_hdr = ((struct mei_ext_meta_hdr *)dev->rd_msg_hdr + 1); + if (check_add_overflow((u32)sizeof(*meta_hdr), + mei_slots2data(meta_hdr->size), + &hdr_size_ext)) { + dev_err(dev->dev, "extended message size too big %d\n", + meta_hdr->size); + return -EBADMSG; + } + if (hdr_size_left < hdr_size_ext) { + dev_err(dev->dev, "corrupted message header len %d\n", + mei_hdr->length); + return -EBADMSG; + } + hdr_size_left -= hdr_size_ext; + ext_hdr_end = meta_hdr->size + 2; for (i = dev->rd_msg_hdr_count; i < ext_hdr_end; i++) { dev->rd_msg_hdr[i] = mei_read_hdr(dev); @@ -376,6 +397,12 @@ int mei_irq_read_handler(struct mei_device *dev, } if (mei_hdr->dma_ring) { + if (hdr_size_left != sizeof(dev->rd_msg_hdr[ext_hdr_end])) { + dev_err(dev->dev, "corrupted message header len %d\n", + mei_hdr->length); + return -EBADMSG; + } + dev->rd_msg_hdr[ext_hdr_end] = mei_read_hdr(dev); dev->rd_msg_hdr_count++; (*slots)--; @@ -520,6 +547,16 @@ int mei_irq_write_handler(struct mei_device *dev, struct list_head *cmpl_list) if (ret) return ret; break; + case MEI_FOP_DMA_MAP: + ret = mei_cl_irq_dma_map(cl, cb, cmpl_list); + if (ret) + return ret; + break; + case MEI_FOP_DMA_UNMAP: + ret = mei_cl_irq_dma_unmap(cl, cb, cmpl_list); + if (ret) + return ret; + break; default: BUG(); } diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c index 9f6682033ed7..28937b6e7e0c 100644 --- a/drivers/misc/mei/main.c +++ b/drivers/misc/mei/main.c @@ -1026,7 +1026,7 @@ static ssize_t tx_queue_limit_show(struct device *device, size = dev->tx_queue_limit; mutex_unlock(&dev->device_lock); - return snprintf(buf, PAGE_SIZE, "%u\n", size); + return sysfs_emit(buf, "%u\n", size); } static ssize_t tx_queue_limit_store(struct device *device, diff --git a/drivers/misc/mei/mei_dev.h b/drivers/misc/mei/mei_dev.h index 8c395bfdf6f3..b7b6ef344e80 100644 --- a/drivers/misc/mei/mei_dev.h +++ b/drivers/misc/mei/mei_dev.h @@ -57,6 +57,7 @@ enum mei_dev_state { MEI_DEV_ENABLED, MEI_DEV_RESETTING, MEI_DEV_DISABLED, + MEI_DEV_POWERING_DOWN, MEI_DEV_POWER_DOWN, MEI_DEV_POWER_UP }; @@ -78,6 +79,8 @@ enum mei_file_transaction_states { * @MEI_FOP_DISCONNECT_RSP: disconnect response * @MEI_FOP_NOTIFY_START: start notification * @MEI_FOP_NOTIFY_STOP: stop notification + * @MEI_FOP_DMA_MAP: request client dma map + * @MEI_FOP_DMA_UNMAP: request client dma unmap */ enum mei_cb_file_ops { MEI_FOP_READ = 0, @@ -87,6 +90,8 @@ enum mei_cb_file_ops { MEI_FOP_DISCONNECT_RSP, MEI_FOP_NOTIFY_START, MEI_FOP_NOTIFY_STOP, + MEI_FOP_DMA_MAP, + MEI_FOP_DMA_UNMAP, }; /** @@ -112,6 +117,13 @@ struct mei_msg_data { unsigned char *data; }; +struct mei_dma_data { + u8 buffer_id; + void *vaddr; + dma_addr_t daddr; + size_t size; +}; + /** * struct mei_dma_dscr - dma address descriptor * @@ -235,6 +247,8 @@ struct mei_cl_vtag { * @rd_pending: pending read credits * @rd_completed_lock: protects rd_completed queue * @rd_completed: completed read + * @dma: dma settings + * @dma_mapped: dma buffer is currently mapped. * * @cldev: device on the mei client bus */ @@ -262,6 +276,8 @@ struct mei_cl { struct list_head rd_pending; spinlock_t rd_completed_lock; /* protects rd_completed queue */ struct list_head rd_completed; + struct mei_dma_data dma; + u8 dma_mapped; struct mei_cl_device *cldev; }; @@ -450,6 +466,7 @@ struct mei_fw_version { * @hbm_f_dr_supported : hbm feature dma ring supported * @hbm_f_vt_supported : hbm feature vtag supported * @hbm_f_cap_supported : hbm feature capabilities message supported + * @hbm_f_cd_supported : hbm feature client dma supported * * @fw_ver : FW versions * @@ -537,6 +554,7 @@ struct mei_device { unsigned int hbm_f_dr_supported:1; unsigned int hbm_f_vt_supported:1; unsigned int hbm_f_cap_supported:1; + unsigned int hbm_f_cd_supported:1; struct mei_fw_version fw_ver[MEI_MAX_FW_VER_BLOCKS]; diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c index 1de9ef7a272b..a7e179626b63 100644 --- a/drivers/misc/mei/pci-me.c +++ b/drivers/misc/mei/pci-me.c @@ -107,6 +107,11 @@ static const struct pci_device_id mei_me_pci_tbl[] = { {MEI_PCI_DEVICE(MEI_DEV_ID_CDF, MEI_ME_PCH8_CFG)}, + {MEI_PCI_DEVICE(MEI_DEV_ID_EBG, MEI_ME_PCH15_SPS_CFG)}, + + {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_S, MEI_ME_PCH15_CFG)}, + {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_LP, MEI_ME_PCH15_CFG)}, + /* required last entry */ {0, } }; diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c index eff481ce08ee..1b2868ca4f2a 100644 --- a/drivers/misc/pci_endpoint_test.c +++ b/drivers/misc/pci_endpoint_test.c @@ -68,7 +68,6 @@ #define PCI_ENDPOINT_TEST_FLAGS 0x2c #define FLAG_USE_DMA BIT(0) -#define PCI_DEVICE_ID_TI_J721E 0xb00d #define PCI_DEVICE_ID_TI_AM654 0xb00c #define PCI_DEVICE_ID_LS1088A 0x80c0 diff --git a/drivers/misc/pti.c b/drivers/misc/pti.c deleted file mode 100644 index 7236ae527b19..000000000000 --- a/drivers/misc/pti.c +++ /dev/null @@ -1,978 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * pti.c - PTI driver for cJTAG data extration - * - * Copyright (C) Intel 2010 - * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * The PTI (Parallel Trace Interface) driver directs trace data routed from - * various parts in the system out through the Intel Penwell PTI port and - * out of the mobile device for analysis with a debugging tool - * (Lauterbach, Fido). This is part of a solution for the MIPI P1149.7, - * compact JTAG, standard. - */ - -#include <linux/init.h> -#include <linux/sched.h> -#include <linux/interrupt.h> -#include <linux/console.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/tty.h> -#include <linux/tty_driver.h> -#include <linux/pci.h> -#include <linux/mutex.h> -#include <linux/miscdevice.h> -#include <linux/intel-pti.h> -#include <linux/slab.h> -#include <linux/uaccess.h> - -#define DRIVERNAME "pti" -#define PCINAME "pciPTI" -#define TTYNAME "ttyPTI" -#define CHARNAME "pti" -#define PTITTY_MINOR_START 0 -#define PTITTY_MINOR_NUM 2 -#define MAX_APP_IDS 16 /* 128 channel ids / u8 bit size */ -#define MAX_OS_IDS 16 /* 128 channel ids / u8 bit size */ -#define MAX_MODEM_IDS 16 /* 128 channel ids / u8 bit size */ -#define MODEM_BASE_ID 71 /* modem master ID address */ -#define CONTROL_ID 72 /* control master ID address */ -#define CONSOLE_ID 73 /* console master ID address */ -#define OS_BASE_ID 74 /* base OS master ID address */ -#define APP_BASE_ID 80 /* base App master ID address */ -#define CONTROL_FRAME_LEN 32 /* PTI control frame maximum size */ -#define USER_COPY_SIZE 8192 /* 8Kb buffer for user space copy */ -#define APERTURE_14 0x3800000 /* offset to first OS write addr */ -#define APERTURE_LEN 0x400000 /* address length */ - -struct pti_tty { - struct pti_masterchannel *mc; -}; - -struct pti_dev { - struct tty_port port[PTITTY_MINOR_NUM]; - unsigned long pti_addr; - unsigned long aperture_base; - void __iomem *pti_ioaddr; - u8 ia_app[MAX_APP_IDS]; - u8 ia_os[MAX_OS_IDS]; - u8 ia_modem[MAX_MODEM_IDS]; -}; - -/* - * This protects access to ia_app, ia_os, and ia_modem, - * which keeps track of channels allocated in - * an aperture write id. - */ -static DEFINE_MUTEX(alloclock); - -static const struct pci_device_id pci_ids[] = { - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x82B)}, - {0} -}; - -static struct tty_driver *pti_tty_driver; -static struct pti_dev *drv_data; - -static unsigned int pti_console_channel; -static unsigned int pti_control_channel; - -/** - * pti_write_to_aperture()- The private write function to PTI HW. - * - * @mc: The 'aperture'. It's part of a write address that holds - * a master and channel ID. - * @buf: Data being written to the HW that will ultimately be seen - * in a debugging tool (Fido, Lauterbach). - * @len: Size of buffer. - * - * Since each aperture is specified by a unique - * master/channel ID, no two processes will be writing - * to the same aperture at the same time so no lock is required. The - * PTI-Output agent will send these out in the order that they arrived, and - * thus, it will intermix these messages. The debug tool can then later - * regroup the appropriate message segments together reconstituting each - * message. - */ -static void pti_write_to_aperture(struct pti_masterchannel *mc, - u8 *buf, - int len) -{ - int dwordcnt; - int final; - int i; - u32 ptiword; - u32 __iomem *aperture; - u8 *p = buf; - - /* - * calculate the aperture offset from the base using the master and - * channel id's. - */ - aperture = drv_data->pti_ioaddr + (mc->master << 15) - + (mc->channel << 8); - - dwordcnt = len >> 2; - final = len - (dwordcnt << 2); /* final = trailing bytes */ - if (final == 0 && dwordcnt != 0) { /* always need a final dword */ - final += 4; - dwordcnt--; - } - - for (i = 0; i < dwordcnt; i++) { - ptiword = be32_to_cpu(*(u32 *)p); - p += 4; - iowrite32(ptiword, aperture); - } - - aperture += PTI_LASTDWORD_DTS; /* adding DTS signals that is EOM */ - - ptiword = 0; - for (i = 0; i < final; i++) - ptiword |= *p++ << (24-(8*i)); - - iowrite32(ptiword, aperture); - return; -} - -/** - * pti_control_frame_built_and_sent()- control frame build and send function. - * - * @mc: The master / channel structure on which the function - * built a control frame. - * @thread_name: The thread name associated with the master / channel or - * 'NULL' if using the 'current' global variable. - * - * To be able to post process the PTI contents on host side, a control frame - * is added before sending any PTI content. So the host side knows on - * each PTI frame the name of the thread using a dedicated master / channel. - * The thread name is retrieved from 'current' global variable if 'thread_name' - * is 'NULL', else it is retrieved from 'thread_name' parameter. - * This function builds this frame and sends it to a master ID CONTROL_ID. - * The overhead is only 32 bytes since the driver only writes to HW - * in 32 byte chunks. - */ -static void pti_control_frame_built_and_sent(struct pti_masterchannel *mc, - const char *thread_name) -{ - /* - * Since we access the comm member in current's task_struct, we only - * need to be as large as what 'comm' in that structure is. - */ - char comm[TASK_COMM_LEN]; - struct pti_masterchannel mccontrol = {.master = CONTROL_ID, - .channel = 0}; - const char *thread_name_p; - const char *control_format = "%3d %3d %s"; - u8 control_frame[CONTROL_FRAME_LEN]; - - if (!thread_name) { - if (!in_interrupt()) - get_task_comm(comm, current); - else - strncpy(comm, "Interrupt", TASK_COMM_LEN); - - /* Absolutely ensure our buffer is zero terminated. */ - comm[TASK_COMM_LEN-1] = 0; - thread_name_p = comm; - } else { - thread_name_p = thread_name; - } - - mccontrol.channel = pti_control_channel; - pti_control_channel = (pti_control_channel + 1) & 0x7f; - - snprintf(control_frame, CONTROL_FRAME_LEN, control_format, mc->master, - mc->channel, thread_name_p); - pti_write_to_aperture(&mccontrol, control_frame, strlen(control_frame)); -} - -/** - * pti_write_full_frame_to_aperture()- high level function to - * write to PTI. - * - * @mc: The 'aperture'. It's part of a write address that holds - * a master and channel ID. - * @buf: Data being written to the HW that will ultimately be seen - * in a debugging tool (Fido, Lauterbach). - * @len: Size of buffer. - * - * All threads sending data (either console, user space application, ...) - * are calling the high level function to write to PTI meaning that it is - * possible to add a control frame before sending the content. - */ -static void pti_write_full_frame_to_aperture(struct pti_masterchannel *mc, - const unsigned char *buf, - int len) -{ - pti_control_frame_built_and_sent(mc, NULL); - pti_write_to_aperture(mc, (u8 *)buf, len); -} - -/** - * get_id()- Allocate a master and channel ID. - * - * @id_array: an array of bits representing what channel - * id's are allocated for writing. - * @max_ids: The max amount of available write IDs to use. - * @base_id: The starting SW channel ID, based on the Intel - * PTI arch. - * @thread_name: The thread name associated with the master / channel or - * 'NULL' if using the 'current' global variable. - * - * Returns: - * pti_masterchannel struct with master, channel ID address - * 0 for error - * - * Each bit in the arrays ia_app and ia_os correspond to a master and - * channel id. The bit is one if the id is taken and 0 if free. For - * every master there are 128 channel id's. - */ -static struct pti_masterchannel *get_id(u8 *id_array, - int max_ids, - int base_id, - const char *thread_name) -{ - struct pti_masterchannel *mc; - int i, j, mask; - - mc = kmalloc(sizeof(struct pti_masterchannel), GFP_KERNEL); - if (mc == NULL) - return NULL; - - /* look for a byte with a free bit */ - for (i = 0; i < max_ids; i++) - if (id_array[i] != 0xff) - break; - if (i == max_ids) { - kfree(mc); - return NULL; - } - /* find the bit in the 128 possible channel opportunities */ - mask = 0x80; - for (j = 0; j < 8; j++) { - if ((id_array[i] & mask) == 0) - break; - mask >>= 1; - } - - /* grab it */ - id_array[i] |= mask; - mc->master = base_id; - mc->channel = ((i & 0xf)<<3) + j; - /* write new master Id / channel Id allocation to channel control */ - pti_control_frame_built_and_sent(mc, thread_name); - return mc; -} - -/* - * The following three functions: - * pti_request_mastercahannel(), mipi_release_masterchannel() - * and pti_writedata() are an API for other kernel drivers to - * access PTI. - */ - -/** - * pti_request_masterchannel()- Kernel API function used to allocate - * a master, channel ID address - * to write to PTI HW. - * - * @type: 0- request Application master, channel aperture ID - * write address. - * 1- request OS master, channel aperture ID write - * address. - * 2- request Modem master, channel aperture ID - * write address. - * Other values, error. - * @thread_name: The thread name associated with the master / channel or - * 'NULL' if using the 'current' global variable. - * - * Returns: - * pti_masterchannel struct - * 0 for error - */ -struct pti_masterchannel *pti_request_masterchannel(u8 type, - const char *thread_name) -{ - struct pti_masterchannel *mc; - - mutex_lock(&alloclock); - - switch (type) { - - case 0: - mc = get_id(drv_data->ia_app, MAX_APP_IDS, - APP_BASE_ID, thread_name); - break; - - case 1: - mc = get_id(drv_data->ia_os, MAX_OS_IDS, - OS_BASE_ID, thread_name); - break; - - case 2: - mc = get_id(drv_data->ia_modem, MAX_MODEM_IDS, - MODEM_BASE_ID, thread_name); - break; - default: - mc = NULL; - } - - mutex_unlock(&alloclock); - return mc; -} -EXPORT_SYMBOL_GPL(pti_request_masterchannel); - -/** - * pti_release_masterchannel()- Kernel API function used to release - * a master, channel ID address - * used to write to PTI HW. - * - * @mc: master, channel apeture ID address to be released. This - * will de-allocate the structure via kfree(). - */ -void pti_release_masterchannel(struct pti_masterchannel *mc) -{ - u8 master, channel, i; - - mutex_lock(&alloclock); - - if (mc) { - master = mc->master; - channel = mc->channel; - - if (master == APP_BASE_ID) { - i = channel >> 3; - drv_data->ia_app[i] &= ~(0x80>>(channel & 0x7)); - } else if (master == OS_BASE_ID) { - i = channel >> 3; - drv_data->ia_os[i] &= ~(0x80>>(channel & 0x7)); - } else { - i = channel >> 3; - drv_data->ia_modem[i] &= ~(0x80>>(channel & 0x7)); - } - - kfree(mc); - } - - mutex_unlock(&alloclock); -} -EXPORT_SYMBOL_GPL(pti_release_masterchannel); - -/** - * pti_writedata()- Kernel API function used to write trace - * debugging data to PTI HW. - * - * @mc: Master, channel aperture ID address to write to. - * Null value will return with no write occurring. - * @buf: Trace debuging data to write to the PTI HW. - * Null value will return with no write occurring. - * @count: Size of buf. Value of 0 or a negative number will - * return with no write occuring. - */ -void pti_writedata(struct pti_masterchannel *mc, u8 *buf, int count) -{ - /* - * since this function is exported, this is treated like an - * API function, thus, all parameters should - * be checked for validity. - */ - if ((mc != NULL) && (buf != NULL) && (count > 0)) - pti_write_to_aperture(mc, buf, count); - return; -} -EXPORT_SYMBOL_GPL(pti_writedata); - -/* - * for the tty_driver_*() basic function descriptions, see tty_driver.h. - * Specific header comments made for PTI-related specifics. - */ - -/** - * pti_tty_driver_open()- Open an Application master, channel aperture - * ID to the PTI device via tty device. - * - * @tty: tty interface. - * @filp: filp interface pased to tty_port_open() call. - * - * Returns: - * int, 0 for success - * otherwise, fail value - * - * The main purpose of using the tty device interface is for - * each tty port to have a unique PTI write aperture. In an - * example use case, ttyPTI0 gets syslogd and an APP aperture - * ID and ttyPTI1 is where the n_tracesink ldisc hooks to route - * modem messages into PTI. Modem trace data does not have to - * go to ttyPTI1, but ttyPTI0 and ttyPTI1 do need to be distinct - * master IDs. These messages go through the PTI HW and out of - * the handheld platform and to the Fido/Lauterbach device. - */ -static int pti_tty_driver_open(struct tty_struct *tty, struct file *filp) -{ - /* - * we actually want to allocate a new channel per open, per - * system arch. HW gives more than plenty channels for a single - * system task to have its own channel to write trace data. This - * also removes a locking requirement for the actual write - * procedure. - */ - return tty_port_open(tty->port, tty, filp); -} - -/** - * pti_tty_driver_close()- close tty device and release Application - * master, channel aperture ID to the PTI device via tty device. - * - * @tty: tty interface. - * @filp: filp interface pased to tty_port_close() call. - * - * The main purpose of using the tty device interface is to route - * syslog daemon messages to the PTI HW and out of the handheld platform - * and to the Fido/Lauterbach device. - */ -static void pti_tty_driver_close(struct tty_struct *tty, struct file *filp) -{ - tty_port_close(tty->port, tty, filp); -} - -/** - * pti_tty_install()- Used to set up specific master-channels - * to tty ports for organizational purposes when - * tracing viewed from debuging tools. - * - * @driver: tty driver information. - * @tty: tty struct containing pti information. - * - * Returns: - * 0 for success - * otherwise, error - */ -static int pti_tty_install(struct tty_driver *driver, struct tty_struct *tty) -{ - int idx = tty->index; - struct pti_tty *pti_tty_data; - int ret = tty_standard_install(driver, tty); - - if (ret == 0) { - pti_tty_data = kmalloc(sizeof(struct pti_tty), GFP_KERNEL); - if (pti_tty_data == NULL) - return -ENOMEM; - - if (idx == PTITTY_MINOR_START) - pti_tty_data->mc = pti_request_masterchannel(0, NULL); - else - pti_tty_data->mc = pti_request_masterchannel(2, NULL); - - if (pti_tty_data->mc == NULL) { - kfree(pti_tty_data); - return -ENXIO; - } - tty->driver_data = pti_tty_data; - } - - return ret; -} - -/** - * pti_tty_cleanup()- Used to de-allocate master-channel resources - * tied to tty's of this driver. - * - * @tty: tty struct containing pti information. - */ -static void pti_tty_cleanup(struct tty_struct *tty) -{ - struct pti_tty *pti_tty_data = tty->driver_data; - if (pti_tty_data == NULL) - return; - pti_release_masterchannel(pti_tty_data->mc); - kfree(pti_tty_data); - tty->driver_data = NULL; -} - -/** - * pti_tty_driver_write()- Write trace debugging data through the char - * interface to the PTI HW. Part of the misc device implementation. - * - * @tty: tty struct containing pti information. - * @buf: trace data to be written. - * @len: # of byte to write. - * - * Returns: - * int, # of bytes written - * otherwise, error - */ -static int pti_tty_driver_write(struct tty_struct *tty, - const unsigned char *buf, int len) -{ - struct pti_tty *pti_tty_data = tty->driver_data; - if ((pti_tty_data != NULL) && (pti_tty_data->mc != NULL)) { - pti_write_to_aperture(pti_tty_data->mc, (u8 *)buf, len); - return len; - } - /* - * we can't write to the pti hardware if the private driver_data - * and the mc address is not there. - */ - else - return -EFAULT; -} - -/** - * pti_tty_write_room()- Always returns 2048. - * - * @tty: contains tty info of the pti driver. - */ -static int pti_tty_write_room(struct tty_struct *tty) -{ - return 2048; -} - -/** - * pti_char_open()- Open an Application master, channel aperture - * ID to the PTI device. Part of the misc device implementation. - * - * @inode: not used. - * @filp: Output- will have a masterchannel struct set containing - * the allocated application PTI aperture write address. - * - * Returns: - * int, 0 for success - * otherwise, a fail value - */ -static int pti_char_open(struct inode *inode, struct file *filp) -{ - struct pti_masterchannel *mc; - - /* - * We really do want to fail immediately if - * pti_request_masterchannel() fails, - * before assigning the value to filp->private_data. - * Slightly easier to debug if this driver needs debugging. - */ - mc = pti_request_masterchannel(0, NULL); - if (mc == NULL) - return -ENOMEM; - filp->private_data = mc; - return 0; -} - -/** - * pti_char_release()- Close a char channel to the PTI device. Part - * of the misc device implementation. - * - * @inode: Not used in this implementaiton. - * @filp: Contains private_data that contains the master, channel - * ID to be released by the PTI device. - * - * Returns: - * always 0 - */ -static int pti_char_release(struct inode *inode, struct file *filp) -{ - pti_release_masterchannel(filp->private_data); - filp->private_data = NULL; - return 0; -} - -/** - * pti_char_write()- Write trace debugging data through the char - * interface to the PTI HW. Part of the misc device implementation. - * - * @filp: Contains private data which is used to obtain - * master, channel write ID. - * @data: trace data to be written. - * @len: # of byte to write. - * @ppose: Not used in this function implementation. - * - * Returns: - * int, # of bytes written - * otherwise, error value - * - * Notes: From side discussions with Alan Cox and experimenting - * with PTI debug HW like Nokia's Fido box and Lauterbach - * devices, 8192 byte write buffer used by USER_COPY_SIZE was - * deemed an appropriate size for this type of usage with - * debugging HW. - */ -static ssize_t pti_char_write(struct file *filp, const char __user *data, - size_t len, loff_t *ppose) -{ - struct pti_masterchannel *mc; - void *kbuf; - const char __user *tmp; - size_t size = USER_COPY_SIZE; - size_t n = 0; - - tmp = data; - mc = filp->private_data; - - kbuf = kmalloc(size, GFP_KERNEL); - if (kbuf == NULL) { - pr_err("%s(%d): buf allocation failed\n", - __func__, __LINE__); - return -ENOMEM; - } - - do { - if (len - n > USER_COPY_SIZE) - size = USER_COPY_SIZE; - else - size = len - n; - - if (copy_from_user(kbuf, tmp, size)) { - kfree(kbuf); - return n ? n : -EFAULT; - } - - pti_write_to_aperture(mc, kbuf, size); - n += size; - tmp += size; - - } while (len > n); - - kfree(kbuf); - return len; -} - -static const struct tty_operations pti_tty_driver_ops = { - .open = pti_tty_driver_open, - .close = pti_tty_driver_close, - .write = pti_tty_driver_write, - .write_room = pti_tty_write_room, - .install = pti_tty_install, - .cleanup = pti_tty_cleanup -}; - -static const struct file_operations pti_char_driver_ops = { - .owner = THIS_MODULE, - .write = pti_char_write, - .open = pti_char_open, - .release = pti_char_release, -}; - -static struct miscdevice pti_char_driver = { - .minor = MISC_DYNAMIC_MINOR, - .name = CHARNAME, - .fops = &pti_char_driver_ops -}; - -/** - * pti_console_write()- Write to the console that has been acquired. - * - * @c: Not used in this implementaiton. - * @buf: Data to be written. - * @len: Length of buf. - */ -static void pti_console_write(struct console *c, const char *buf, unsigned len) -{ - static struct pti_masterchannel mc = {.master = CONSOLE_ID, - .channel = 0}; - - mc.channel = pti_console_channel; - pti_console_channel = (pti_console_channel + 1) & 0x7f; - - pti_write_full_frame_to_aperture(&mc, buf, len); -} - -/** - * pti_console_device()- Return the driver tty structure and set the - * associated index implementation. - * - * @c: Console device of the driver. - * @index: index associated with c. - * - * Returns: - * always value of pti_tty_driver structure when this function - * is called. - */ -static struct tty_driver *pti_console_device(struct console *c, int *index) -{ - *index = c->index; - return pti_tty_driver; -} - -/** - * pti_console_setup()- Initialize console variables used by the driver. - * - * @c: Not used. - * @opts: Not used. - * - * Returns: - * always 0. - */ -static int pti_console_setup(struct console *c, char *opts) -{ - pti_console_channel = 0; - pti_control_channel = 0; - return 0; -} - -/* - * pti_console struct, used to capture OS printk()'s and shift - * out to the PTI device for debugging. This cannot be - * enabled upon boot because of the possibility of eating - * any serial console printk's (race condition discovered). - * The console should be enabled upon when the tty port is - * used for the first time. Since the primary purpose for - * the tty port is to hook up syslog to it, the tty port - * will be open for a really long time. - */ -static struct console pti_console = { - .name = TTYNAME, - .write = pti_console_write, - .device = pti_console_device, - .setup = pti_console_setup, - .flags = CON_PRINTBUFFER, - .index = 0, -}; - -/** - * pti_port_activate()- Used to start/initialize any items upon - * first opening of tty_port(). - * - * @port: The tty port number of the PTI device. - * @tty: The tty struct associated with this device. - * - * Returns: - * always returns 0 - * - * Notes: The primary purpose of the PTI tty port 0 is to hook - * the syslog daemon to it; thus this port will be open for a - * very long time. - */ -static int pti_port_activate(struct tty_port *port, struct tty_struct *tty) -{ - if (port->tty->index == PTITTY_MINOR_START) - console_start(&pti_console); - return 0; -} - -/** - * pti_port_shutdown()- Used to stop/shutdown any items upon the - * last tty port close. - * - * @port: The tty port number of the PTI device. - * - * Notes: The primary purpose of the PTI tty port 0 is to hook - * the syslog daemon to it; thus this port will be open for a - * very long time. - */ -static void pti_port_shutdown(struct tty_port *port) -{ - if (port->tty->index == PTITTY_MINOR_START) - console_stop(&pti_console); -} - -static const struct tty_port_operations tty_port_ops = { - .activate = pti_port_activate, - .shutdown = pti_port_shutdown, -}; - -/* - * Note the _probe() call sets everything up and ties the char and tty - * to successfully detecting the PTI device on the pci bus. - */ - -/** - * pti_pci_probe()- Used to detect pti on the pci bus and set - * things up in the driver. - * - * @pdev: pci_dev struct values for pti. - * @ent: pci_device_id struct for pti driver. - * - * Returns: - * 0 for success - * otherwise, error - */ -static int pti_pci_probe(struct pci_dev *pdev, - const struct pci_device_id *ent) -{ - unsigned int a; - int retval; - int pci_bar = 1; - - dev_dbg(&pdev->dev, "%s %s(%d): PTI PCI ID %04x:%04x\n", __FILE__, - __func__, __LINE__, pdev->vendor, pdev->device); - - retval = misc_register(&pti_char_driver); - if (retval) { - pr_err("%s(%d): CHAR registration failed of pti driver\n", - __func__, __LINE__); - pr_err("%s(%d): Error value returned: %d\n", - __func__, __LINE__, retval); - goto err; - } - - retval = pci_enable_device(pdev); - if (retval != 0) { - dev_err(&pdev->dev, - "%s: pci_enable_device() returned error %d\n", - __func__, retval); - goto err_unreg_misc; - } - - drv_data = kzalloc(sizeof(*drv_data), GFP_KERNEL); - if (drv_data == NULL) { - retval = -ENOMEM; - dev_err(&pdev->dev, - "%s(%d): kmalloc() returned NULL memory.\n", - __func__, __LINE__); - goto err_disable_pci; - } - drv_data->pti_addr = pci_resource_start(pdev, pci_bar); - - retval = pci_request_region(pdev, pci_bar, dev_name(&pdev->dev)); - if (retval != 0) { - dev_err(&pdev->dev, - "%s(%d): pci_request_region() returned error %d\n", - __func__, __LINE__, retval); - goto err_free_dd; - } - drv_data->aperture_base = drv_data->pti_addr+APERTURE_14; - drv_data->pti_ioaddr = - ioremap((u32)drv_data->aperture_base, - APERTURE_LEN); - if (!drv_data->pti_ioaddr) { - retval = -ENOMEM; - goto err_rel_reg; - } - - pci_set_drvdata(pdev, drv_data); - - for (a = 0; a < PTITTY_MINOR_NUM; a++) { - struct tty_port *port = &drv_data->port[a]; - tty_port_init(port); - port->ops = &tty_port_ops; - - tty_port_register_device(port, pti_tty_driver, a, &pdev->dev); - } - - register_console(&pti_console); - - return 0; -err_rel_reg: - pci_release_region(pdev, pci_bar); -err_free_dd: - kfree(drv_data); -err_disable_pci: - pci_disable_device(pdev); -err_unreg_misc: - misc_deregister(&pti_char_driver); -err: - return retval; -} - -/** - * pti_pci_remove()- Driver exit method to remove PTI from - * PCI bus. - * @pdev: variable containing pci info of PTI. - */ -static void pti_pci_remove(struct pci_dev *pdev) -{ - struct pti_dev *drv_data = pci_get_drvdata(pdev); - unsigned int a; - - unregister_console(&pti_console); - - for (a = 0; a < PTITTY_MINOR_NUM; a++) { - tty_unregister_device(pti_tty_driver, a); - tty_port_destroy(&drv_data->port[a]); - } - - iounmap(drv_data->pti_ioaddr); - kfree(drv_data); - pci_release_region(pdev, 1); - pci_disable_device(pdev); - - misc_deregister(&pti_char_driver); -} - -static struct pci_driver pti_pci_driver = { - .name = PCINAME, - .id_table = pci_ids, - .probe = pti_pci_probe, - .remove = pti_pci_remove, -}; - -/** - * pti_init()- Overall entry/init call to the pti driver. - * It starts the registration process with the kernel. - * - * Returns: - * int __init, 0 for success - * otherwise value is an error - * - */ -static int __init pti_init(void) -{ - int retval; - - /* First register module as tty device */ - - pti_tty_driver = alloc_tty_driver(PTITTY_MINOR_NUM); - if (pti_tty_driver == NULL) { - pr_err("%s(%d): Memory allocation failed for ptiTTY driver\n", - __func__, __LINE__); - return -ENOMEM; - } - - pti_tty_driver->driver_name = DRIVERNAME; - pti_tty_driver->name = TTYNAME; - pti_tty_driver->major = 0; - pti_tty_driver->minor_start = PTITTY_MINOR_START; - pti_tty_driver->type = TTY_DRIVER_TYPE_SYSTEM; - pti_tty_driver->subtype = SYSTEM_TYPE_SYSCONS; - pti_tty_driver->flags = TTY_DRIVER_REAL_RAW | - TTY_DRIVER_DYNAMIC_DEV; - pti_tty_driver->init_termios = tty_std_termios; - - tty_set_operations(pti_tty_driver, &pti_tty_driver_ops); - - retval = tty_register_driver(pti_tty_driver); - if (retval) { - pr_err("%s(%d): TTY registration failed of pti driver\n", - __func__, __LINE__); - pr_err("%s(%d): Error value returned: %d\n", - __func__, __LINE__, retval); - - goto put_tty; - } - - retval = pci_register_driver(&pti_pci_driver); - if (retval) { - pr_err("%s(%d): PCI registration failed of pti driver\n", - __func__, __LINE__); - pr_err("%s(%d): Error value returned: %d\n", - __func__, __LINE__, retval); - goto unreg_tty; - } - - return 0; -unreg_tty: - tty_unregister_driver(pti_tty_driver); -put_tty: - put_tty_driver(pti_tty_driver); - pti_tty_driver = NULL; - return retval; -} - -/** - * pti_exit()- Unregisters this module as a tty and pci driver. - */ -static void __exit pti_exit(void) -{ - tty_unregister_driver(pti_tty_driver); - pci_unregister_driver(&pti_pci_driver); - put_tty_driver(pti_tty_driver); -} - -module_init(pti_init); -module_exit(pti_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Ken Mills, Jay Freyensee"); -MODULE_DESCRIPTION("PTI Driver"); - diff --git a/drivers/misc/pvpanic.c b/drivers/misc/pvpanic.c index 41cab297d66e..9f350e05ef68 100644 --- a/drivers/misc/pvpanic.c +++ b/drivers/misc/pvpanic.c @@ -19,6 +19,47 @@ #include <uapi/misc/pvpanic.h> static void __iomem *base; +static unsigned int capability = PVPANIC_PANICKED | PVPANIC_CRASH_LOADED; +static unsigned int events; + +static ssize_t capability_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%x\n", capability); +} +static DEVICE_ATTR_RO(capability); + +static ssize_t events_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%x\n", events); +} + +static ssize_t events_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + unsigned int tmp; + int err; + + err = kstrtouint(buf, 16, &tmp); + if (err) + return err; + + if ((tmp & capability) != tmp) + return -EINVAL; + + events = tmp; + + return count; + +} +static DEVICE_ATTR_RW(events); + +static struct attribute *pvpanic_dev_attrs[] = { + &dev_attr_capability.attr, + &dev_attr_events.attr, + NULL +}; +ATTRIBUTE_GROUPS(pvpanic_dev); MODULE_AUTHOR("Hu Tao <hutao@cn.fujitsu.com>"); MODULE_DESCRIPTION("pvpanic device driver"); @@ -27,7 +68,8 @@ MODULE_LICENSE("GPL"); static void pvpanic_send_event(unsigned int event) { - iowrite8(event, base); + if (event & capability & events) + iowrite8(event, base); } static int @@ -73,8 +115,13 @@ static int pvpanic_mmio_probe(struct platform_device *pdev) return -EINVAL; } - atomic_notifier_chain_register(&panic_notifier_list, - &pvpanic_panic_nb); + /* initlize capability by RDPT */ + capability &= ioread8(base); + events = capability; + + if (capability) + atomic_notifier_chain_register(&panic_notifier_list, + &pvpanic_panic_nb); return 0; } @@ -82,8 +129,9 @@ static int pvpanic_mmio_probe(struct platform_device *pdev) static int pvpanic_mmio_remove(struct platform_device *pdev) { - atomic_notifier_chain_unregister(&panic_notifier_list, - &pvpanic_panic_nb); + if (capability) + atomic_notifier_chain_unregister(&panic_notifier_list, + &pvpanic_panic_nb); return 0; } @@ -104,6 +152,7 @@ static struct platform_driver pvpanic_mmio_driver = { .name = "pvpanic-mmio", .of_match_table = pvpanic_mmio_match, .acpi_match_table = pvpanic_device_ids, + .dev_groups = pvpanic_dev_groups, }, .probe = pvpanic_mmio_probe, .remove = pvpanic_mmio_remove, diff --git a/drivers/misc/sgi-xp/xpnet.c b/drivers/misc/sgi-xp/xpnet.c index 23837d0d6f4a..2508f83bdc3f 100644 --- a/drivers/misc/sgi-xp/xpnet.c +++ b/drivers/misc/sgi-xp/xpnet.c @@ -208,7 +208,7 @@ xpnet_receive(short partid, int channel, struct xpnet_message *msg) } else { dst = (void *)((u64)skb->data & ~(L1_CACHE_BYTES - 1)); dev_dbg(xpnet, "transferring buffer to the skb->data area;\n\t" - "xp_remote_memcpy(0x%p, 0x%p, %hu)\n", dst, + "xp_remote_memcpy(0x%p, 0x%p, %u)\n", dst, (void *)msg->buf_pa, msg->size); ret = xp_remote_memcpy(xp_pa(dst), msg->buf_pa, msg->size); @@ -218,7 +218,7 @@ xpnet_receive(short partid, int channel, struct xpnet_message *msg) * !!! appears in_use and we can't just call * !!! dev_kfree_skb. */ - dev_err(xpnet, "xp_remote_memcpy(0x%p, 0x%p, 0x%hx) " + dev_err(xpnet, "xp_remote_memcpy(0x%p, 0x%p, 0x%x) " "returned error=0x%x\n", dst, (void *)msg->buf_pa, msg->size, ret); diff --git a/drivers/misc/vmw_vmci/vmci_queue_pair.c b/drivers/misc/vmw_vmci/vmci_queue_pair.c index c49065887e8f..880c33ab9f47 100644 --- a/drivers/misc/vmw_vmci/vmci_queue_pair.c +++ b/drivers/misc/vmw_vmci/vmci_queue_pair.c @@ -237,7 +237,9 @@ static struct qp_list qp_guest_endpoints = { #define QPE_NUM_PAGES(_QPE) ((u32) \ (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \ DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2)) - +#define QP_SIZES_ARE_VALID(_prod_qsize, _cons_qsize) \ + ((_prod_qsize) + (_cons_qsize) >= max(_prod_qsize, _cons_qsize) && \ + (_prod_qsize) + (_cons_qsize) <= VMCI_MAX_GUEST_QP_MEMORY) /* * Frees kernel VA space for a given queue and its queue header, and @@ -528,7 +530,7 @@ static struct vmci_queue *qp_host_alloc_queue(u64 size) u64 num_pages; const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if)); - if (size > SIZE_MAX - PAGE_SIZE) + if (size > min_t(size_t, VMCI_MAX_GUEST_QP_MEMORY, SIZE_MAX - PAGE_SIZE)) return NULL; num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1; if (num_pages > (SIZE_MAX - queue_size) / @@ -537,6 +539,9 @@ static struct vmci_queue *qp_host_alloc_queue(u64 size) queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page); + if (queue_size + queue_page_size > KMALLOC_MAX_SIZE) + return NULL; + queue = kzalloc(queue_size + queue_page_size, GFP_KERNEL); if (queue) { queue->q_header = NULL; @@ -630,7 +635,7 @@ static void qp_release_pages(struct page **pages, for (i = 0; i < num_pages; i++) { if (dirty) - set_page_dirty(pages[i]); + set_page_dirty_lock(pages[i]); put_page(pages[i]); pages[i] = NULL; @@ -1207,7 +1212,7 @@ static int qp_alloc_guest_work(struct vmci_handle *handle, } else { result = qp_alloc_hypercall(queue_pair_entry); if (result < VMCI_SUCCESS) { - pr_warn("qp_alloc_hypercall result = %d\n", result); + pr_devel("qp_alloc_hypercall result = %d\n", result); goto error; } } @@ -1929,6 +1934,9 @@ int vmci_qp_broker_alloc(struct vmci_handle handle, struct vmci_qp_page_store *page_store, struct vmci_ctx *context) { + if (!QP_SIZES_ARE_VALID(produce_size, consume_size)) + return VMCI_ERROR_NO_RESOURCES; + return qp_broker_alloc(handle, peer, flags, priv_flags, produce_size, consume_size, page_store, context, NULL, NULL, NULL, NULL); @@ -2685,8 +2693,7 @@ int vmci_qpair_alloc(struct vmci_qp **qpair, * used by the device is NO_RESOURCES, so use that here too. */ - if (produce_qsize + consume_qsize < max(produce_qsize, consume_qsize) || - produce_qsize + consume_qsize > VMCI_MAX_GUEST_QP_MEMORY) + if (!QP_SIZES_ARE_VALID(produce_qsize, consume_qsize)) return VMCI_ERROR_NO_RESOURCES; retval = vmci_route(&src, &dst, false, &route); diff --git a/drivers/misc/vmw_vmci/vmci_queue_pair.h b/drivers/misc/vmw_vmci/vmci_queue_pair.h index 00017fc29a52..c4e6e924d9be 100644 --- a/drivers/misc/vmw_vmci/vmci_queue_pair.h +++ b/drivers/misc/vmw_vmci/vmci_queue_pair.h @@ -104,7 +104,7 @@ struct vmci_qp_dtch_info { struct vmci_qp_page_store { /* Reference to pages backing the queue pair. */ u64 pages; - /* Length of pageList/virtual addres range (in pages). */ + /* Length of pageList/virtual address range (in pages). */ u32 len; }; diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c index 646823ddd317..2d73407ee52e 100644 --- a/drivers/mmc/host/sdhci.c +++ b/drivers/mmc/host/sdhci.c @@ -20,7 +20,6 @@ #include <linux/slab.h> #include <linux/scatterlist.h> #include <linux/sizes.h> -#include <linux/swiotlb.h> #include <linux/regulator/consumer.h> #include <linux/pm_runtime.h> #include <linux/of.h> @@ -4582,12 +4581,8 @@ int sdhci_setup_host(struct sdhci_host *host) mmc->max_segs = SDHCI_MAX_SEGS; } else if (host->flags & SDHCI_USE_SDMA) { mmc->max_segs = 1; - if (swiotlb_max_segment()) { - unsigned int max_req_size = (1 << IO_TLB_SHIFT) * - IO_TLB_SEGSIZE; - mmc->max_req_size = min(mmc->max_req_size, - max_req_size); - } + mmc->max_req_size = min_t(size_t, mmc->max_req_size, + dma_max_mapping_size(mmc_dev(mmc))); } else { /* PIO */ mmc->max_segs = SDHCI_MAX_SEGS; } diff --git a/drivers/most/core.c b/drivers/most/core.c index 353ab277cbc6..e4412c7d25b0 100644 --- a/drivers/most/core.c +++ b/drivers/most/core.c @@ -379,7 +379,7 @@ static struct attribute *channel_attrs[] = { NULL, }; -static struct attribute_group channel_attr_group = { +static const struct attribute_group channel_attr_group = { .attrs = channel_attrs, .is_visible = channel_attr_is_visible, }; @@ -436,7 +436,7 @@ static struct attribute *interface_attrs[] = { NULL, }; -static struct attribute_group interface_attr_group = { +static const struct attribute_group interface_attr_group = { .attrs = interface_attrs, }; @@ -718,7 +718,7 @@ static struct attribute *mc_attrs[] = { NULL, }; -static struct attribute_group mc_attr_group = { +static const struct attribute_group mc_attr_group = { .attrs = mc_attrs, }; diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig index 45d12b0e9a2f..b09bed554f26 100644 --- a/drivers/net/Kconfig +++ b/drivers/net/Kconfig @@ -88,7 +88,7 @@ config WIREGUARD select CRYPTO_CURVE25519_X86 if X86 && 64BIT select ARM_CRYPTO if ARM select ARM64_CRYPTO if ARM64 - select CRYPTO_CHACHA20_NEON if (ARM || ARM64) && KERNEL_MODE_NEON + select CRYPTO_CHACHA20_NEON if ARM || (ARM64 && KERNEL_MODE_NEON) select CRYPTO_POLY1305_NEON if ARM64 && KERNEL_MODE_NEON select CRYPTO_POLY1305_ARM if ARM select CRYPTO_BLAKE2S_ARM if ARM diff --git a/drivers/net/can/dev/dev.c b/drivers/net/can/dev/dev.c index d9281ae853f8..311d8564d611 100644 --- a/drivers/net/can/dev/dev.c +++ b/drivers/net/can/dev/dev.c @@ -239,6 +239,7 @@ void can_setup(struct net_device *dev) struct net_device *alloc_candev_mqs(int sizeof_priv, unsigned int echo_skb_max, unsigned int txqs, unsigned int rxqs) { + struct can_ml_priv *can_ml; struct net_device *dev; struct can_priv *priv; int size; @@ -270,7 +271,8 @@ struct net_device *alloc_candev_mqs(int sizeof_priv, unsigned int echo_skb_max, priv = netdev_priv(dev); priv->dev = dev; - dev->ml_priv = (void *)priv + ALIGN(sizeof_priv, NETDEV_ALIGN); + can_ml = (void *)priv + ALIGN(sizeof_priv, NETDEV_ALIGN); + can_set_ml_priv(dev, can_ml); if (echo_skb_max) { priv->echo_skb_max = echo_skb_max; diff --git a/drivers/net/can/slcan.c b/drivers/net/can/slcan.c index a1bd1be09548..30c8d53c9745 100644 --- a/drivers/net/can/slcan.c +++ b/drivers/net/can/slcan.c @@ -516,6 +516,7 @@ static struct slcan *slc_alloc(void) int i; char name[IFNAMSIZ]; struct net_device *dev = NULL; + struct can_ml_priv *can_ml; struct slcan *sl; int size; @@ -538,7 +539,8 @@ static struct slcan *slc_alloc(void) dev->base_addr = i; sl = netdev_priv(dev); - dev->ml_priv = (void *)sl + ALIGN(sizeof(*sl), NETDEV_ALIGN); + can_ml = (void *)sl + ALIGN(sizeof(*sl), NETDEV_ALIGN); + can_set_ml_priv(dev, can_ml); /* Initialize channel control data */ sl->magic = SLCAN_MAGIC; diff --git a/drivers/net/can/vcan.c b/drivers/net/can/vcan.c index 39ca14b0585d..067705e2850b 100644 --- a/drivers/net/can/vcan.c +++ b/drivers/net/can/vcan.c @@ -153,7 +153,7 @@ static void vcan_setup(struct net_device *dev) dev->addr_len = 0; dev->tx_queue_len = 0; dev->flags = IFF_NOARP; - dev->ml_priv = netdev_priv(dev); + can_set_ml_priv(dev, netdev_priv(dev)); /* set flags according to driver capabilities */ if (echo) diff --git a/drivers/net/can/vxcan.c b/drivers/net/can/vxcan.c index f9a524c5f6d6..8861a7d875e7 100644 --- a/drivers/net/can/vxcan.c +++ b/drivers/net/can/vxcan.c @@ -141,6 +141,8 @@ static const struct net_device_ops vxcan_netdev_ops = { static void vxcan_setup(struct net_device *dev) { + struct can_ml_priv *can_ml; + dev->type = ARPHRD_CAN; dev->mtu = CANFD_MTU; dev->hard_header_len = 0; @@ -149,7 +151,9 @@ static void vxcan_setup(struct net_device *dev) dev->flags = (IFF_NOARP|IFF_ECHO); dev->netdev_ops = &vxcan_netdev_ops; dev->needs_free_netdev = true; - dev->ml_priv = netdev_priv(dev) + ALIGN(sizeof(struct vxcan_priv), NETDEV_ALIGN); + + can_ml = netdev_priv(dev) + ALIGN(sizeof(struct vxcan_priv), NETDEV_ALIGN); + can_set_ml_priv(dev, can_ml); } /* forward declaration for rtnl_create_link() */ diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c index ae86ded1e2a1..a162499bcafc 100644 --- a/drivers/net/dsa/b53/b53_common.c +++ b/drivers/net/dsa/b53/b53_common.c @@ -543,6 +543,19 @@ static void b53_port_set_mcast_flood(struct b53_device *dev, int port, b53_write16(dev, B53_CTRL_PAGE, B53_IPMC_FLOOD_MASK, mc); } +static void b53_port_set_learning(struct b53_device *dev, int port, + bool learning) +{ + u16 reg; + + b53_read16(dev, B53_CTRL_PAGE, B53_DIS_LEARNING, ®); + if (learning) + reg &= ~BIT(port); + else + reg |= BIT(port); + b53_write16(dev, B53_CTRL_PAGE, B53_DIS_LEARNING, reg); +} + int b53_enable_port(struct dsa_switch *ds, int port, struct phy_device *phy) { struct b53_device *dev = ds->priv; @@ -557,6 +570,7 @@ int b53_enable_port(struct dsa_switch *ds, int port, struct phy_device *phy) b53_port_set_ucast_flood(dev, port, true); b53_port_set_mcast_flood(dev, port, true); + b53_port_set_learning(dev, port, false); if (dev->ops->irq_enable) ret = dev->ops->irq_enable(dev, port); @@ -691,6 +705,7 @@ static void b53_enable_cpu_port(struct b53_device *dev, int port) b53_port_set_ucast_flood(dev, port, true); b53_port_set_mcast_flood(dev, port, true); + b53_port_set_learning(dev, port, false); } static void b53_enable_mib(struct b53_device *dev) @@ -1953,19 +1968,20 @@ void b53_br_fast_age(struct dsa_switch *ds, int port) } EXPORT_SYMBOL(b53_br_fast_age); -static int b53_br_flags_pre(struct dsa_switch *ds, int port, - struct switchdev_brport_flags flags, - struct netlink_ext_ack *extack) +int b53_br_flags_pre(struct dsa_switch *ds, int port, + struct switchdev_brport_flags flags, + struct netlink_ext_ack *extack) { - if (flags.mask & ~(BR_FLOOD | BR_MCAST_FLOOD)) + if (flags.mask & ~(BR_FLOOD | BR_MCAST_FLOOD | BR_LEARNING)) return -EINVAL; return 0; } +EXPORT_SYMBOL(b53_br_flags_pre); -static int b53_br_flags(struct dsa_switch *ds, int port, - struct switchdev_brport_flags flags, - struct netlink_ext_ack *extack) +int b53_br_flags(struct dsa_switch *ds, int port, + struct switchdev_brport_flags flags, + struct netlink_ext_ack *extack) { if (flags.mask & BR_FLOOD) b53_port_set_ucast_flood(ds->priv, port, @@ -1973,17 +1989,22 @@ static int b53_br_flags(struct dsa_switch *ds, int port, if (flags.mask & BR_MCAST_FLOOD) b53_port_set_mcast_flood(ds->priv, port, !!(flags.val & BR_MCAST_FLOOD)); + if (flags.mask & BR_LEARNING) + b53_port_set_learning(ds->priv, port, + !!(flags.val & BR_LEARNING)); return 0; } +EXPORT_SYMBOL(b53_br_flags); -static int b53_set_mrouter(struct dsa_switch *ds, int port, bool mrouter, - struct netlink_ext_ack *extack) +int b53_set_mrouter(struct dsa_switch *ds, int port, bool mrouter, + struct netlink_ext_ack *extack) { b53_port_set_mcast_flood(ds->priv, port, mrouter); return 0; } +EXPORT_SYMBOL(b53_set_mrouter); static bool b53_possible_cpu_port(struct dsa_switch *ds, int port) { diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h index faf983fbca82..8419bb7f4505 100644 --- a/drivers/net/dsa/b53/b53_priv.h +++ b/drivers/net/dsa/b53/b53_priv.h @@ -326,6 +326,14 @@ int b53_br_join(struct dsa_switch *ds, int port, struct net_device *bridge); void b53_br_leave(struct dsa_switch *ds, int port, struct net_device *bridge); void b53_br_set_stp_state(struct dsa_switch *ds, int port, u8 state); void b53_br_fast_age(struct dsa_switch *ds, int port); +int b53_br_flags_pre(struct dsa_switch *ds, int port, + struct switchdev_brport_flags flags, + struct netlink_ext_ack *extack); +int b53_br_flags(struct dsa_switch *ds, int port, + struct switchdev_brport_flags flags, + struct netlink_ext_ack *extack); +int b53_set_mrouter(struct dsa_switch *ds, int port, bool mrouter, + struct netlink_ext_ack *extack); int b53_setup_devlink_resources(struct dsa_switch *ds); void b53_port_event(struct dsa_switch *ds, int port); void b53_phylink_validate(struct dsa_switch *ds, int port, diff --git a/drivers/net/dsa/b53/b53_regs.h b/drivers/net/dsa/b53/b53_regs.h index c90985c294a2..b2c539a42154 100644 --- a/drivers/net/dsa/b53/b53_regs.h +++ b/drivers/net/dsa/b53/b53_regs.h @@ -115,6 +115,7 @@ #define B53_UC_FLOOD_MASK 0x32 #define B53_MC_FLOOD_MASK 0x34 #define B53_IPMC_FLOOD_MASK 0x36 +#define B53_DIS_LEARNING 0x3c /* * Override Ports 0-7 State on devices with xMII interfaces (8 bit) diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c index 1857aa9aa84a..5ee8103b8e9c 100644 --- a/drivers/net/dsa/bcm_sf2.c +++ b/drivers/net/dsa/bcm_sf2.c @@ -223,23 +223,10 @@ static int bcm_sf2_port_setup(struct dsa_switch *ds, int port, reg &= ~P_TXQ_PSM_VDD(port); core_writel(priv, reg, CORE_MEM_PSM_VDD_CTRL); - /* Enable learning */ - reg = core_readl(priv, CORE_DIS_LEARN); - reg &= ~BIT(port); - core_writel(priv, reg, CORE_DIS_LEARN); - /* Enable Broadcom tags for that port if requested */ - if (priv->brcm_tag_mask & BIT(port)) { + if (priv->brcm_tag_mask & BIT(port)) b53_brcm_hdr_setup(ds, port); - /* Disable learning on ASP port */ - if (port == 7) { - reg = core_readl(priv, CORE_DIS_LEARN); - reg |= BIT(port); - core_writel(priv, reg, CORE_DIS_LEARN); - } - } - /* Configure Traffic Class to QoS mapping, allow each priority to map * to a different queue number */ @@ -1117,7 +1104,10 @@ static const struct dsa_switch_ops bcm_sf2_ops = { .set_mac_eee = b53_set_mac_eee, .port_bridge_join = b53_br_join, .port_bridge_leave = b53_br_leave, + .port_pre_bridge_flags = b53_br_flags_pre, + .port_bridge_flags = b53_br_flags, .port_stp_state_set = b53_br_set_stp_state, + .port_set_mrouter = b53_set_mrouter, .port_fast_age = b53_br_fast_age, .port_vlan_filtering = b53_vlan_filtering, .port_vlan_add = b53_vlan_add, diff --git a/drivers/net/dsa/sja1105/sja1105_static_config.c b/drivers/net/dsa/sja1105/sja1105_static_config.c index 139b7b4fbd0d..a8efb7fac395 100644 --- a/drivers/net/dsa/sja1105/sja1105_static_config.c +++ b/drivers/net/dsa/sja1105/sja1105_static_config.c @@ -85,7 +85,7 @@ u32 sja1105_crc32(const void *buf, size_t len) /* seed */ crc = ~0; for (i = 0; i < len; i += 4) { - sja1105_unpack((void *)buf + i, &word, 31, 0, 4); + sja1105_unpack(buf + i, &word, 31, 0, 4); crc = crc32_le(crc, (u8 *)&word, 4); } return ~crc; diff --git a/drivers/net/ethernet/atheros/ag71xx.c b/drivers/net/ethernet/atheros/ag71xx.c index dd5c8a9038bb..a60ce9030581 100644 --- a/drivers/net/ethernet/atheros/ag71xx.c +++ b/drivers/net/ethernet/atheros/ag71xx.c @@ -223,8 +223,6 @@ #define AG71XX_REG_RX_SM 0x01b0 #define AG71XX_REG_TX_SM 0x01b4 -#define ETH_SWITCH_HEADER_LEN 2 - #define AG71XX_DEFAULT_MSG_ENABLE \ (NETIF_MSG_DRV \ | NETIF_MSG_PROBE \ @@ -933,7 +931,7 @@ static void ag71xx_hw_setup(struct ag71xx *ag) static unsigned int ag71xx_max_frame_len(unsigned int mtu) { - return ETH_SWITCH_HEADER_LEN + ETH_HLEN + VLAN_HLEN + mtu + ETH_FCS_LEN; + return ETH_HLEN + VLAN_HLEN + mtu + ETH_FCS_LEN; } static void ag71xx_hw_set_macaddr(struct ag71xx *ag, unsigned char *mac) diff --git a/drivers/net/ethernet/broadcom/bcm4908_enet.c b/drivers/net/ethernet/broadcom/bcm4908_enet.c index 9be33dc98072..0b70e9e0ddad 100644 --- a/drivers/net/ethernet/broadcom/bcm4908_enet.c +++ b/drivers/net/ethernet/broadcom/bcm4908_enet.c @@ -570,6 +570,7 @@ static int bcm4908_enet_poll(struct napi_struct *napi, int weight) if (len < ETH_ZLEN || (ctl & (DMA_CTL_STATUS_SOP | DMA_CTL_STATUS_EOP)) != (DMA_CTL_STATUS_SOP | DMA_CTL_STATUS_EOP)) { + kfree_skb(slot.skb); enet->netdev->stats.rx_dropped++; break; } @@ -582,6 +583,8 @@ static int bcm4908_enet_poll(struct napi_struct *napi, int weight) enet->netdev->stats.rx_packets++; enet->netdev->stats.rx_bytes += len; + + handled++; } if (handled < weight) { diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.c b/drivers/net/ethernet/broadcom/bcm63xx_enet.c index fd8767213165..977f097fc7bf 100644 --- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c +++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c @@ -1192,7 +1192,6 @@ static int bcm_enet_stop(struct net_device *dev) kdev = &priv->pdev->dev; netif_stop_queue(dev); - netdev_reset_queue(dev); napi_disable(&priv->napi); if (priv->has_phy) phy_stop(dev->phydev); @@ -1231,6 +1230,9 @@ static int bcm_enet_stop(struct net_device *dev) if (priv->has_phy) phy_disconnect(dev->phydev); + /* reset BQL after forced tx reclaim to prevent kernel panic */ + netdev_reset_queue(dev); + return 0; } @@ -2343,7 +2345,6 @@ static int bcm_enetsw_stop(struct net_device *dev) del_timer_sync(&priv->swphy_poll); netif_stop_queue(dev); - netdev_reset_queue(dev); napi_disable(&priv->napi); del_timer_sync(&priv->rx_timeout); @@ -2371,6 +2372,9 @@ static int bcm_enetsw_stop(struct net_device *dev) free_irq(priv->irq_tx, dev); free_irq(priv->irq_rx, dev); + /* reset BQL after forced tx reclaim to prevent kernel panic */ + netdev_reset_queue(dev); + return 0; } diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c index ccfe52a50a66..720dc99bd1fc 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c @@ -2670,7 +2670,6 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal, u32 hash; u64 ns; - np = container_of(&portal, struct dpaa_napi_portal, p); dpaa_fq = container_of(fq, struct dpaa_fq, fq_base); fd_status = be32_to_cpu(fd->status); fd_format = qm_fd_get_format(fd); @@ -2685,6 +2684,7 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal, percpu_priv = this_cpu_ptr(priv->percpu_priv); percpu_stats = &percpu_priv->stats; + np = &percpu_priv->np; if (unlikely(dpaa_eth_napi_schedule(percpu_priv, portal, sched_napi))) return qman_cb_dqrr_stop; diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 1c0e4beb56e7..118a4bd3f877 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -1172,12 +1172,25 @@ static int ibmvnic_open(struct net_device *netdev) struct ibmvnic_adapter *adapter = netdev_priv(netdev); int rc; - /* If device failover is pending, just set device state and return. - * Device operation will be handled by reset routine. + ASSERT_RTNL(); + + /* If device failover is pending or we are about to reset, just set + * device state and return. Device operation will be handled by reset + * routine. + * + * It should be safe to overwrite the adapter->state here. Since + * we hold the rtnl, either the reset has not actually started or + * the rtnl got dropped during the set_link_state() in do_reset(). + * In the former case, no one else is changing the state (again we + * have the rtnl) and in the latter case, do_reset() will detect and + * honor our setting below. */ - if (adapter->failover_pending) { + if (adapter->failover_pending || (test_bit(0, &adapter->resetting))) { + netdev_dbg(netdev, "[S:%d FOP:%d] Resetting, deferring open\n", + adapter->state, adapter->failover_pending); adapter->state = VNIC_OPEN; - return 0; + rc = 0; + goto out; } if (adapter->state != VNIC_CLOSED) { @@ -1196,10 +1209,12 @@ static int ibmvnic_open(struct net_device *netdev) rc = __ibmvnic_open(netdev); out: - /* If open fails due to a pending failover, set device state and - * return. Device operation will be handled by reset routine. + /* If open failed and there is a pending failover or in-progress reset, + * set device state and return. Device operation will be handled by + * reset routine. See also comments above regarding rtnl. */ - if (rc && adapter->failover_pending) { + if (rc && + (adapter->failover_pending || (test_bit(0, &adapter->resetting)))) { adapter->state = VNIC_OPEN; rc = 0; } @@ -1928,6 +1943,14 @@ static int do_reset(struct ibmvnic_adapter *adapter, if (rwi->reset_reason == VNIC_RESET_FAILOVER) adapter->failover_pending = false; + /* read the state and check (again) after getting rtnl */ + reset_state = adapter->state; + + if (reset_state == VNIC_REMOVING || reset_state == VNIC_REMOVED) { + rc = -EBUSY; + goto out; + } + netif_carrier_off(netdev); old_num_rx_queues = adapter->req_rx_queues; @@ -1958,11 +1981,27 @@ static int do_reset(struct ibmvnic_adapter *adapter, if (rc) goto out; + if (adapter->state == VNIC_OPEN) { + /* When we dropped rtnl, ibmvnic_open() got + * it and noticed that we are resetting and + * set the adapter state to OPEN. Update our + * new "target" state, and resume the reset + * from VNIC_CLOSING state. + */ + netdev_dbg(netdev, + "Open changed state from %d, updating.\n", + reset_state); + reset_state = VNIC_OPEN; + adapter->state = VNIC_CLOSING; + } + if (adapter->state != VNIC_CLOSING) { + /* If someone else changed the adapter state + * when we dropped the rtnl, fail the reset + */ rc = -1; goto out; } - adapter->state = VNIC_CLOSED; } } @@ -2106,6 +2145,14 @@ static int do_hard_reset(struct ibmvnic_adapter *adapter, netdev_dbg(adapter->netdev, "Hard resetting driver (%d)\n", rwi->reset_reason); + /* read the state and check (again) after getting rtnl */ + reset_state = adapter->state; + + if (reset_state == VNIC_REMOVING || reset_state == VNIC_REMOVED) { + rc = -EBUSY; + goto out; + } + netif_carrier_off(netdev); adapter->reset_reason = rwi->reset_reason; diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index a8a2b5f683a2..c70dec65a572 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -5083,7 +5083,7 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags) enum i40e_admin_queue_err adq_err; struct i40e_vsi *vsi = np->vsi; struct i40e_pf *pf = vsi->back; - bool is_reset_needed; + u32 reset_needed = 0; i40e_status status; u32 i, j; @@ -5128,9 +5128,11 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags) flags_complete: changed_flags = orig_flags ^ new_flags; - is_reset_needed = !!(changed_flags & (I40E_FLAG_VEB_STATS_ENABLED | - I40E_FLAG_LEGACY_RX | I40E_FLAG_SOURCE_PRUNING_DISABLED | - I40E_FLAG_DISABLE_FW_LLDP)); + if (changed_flags & I40E_FLAG_DISABLE_FW_LLDP) + reset_needed = I40E_PF_RESET_AND_REBUILD_FLAG; + if (changed_flags & (I40E_FLAG_VEB_STATS_ENABLED | + I40E_FLAG_LEGACY_RX | I40E_FLAG_SOURCE_PRUNING_DISABLED)) + reset_needed = BIT(__I40E_PF_RESET_REQUESTED); /* Before we finalize any flag changes, we need to perform some * checks to ensure that the changes are supported and safe. @@ -5252,7 +5254,7 @@ flags_complete: case I40E_AQ_RC_EEXIST: dev_warn(&pf->pdev->dev, "FW LLDP agent is already running\n"); - is_reset_needed = false; + reset_needed = 0; break; case I40E_AQ_RC_EPERM: dev_warn(&pf->pdev->dev, @@ -5281,8 +5283,8 @@ flags_complete: /* Issue reset to cause things to take effect, as additional bits * are added we will need to create a mask of bits requiring reset */ - if (is_reset_needed) - i40e_do_reset(pf, BIT(__I40E_PF_RESET_REQUESTED), true); + if (reset_needed) + i40e_do_reset(pf, reset_needed, true); return 0; } diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 8bb8eb65add9..353deae139f9 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -2616,7 +2616,7 @@ static void i40e_sync_filters_subtask(struct i40e_pf *pf) return; if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state)) return; - if (test_and_set_bit(__I40E_VF_DISABLE, pf->state)) { + if (test_bit(__I40E_VF_DISABLE, pf->state)) { set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); return; } @@ -2634,7 +2634,6 @@ static void i40e_sync_filters_subtask(struct i40e_pf *pf) } } } - clear_bit(__I40E_VF_DISABLE, pf->state); } /** @@ -5937,7 +5936,7 @@ static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc; ch->seid = ctxt.seid; ch->vsi_number = ctxt.vsi_number; - ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx); + ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx); /* copy just the sections touched not the entire info * since not all sections are valid as returned by @@ -7977,8 +7976,8 @@ static inline void i40e_set_cld_element(struct i40e_cloud_filter *filter, struct i40e_aqc_cloud_filters_element_data *cld) { - int i, j; u32 ipa; + int i; memset(cld, 0, sizeof(*cld)); ether_addr_copy(cld->outer_mac, filter->dst_mac); @@ -7989,14 +7988,14 @@ i40e_set_cld_element(struct i40e_cloud_filter *filter, if (filter->n_proto == ETH_P_IPV6) { #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) - for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6); - i++, j += 2) { + for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) { ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); - ipa = cpu_to_le32(ipa); - memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa)); + + *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa); } } else { ipa = be32_to_cpu(filter->dst_ipv4); + memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); } @@ -8044,6 +8043,8 @@ int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, if (filter->flags >= ARRAY_SIZE(flag_table)) return I40E_ERR_CONFIG; + memset(&cld_filter, 0, sizeof(cld_filter)); + /* copy element needed to add cloud filter from filter */ i40e_set_cld_element(filter, &cld_filter); @@ -8107,10 +8108,13 @@ int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, return -EOPNOTSUPP; /* adding filter using src_port/src_ip is not supported at this stage */ - if (filter->src_port || filter->src_ipv4 || + if (filter->src_port || + (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) || !ipv6_addr_any(&filter->ip.v6.src_ip6)) return -EOPNOTSUPP; + memset(&cld_filter, 0, sizeof(cld_filter)); + /* copy element needed to add cloud filter from filter */ i40e_set_cld_element(filter, &cld_filter.element); @@ -8134,7 +8138,7 @@ int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); } - } else if (filter->dst_ipv4 || + } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) || !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { cld_filter.element.flags = cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); @@ -8928,11 +8932,6 @@ void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) dev_dbg(&pf->pdev->dev, "PFR requested\n"); i40e_handle_reset_warning(pf, lock_acquired); - dev_info(&pf->pdev->dev, - pf->flags & I40E_FLAG_DISABLE_FW_LLDP ? - "FW LLDP is disabled\n" : - "FW LLDP is enabled\n"); - } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) { /* Request a PF Reset * @@ -8940,6 +8939,10 @@ void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) */ i40e_prep_for_reset(pf); i40e_reset_and_rebuild(pf, true, lock_acquired); + dev_info(&pf->pdev->dev, + pf->flags & I40E_FLAG_DISABLE_FW_LLDP ? + "FW LLDP is disabled\n" : + "FW LLDP is enabled\n"); } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { int v; @@ -10462,7 +10465,6 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) int old_recovery_mode_bit = test_bit(__I40E_RECOVERY_MODE, pf->state); struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; struct i40e_hw *hw = &pf->hw; - u8 set_fc_aq_fail = 0; i40e_status ret; u32 val; int v; @@ -10605,13 +10607,6 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) i40e_stat_str(&pf->hw, ret), i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); - /* make sure our flow control settings are restored */ - ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); - if (ret) - dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", - i40e_stat_str(&pf->hw, ret), - i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); - /* Rebuild the VSIs and VEBs that existed before reset. * They are still in our local switch element arrays, so only * need to rebuild the switch model in the HW. @@ -12191,6 +12186,8 @@ i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf) struct i40e_aqc_configure_partition_bw_data bw_data; i40e_status status; + memset(&bw_data, 0, sizeof(bw_data)); + /* Set the valid bit for this PF */ bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; @@ -15198,7 +15195,6 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int err; u32 val; u32 i; - u8 set_fc_aq_fail; err = pci_enable_device_mem(pdev); if (err) @@ -15537,24 +15533,6 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list); - /* Make sure flow control is set according to current settings */ - err = i40e_set_fc(hw, &set_fc_aq_fail, true); - if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) - dev_dbg(&pf->pdev->dev, - "Set fc with err %s aq_err %s on get_phy_cap\n", - i40e_stat_str(hw, err), - i40e_aq_str(hw, hw->aq.asq_last_status)); - if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) - dev_dbg(&pf->pdev->dev, - "Set fc with err %s aq_err %s on set_phy_config\n", - i40e_stat_str(hw, err), - i40e_aq_str(hw, hw->aq.asq_last_status)); - if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) - dev_dbg(&pf->pdev->dev, - "Set fc with err %s aq_err %s on get_link_info\n", - i40e_stat_str(hw, err), - i40e_aq_str(hw, hw->aq.asq_last_status)); - /* if FDIR VSI was set up, start it now */ for (i = 0; i < pf->num_alloc_vsi; i++) { if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { @@ -15611,6 +15589,8 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (err) { dev_info(&pdev->dev, "setup of misc vector failed: %d\n", err); + i40e_cloud_filter_exit(pf); + i40e_fdir_teardown(pf); goto err_vsis; } } diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index f6f1af94cca0..627794b31e33 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -1948,7 +1948,7 @@ void i40e_process_skb_fields(struct i40e_ring *rx_ring, skb_record_rx_queue(skb, rx_ring->queue_index); if (qword & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) { - u16 vlan_tag = rx_desc->wb.qword0.lo_dword.l2tag1; + __le16 vlan_tag = rx_desc->wb.qword0.lo_dword.l2tag1; __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), le16_to_cpu(vlan_tag)); @@ -3223,13 +3223,16 @@ static int i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags, l4_proto = ip.v4->protocol; } else if (*tx_flags & I40E_TX_FLAGS_IPV6) { + int ret; + tunnel |= I40E_TX_CTX_EXT_IP_IPV6; exthdr = ip.hdr + sizeof(*ip.v6); l4_proto = ip.v6->nexthdr; - if (l4.hdr != exthdr) - ipv6_skip_exthdr(skb, exthdr - skb->data, - &l4_proto, &frag_off); + ret = ipv6_skip_exthdr(skb, exthdr - skb->data, + &l4_proto, &frag_off); + if (ret < 0) + return -1; } /* define outer transport */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c index 4f11f7bf75d1..fc32c5019b0f 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c @@ -453,7 +453,7 @@ static void i40e_set_rs_bit(struct i40e_ring *xdp_ring) struct i40e_tx_desc *tx_desc; tx_desc = I40E_TX_DESC(xdp_ring, ntu); - tx_desc->cmd_type_offset_bsz |= (I40E_TX_DESC_CMD_RS << I40E_TXD_QW1_CMD_SHIFT); + tx_desc->cmd_type_offset_bsz |= cpu_to_le64(I40E_TX_DESC_CMD_RS << I40E_TXD_QW1_CMD_SHIFT); } /** diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index dae8280ce17c..357706444dd5 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -454,9 +454,7 @@ struct ice_pf { struct ice_hw_port_stats stats_prev; struct ice_hw hw; u8 stat_prev_loaded:1; /* has previous stats been loaded */ -#ifdef CONFIG_DCB u16 dcbx_cap; -#endif /* CONFIG_DCB */ u32 tx_timeout_count; unsigned long tx_timeout_last_recovery; u32 tx_timeout_recovery_level; diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_nl.c b/drivers/net/ethernet/intel/ice/ice_dcb_nl.c index fcfefad00d1c..468a63f7eff9 100644 --- a/drivers/net/ethernet/intel/ice/ice_dcb_nl.c +++ b/drivers/net/ethernet/intel/ice/ice_dcb_nl.c @@ -134,7 +134,7 @@ ice_dcbnl_getnumtcs(struct net_device *dev, int __always_unused tcid, u8 *num) if (!test_bit(ICE_FLAG_DCB_CAPABLE, pf->flags)) return -EINVAL; - *num = IEEE_8021QAZ_MAX_TCS; + *num = pf->hw.func_caps.common_cap.maxtc; return 0; } @@ -159,6 +159,10 @@ static u8 ice_dcbnl_setdcbx(struct net_device *netdev, u8 mode) struct ice_pf *pf = ice_netdev_to_pf(netdev); struct ice_qos_cfg *qos_cfg; + /* if FW LLDP agent is running, DCBNL not allowed to change mode */ + if (test_bit(ICE_FLAG_FW_LLDP_AGENT, pf->flags)) + return ICE_DCB_NO_HW_CHG; + /* No support for LLD_MANAGED modes or CEE+IEEE */ if ((mode & DCB_CAP_DCBX_LLD_MANAGED) || ((mode & DCB_CAP_DCBX_VER_IEEE) && (mode & DCB_CAP_DCBX_VER_CEE)) || diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index 5636c9b23896..2dcfa685b763 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -8,6 +8,7 @@ #include "ice_fltr.h" #include "ice_lib.h" #include "ice_dcb_lib.h" +#include <net/dcbnl.h> struct ice_stats { char stat_string[ETH_GSTRING_LEN]; @@ -1238,6 +1239,9 @@ static int ice_set_priv_flags(struct net_device *netdev, u32 flags) status = ice_init_pf_dcb(pf, true); if (status) dev_warn(dev, "Fail to init DCB\n"); + + pf->dcbx_cap &= ~DCB_CAP_DCBX_LLD_MANAGED; + pf->dcbx_cap |= DCB_CAP_DCBX_HOST; } else { enum ice_status status; bool dcbx_agent_status; @@ -1280,6 +1284,9 @@ static int ice_set_priv_flags(struct net_device *netdev, u32 flags) if (status) dev_dbg(dev, "Fail to enable MIB change events\n"); + pf->dcbx_cap &= ~DCB_CAP_DCBX_HOST; + pf->dcbx_cap |= DCB_CAP_DCBX_LLD_MANAGED; + ice_nway_reset(netdev); } } @@ -3322,6 +3329,18 @@ ice_get_channels(struct net_device *dev, struct ethtool_channels *ch) } /** + * ice_get_valid_rss_size - return valid number of RSS queues + * @hw: pointer to the HW structure + * @new_size: requested RSS queues + */ +static int ice_get_valid_rss_size(struct ice_hw *hw, int new_size) +{ + struct ice_hw_common_caps *caps = &hw->func_caps.common_cap; + + return min_t(int, new_size, BIT(caps->rss_table_entry_width)); +} + +/** * ice_vsi_set_dflt_rss_lut - set default RSS LUT with requested RSS size * @vsi: VSI to reconfigure RSS LUT on * @req_rss_size: requested range of queue numbers for hashing @@ -3348,14 +3367,10 @@ static int ice_vsi_set_dflt_rss_lut(struct ice_vsi *vsi, int req_rss_size) return -ENOMEM; /* set RSS LUT parameters */ - if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags)) { + if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags)) vsi->rss_size = 1; - } else { - struct ice_hw_common_caps *caps = &hw->func_caps.common_cap; - - vsi->rss_size = min_t(int, req_rss_size, - BIT(caps->rss_table_entry_width)); - } + else + vsi->rss_size = ice_get_valid_rss_size(hw, req_rss_size); /* create/set RSS LUT */ ice_fill_rss_lut(lut, vsi->rss_table_size, vsi->rss_size); @@ -3434,9 +3449,12 @@ static int ice_set_channels(struct net_device *dev, struct ethtool_channels *ch) ice_vsi_recfg_qs(vsi, new_rx, new_tx); - if (new_rx && !netif_is_rxfh_configured(dev)) + if (!netif_is_rxfh_configured(dev)) return ice_vsi_set_dflt_rss_lut(vsi, new_rx); + /* Update rss_size due to change in Rx queues */ + vsi->rss_size = ice_get_valid_rss_size(&pf->hw, new_rx); + return 0; } diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c index bf5fd812ea0e..1f38a8d0c525 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c @@ -1919,6 +1919,29 @@ static int ice_vc_get_ver_msg(struct ice_vf *vf, u8 *msg) } /** + * ice_vc_get_max_frame_size - get max frame size allowed for VF + * @vf: VF used to determine max frame size + * + * Max frame size is determined based on the current port's max frame size and + * whether a port VLAN is configured on this VF. The VF is not aware whether + * it's in a port VLAN so the PF needs to account for this in max frame size + * checks and sending the max frame size to the VF. + */ +static u16 ice_vc_get_max_frame_size(struct ice_vf *vf) +{ + struct ice_vsi *vsi = vf->pf->vsi[vf->lan_vsi_idx]; + struct ice_port_info *pi = vsi->port_info; + u16 max_frame_size; + + max_frame_size = pi->phy.link_info.max_frame_size; + + if (vf->port_vlan_info) + max_frame_size -= VLAN_HLEN; + + return max_frame_size; +} + +/** * ice_vc_get_vf_res_msg * @vf: pointer to the VF info * @msg: pointer to the msg buffer @@ -2000,6 +2023,7 @@ static int ice_vc_get_vf_res_msg(struct ice_vf *vf, u8 *msg) vfres->max_vectors = pf->num_msix_per_vf; vfres->rss_key_size = ICE_VSIQF_HKEY_ARRAY_SIZE; vfres->rss_lut_size = ICE_VSIQF_HLUT_ARRAY_SIZE; + vfres->max_mtu = ice_vc_get_max_frame_size(vf); vfres->vsi_res[0].vsi_id = vf->lan_vsi_num; vfres->vsi_res[0].vsi_type = VIRTCHNL_VSI_SRIOV; @@ -2420,7 +2444,7 @@ static int ice_vc_cfg_promiscuous_mode_msg(struct ice_vf *vf, u8 *msg) } if (!test_bit(ICE_FLAG_VF_TRUE_PROMISC_ENA, pf->flags)) { - bool set_dflt_vsi = !!(info->flags & FLAG_VF_UNICAST_PROMISC); + bool set_dflt_vsi = alluni || allmulti; if (set_dflt_vsi && !ice_is_dflt_vsi_in_use(pf->first_sw)) /* only attempt to set the default forwarding VSI if @@ -2998,6 +3022,8 @@ static int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg) /* copy Rx queue info from VF into VSI */ if (qpi->rxq.ring_len > 0) { + u16 max_frame_size = ice_vc_get_max_frame_size(vf); + num_rxq++; vsi->rx_rings[i]->dma = qpi->rxq.dma_ring_addr; vsi->rx_rings[i]->count = qpi->rxq.ring_len; @@ -3010,7 +3036,7 @@ static int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg) } vsi->rx_buf_len = qpi->rxq.databuffer_size; vsi->rx_rings[i]->rx_buf_len = vsi->rx_buf_len; - if (qpi->rxq.max_pkt_size >= (16 * 1024) || + if (qpi->rxq.max_pkt_size > max_frame_size || qpi->rxq.max_pkt_size < 64) { v_ret = VIRTCHNL_STATUS_ERR_PARAM; goto error_param; @@ -3018,6 +3044,11 @@ static int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg) } vsi->max_frame = qpi->rxq.max_pkt_size; + /* add space for the port VLAN since the VF driver is not + * expected to account for it in the MTU calculation + */ + if (vf->port_vlan_info) + vsi->max_frame += VLAN_HLEN; } /* VF can request to configure less than allocated queues or default diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index 0507369bb54d..1767c60056c5 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -4699,9 +4699,10 @@ static void mvpp2_irqs_deinit(struct mvpp2_port *port) } } -static bool mvpp22_rss_is_supported(void) +static bool mvpp22_rss_is_supported(struct mvpp2_port *port) { - return queue_mode == MVPP2_QDIST_MULTI_MODE; + return (queue_mode == MVPP2_QDIST_MULTI_MODE) && + !(port->flags & MVPP2_F_LOOPBACK); } static int mvpp2_open(struct net_device *dev) @@ -5513,7 +5514,7 @@ static int mvpp2_ethtool_get_rxnfc(struct net_device *dev, struct mvpp2_port *port = netdev_priv(dev); int ret = 0, i, loc = 0; - if (!mvpp22_rss_is_supported()) + if (!mvpp22_rss_is_supported(port)) return -EOPNOTSUPP; switch (info->cmd) { @@ -5548,7 +5549,7 @@ static int mvpp2_ethtool_set_rxnfc(struct net_device *dev, struct mvpp2_port *port = netdev_priv(dev); int ret = 0; - if (!mvpp22_rss_is_supported()) + if (!mvpp22_rss_is_supported(port)) return -EOPNOTSUPP; switch (info->cmd) { @@ -5569,7 +5570,9 @@ static int mvpp2_ethtool_set_rxnfc(struct net_device *dev, static u32 mvpp2_ethtool_get_rxfh_indir_size(struct net_device *dev) { - return mvpp22_rss_is_supported() ? MVPP22_RSS_TABLE_ENTRIES : 0; + struct mvpp2_port *port = netdev_priv(dev); + + return mvpp22_rss_is_supported(port) ? MVPP22_RSS_TABLE_ENTRIES : 0; } static int mvpp2_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, @@ -5578,7 +5581,7 @@ static int mvpp2_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, struct mvpp2_port *port = netdev_priv(dev); int ret = 0; - if (!mvpp22_rss_is_supported()) + if (!mvpp22_rss_is_supported(port)) return -EOPNOTSUPP; if (indir) @@ -5596,7 +5599,7 @@ static int mvpp2_ethtool_set_rxfh(struct net_device *dev, const u32 *indir, struct mvpp2_port *port = netdev_priv(dev); int ret = 0; - if (!mvpp22_rss_is_supported()) + if (!mvpp22_rss_is_supported(port)) return -EOPNOTSUPP; if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_CRC32) @@ -5617,7 +5620,7 @@ static int mvpp2_ethtool_get_rxfh_context(struct net_device *dev, u32 *indir, struct mvpp2_port *port = netdev_priv(dev); int ret = 0; - if (!mvpp22_rss_is_supported()) + if (!mvpp22_rss_is_supported(port)) return -EOPNOTSUPP; if (rss_context >= MVPP22_N_RSS_TABLES) return -EINVAL; @@ -5639,7 +5642,7 @@ static int mvpp2_ethtool_set_rxfh_context(struct net_device *dev, struct mvpp2_port *port = netdev_priv(dev); int ret; - if (!mvpp22_rss_is_supported()) + if (!mvpp22_rss_is_supported(port)) return -EOPNOTSUPP; if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_CRC32) @@ -5956,7 +5959,7 @@ static int mvpp2_port_init(struct mvpp2_port *port) mvpp2_cls_oversize_rxq_set(port); mvpp2_cls_port_config(port); - if (mvpp22_rss_is_supported()) + if (mvpp22_rss_is_supported(port)) mvpp22_port_rss_init(port); /* Provide an initial Rx packet size */ @@ -6861,7 +6864,7 @@ static int mvpp2_port_probe(struct platform_device *pdev, dev->hw_features |= features | NETIF_F_RXCSUM | NETIF_F_GRO | NETIF_F_HW_VLAN_CTAG_FILTER; - if (mvpp22_rss_is_supported()) { + if (mvpp22_rss_is_supported(port)) { dev->hw_features |= NETIF_F_RXHASH; dev->features |= NETIF_F_NTUPLE; } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index 094124b695dc..aa2ca8780b9c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -473,7 +473,7 @@ static ssize_t rvu_dbg_qsize_write(struct file *filp, u16 pcifunc; int ret, lf; - cmd_buf = memdup_user(buffer, count); + cmd_buf = memdup_user(buffer, count + 1); if (IS_ERR(cmd_buf)) return -ENOMEM; diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index ebe1406c6e64..dbec8e187a68 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c @@ -4806,12 +4806,11 @@ static struct net_device *sky2_init_netdev(struct sky2_hw *hw, unsigned port, if (!is_valid_ether_addr(dev->dev_addr)) { struct sockaddr sa = { AF_UNSPEC }; - netdev_warn(dev, - "Invalid MAC address, defaulting to random\n"); + dev_warn(&hw->pdev->dev, "Invalid MAC address, defaulting to random\n"); eth_hw_addr_random(dev); memcpy(sa.sa_data, dev->dev_addr, ETH_ALEN); if (sky2_set_mac_address(dev, &sa)) - netdev_warn(dev, "Failed to set MAC address.\n"); + dev_warn(&hw->pdev->dev, "Failed to set MAC address.\n"); } return dev; diff --git a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c index 394f43add85c..a99e71bc7b3c 100644 --- a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c +++ b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c @@ -4986,6 +4986,7 @@ static int mlx4_do_mirror_rule(struct mlx4_dev *dev, struct res_fs_rule *fs_rule if (!fs_rule->mirr_mbox) { mlx4_err(dev, "rule mirroring mailbox is null\n"); + mlx4_free_cmd_mailbox(dev, mailbox); return -EINVAL; } memcpy(mailbox->buf, fs_rule->mirr_mbox, fs_rule->mirr_mbox_size); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 2f2c352f301e..c568896cfb23 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -237,8 +237,8 @@ static void mlx5_set_driver_version(struct mlx5_core_dev *dev) remaining_size = max_t(int, 0, driver_ver_sz - strlen(string)); snprintf(string + strlen(string), remaining_size, "%u.%u.%u", - (u8)((LINUX_VERSION_CODE >> 16) & 0xff), (u8)((LINUX_VERSION_CODE >> 8) & 0xff), - (u16)(LINUX_VERSION_CODE & 0xffff)); + LINUX_VERSION_MAJOR, LINUX_VERSION_PATCHLEVEL, + LINUX_VERSION_SUBLEVEL); /*Send the command*/ MLX5_SET(set_driver_version_in, in, opcode, diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 0a20dae32184..f704da3f214c 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -2285,14 +2285,14 @@ static void r8168dp_hw_jumbo_disable(struct rtl8169_private *tp) static void r8168e_hw_jumbo_enable(struct rtl8169_private *tp) { - RTL_W8(tp, MaxTxPacketSize, 0x3f); + RTL_W8(tp, MaxTxPacketSize, 0x24); RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0); RTL_W8(tp, Config4, RTL_R8(tp, Config4) | 0x01); } static void r8168e_hw_jumbo_disable(struct rtl8169_private *tp) { - RTL_W8(tp, MaxTxPacketSize, 0x0c); + RTL_W8(tp, MaxTxPacketSize, 0x3f); RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0); RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~0x01); } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c index b7a0c57dfbfb..d23be45a64e5 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c @@ -218,6 +218,7 @@ static int visconti_eth_dwmac_probe(struct platform_device *pdev) goto remove_config; } + spin_lock_init(&dwmac->lock); dwmac->reg = stmmac_res.addr; plat_dat->bsp_priv = dwmac; plat_dat->fix_mac_speed = visconti_eth_fix_mac_speed; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c index 56985542e202..44bb133c3000 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c @@ -316,6 +316,32 @@ static int tc_setup_cbs(struct stmmac_priv *priv, if (!priv->dma_cap.av) return -EOPNOTSUPP; + /* Port Transmit Rate and Speed Divider */ + switch (priv->speed) { + case SPEED_10000: + ptr = 32; + speed_div = 10000000; + break; + case SPEED_5000: + ptr = 32; + speed_div = 5000000; + break; + case SPEED_2500: + ptr = 8; + speed_div = 2500000; + break; + case SPEED_1000: + ptr = 8; + speed_div = 1000000; + break; + case SPEED_100: + ptr = 4; + speed_div = 100000; + break; + default: + return -EOPNOTSUPP; + } + mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; if (mode_to_use == MTL_QUEUE_DCB && qopt->enable) { ret = stmmac_dma_qmode(priv, priv->ioaddr, queue, MTL_QUEUE_AVB); @@ -332,10 +358,6 @@ static int tc_setup_cbs(struct stmmac_priv *priv, priv->plat->tx_queues_cfg[queue].mode_to_use = MTL_QUEUE_DCB; } - /* Port Transmit Rate and Speed Divider */ - ptr = (priv->speed == SPEED_100) ? 4 : 8; - speed_div = (priv->speed == SPEED_100) ? 100000 : 1000000; - /* Final adjustments for HW */ value = div_s64(qopt->idleslope * 1024ll * ptr, speed_div); priv->plat->tx_queues_cfg[queue].idle_slope = value & GENMASK(31, 0); diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index 9a70f05baf6e..39c00f050fbd 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -543,7 +543,6 @@ static int gtp_build_skb_ip4(struct sk_buff *skb, struct net_device *dev, if (!skb_is_gso(skb) && (iph->frag_off & htons(IP_DF)) && mtu < ntohs(iph->tot_len)) { netdev_dbg(dev, "packet too big, fragmentation needed\n"); - memset(IPCB(skb), 0, sizeof(*IPCB(skb))); icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu)); goto err_rt; diff --git a/drivers/net/phy/icplus.c b/drivers/net/phy/icplus.c index 3e431737c1ba..a00a667454a9 100644 --- a/drivers/net/phy/icplus.c +++ b/drivers/net/phy/icplus.c @@ -239,7 +239,7 @@ static int ip101a_g_config_intr_pin(struct phy_device *phydev) oldpage = phy_select_page(phydev, IP101G_DEFAULT_PAGE); if (oldpage < 0) - return oldpage; + goto out; /* configure the RXER/INTR_32 pin of the 32-pin IP101GR if needed: */ switch (priv->sel_intr32) { @@ -314,7 +314,7 @@ static int ip101a_g_read_status(struct phy_device *phydev) oldpage = phy_select_page(phydev, IP101G_DEFAULT_PAGE); if (oldpage < 0) - return oldpage; + goto out; ret = __phy_read(phydev, IP10XX_SPEC_CTRL_STATUS); if (ret < 0) @@ -349,7 +349,8 @@ out: static int ip101a_g_config_mdix(struct phy_device *phydev) { u16 ctrl = 0, ctrl2 = 0; - int oldpage, ret; + int oldpage; + int ret = 0; switch (phydev->mdix_ctrl) { case ETH_TP_MDI: @@ -367,7 +368,7 @@ static int ip101a_g_config_mdix(struct phy_device *phydev) oldpage = phy_select_page(phydev, IP101G_DEFAULT_PAGE); if (oldpage < 0) - return oldpage; + goto out; ret = __phy_modify(phydev, IP10XX_SPEC_CTRL_STATUS, IP101A_G_AUTO_MDIX_DIS, ctrl); diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index 7ec6f70d6a82..a14a00328fa3 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -1303,6 +1303,7 @@ static struct phy_driver ksphy_driver[] = { .driver_data = &ksz8081_type, .probe = kszphy_probe, .config_init = ksz8081_config_init, + .soft_reset = genphy_soft_reset, .config_intr = kszphy_config_intr, .handle_interrupt = kszphy_handle_interrupt, .get_sset_count = kszphy_get_sset_count, diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c index 6c3d8c2abd38..17a050521b86 100644 --- a/drivers/net/usb/qmi_wwan.c +++ b/drivers/net/usb/qmi_wwan.c @@ -1318,6 +1318,7 @@ static const struct usb_device_id products[] = { {QMI_FIXED_INTF(0x19d2, 0x1255, 4)}, {QMI_FIXED_INTF(0x19d2, 0x1256, 4)}, {QMI_FIXED_INTF(0x19d2, 0x1270, 5)}, /* ZTE MF667 */ + {QMI_FIXED_INTF(0x19d2, 0x1275, 3)}, /* ZTE P685M */ {QMI_FIXED_INTF(0x19d2, 0x1401, 2)}, {QMI_FIXED_INTF(0x19d2, 0x1402, 2)}, /* ZTE MF60 */ {QMI_FIXED_INTF(0x19d2, 0x1424, 2)}, diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index 2d7cc63bef89..b246817f3405 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -2632,21 +2632,24 @@ static inline u8 rtl8152_get_speed(struct r8152 *tp) return ocp_read_byte(tp, MCU_TYPE_PLA, PLA_PHYSTATUS); } -static void rtl_set_eee_plus(struct r8152 *tp) +static void rtl_eee_plus_en(struct r8152 *tp, bool enable) { u32 ocp_data; - u8 speed; - speed = rtl8152_get_speed(tp); - if (speed & _10bps) { - ocp_data = ocp_read_word(tp, MCU_TYPE_PLA, PLA_EEEP_CR); + ocp_data = ocp_read_word(tp, MCU_TYPE_PLA, PLA_EEEP_CR); + if (enable) ocp_data |= EEEP_CR_EEEP_TX; - ocp_write_word(tp, MCU_TYPE_PLA, PLA_EEEP_CR, ocp_data); - } else { - ocp_data = ocp_read_word(tp, MCU_TYPE_PLA, PLA_EEEP_CR); + else ocp_data &= ~EEEP_CR_EEEP_TX; - ocp_write_word(tp, MCU_TYPE_PLA, PLA_EEEP_CR, ocp_data); - } + ocp_write_word(tp, MCU_TYPE_PLA, PLA_EEEP_CR, ocp_data); +} + +static void rtl_set_eee_plus(struct r8152 *tp) +{ + if (rtl8152_get_speed(tp) & _10bps) + rtl_eee_plus_en(tp, true); + else + rtl_eee_plus_en(tp, false); } static void rxdy_gated_en(struct r8152 *tp, bool enable) @@ -3150,10 +3153,22 @@ static void r8153b_ups_flags(struct r8152 *tp) ocp_write_dword(tp, MCU_TYPE_USB, USB_UPS_FLAGS, ups_flags); } -static void r8153b_green_en(struct r8152 *tp, bool enable) +static void rtl_green_en(struct r8152 *tp, bool enable) { u16 data; + data = sram_read(tp, SRAM_GREEN_CFG); + if (enable) + data |= GREEN_ETH_EN; + else + data &= ~GREEN_ETH_EN; + sram_write(tp, SRAM_GREEN_CFG, data); + + tp->ups_info.green = enable; +} + +static void r8153b_green_en(struct r8152 *tp, bool enable) +{ if (enable) { sram_write(tp, 0x8045, 0); /* 10M abiq&ldvbias */ sram_write(tp, 0x804d, 0x1222); /* 100M short abiq&ldvbias */ @@ -3164,11 +3179,7 @@ static void r8153b_green_en(struct r8152 *tp, bool enable) sram_write(tp, 0x805d, 0x2444); /* 1000M short abiq&ldvbias */ } - data = sram_read(tp, SRAM_GREEN_CFG); - data |= GREEN_ETH_EN; - sram_write(tp, SRAM_GREEN_CFG, data); - - tp->ups_info.green = enable; + rtl_green_en(tp, true); } static u16 r8153_phy_status(struct r8152 *tp, u16 desired) @@ -3360,7 +3371,7 @@ static void rtl8153b_runtime_enable(struct r8152 *tp, bool enable) r8153b_ups_en(tp, false); r8153_queue_wake(tp, false); rtl_runtime_suspend_enable(tp, false); - if (tp->udev->speed != USB_SPEED_HIGH) + if (tp->udev->speed >= USB_SPEED_SUPER) r8153b_u1u2en(tp, true); } } @@ -5056,7 +5067,7 @@ static void rtl8153b_up(struct r8152 *tp) r8153_aldps_en(tp, true); - if (tp->udev->speed != USB_SPEED_HIGH) + if (tp->udev->speed >= USB_SPEED_SUPER) r8153b_u1u2en(tp, true); } @@ -5572,8 +5583,9 @@ static void r8153b_init(struct r8152 *tp) ocp_data |= POLL_LINK_CHG; ocp_write_word(tp, MCU_TYPE_PLA, PLA_EXTRA_STATUS, ocp_data); - if (tp->udev->speed != USB_SPEED_HIGH) + if (tp->udev->speed >= USB_SPEED_SUPER) r8153b_u1u2en(tp, true); + usb_enable_lpm(tp->udev); /* MAC clock speed down */ @@ -5756,6 +5768,9 @@ static int rtl8152_runtime_suspend(struct r8152 *tp) struct net_device *netdev = tp->netdev; int ret = 0; + if (!tp->rtl_ops.autosuspend_en) + return -EBUSY; + set_bit(SELECTIVE_SUSPEND, &tp->flags); smp_mb__after_atomic(); @@ -6155,6 +6170,11 @@ rtl_ethtool_get_eee(struct net_device *net, struct ethtool_eee *edata) struct r8152 *tp = netdev_priv(net); int ret; + if (!tp->rtl_ops.eee_get) { + ret = -EOPNOTSUPP; + goto out; + } + ret = usb_autopm_get_interface(tp->intf); if (ret < 0) goto out; @@ -6177,6 +6197,11 @@ rtl_ethtool_set_eee(struct net_device *net, struct ethtool_eee *edata) struct r8152 *tp = netdev_priv(net); int ret; + if (!tp->rtl_ops.eee_set) { + ret = -EOPNOTSUPP; + goto out; + } + ret = usb_autopm_get_interface(tp->intf); if (ret < 0) goto out; @@ -6576,7 +6601,7 @@ static int rtl_ops_init(struct r8152 *tp) default: ret = -ENODEV; - netif_err(tp, probe, tp->netdev, "Unknown Device\n"); + dev_err(&tp->intf->dev, "Unknown Device\n"); break; } @@ -6833,7 +6858,7 @@ static int rtl8152_probe(struct usb_interface *intf, ret = register_netdev(netdev); if (ret != 0) { - netif_err(tp, probe, netdev, "couldn't register the device\n"); + dev_err(&intf->dev, "couldn't register the device\n"); goto out1; } diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index ba8e63792549..82e520d2cb12 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -729,6 +729,7 @@ static struct sk_buff *receive_small(struct net_device *dev, fallthrough; case XDP_ABORTED: trace_xdp_exception(vi->dev, xdp_prog, act); + goto err_xdp; case XDP_DROP: goto err_xdp; } diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c index 3929e437382b..666dd201c3d5 100644 --- a/drivers/net/vxlan.c +++ b/drivers/net/vxlan.c @@ -4721,7 +4721,6 @@ static void vxlan_destroy_tunnels(struct net *net, struct list_head *head) struct vxlan_net *vn = net_generic(net, vxlan_net_id); struct vxlan_dev *vxlan, *next; struct net_device *dev, *aux; - unsigned int h; for_each_netdev_safe(net, dev, aux) if (dev->rtnl_link_ops == &vxlan_link_ops) @@ -4735,14 +4734,13 @@ static void vxlan_destroy_tunnels(struct net *net, struct list_head *head) unregister_netdevice_queue(vxlan->dev, head); } - for (h = 0; h < PORT_HASH_SIZE; ++h) - WARN_ON_ONCE(!hlist_empty(&vn->sock_list[h])); } static void __net_exit vxlan_exit_batch_net(struct list_head *net_list) { struct net *net; LIST_HEAD(list); + unsigned int h; rtnl_lock(); list_for_each_entry(net, net_list, exit_list) { @@ -4755,6 +4753,13 @@ static void __net_exit vxlan_exit_batch_net(struct list_head *net_list) unregister_netdevice_many(&list); rtnl_unlock(); + + list_for_each_entry(net, net_list, exit_list) { + struct vxlan_net *vn = net_generic(net, vxlan_net_id); + + for (h = 0; h < PORT_HASH_SIZE; ++h) + WARN_ON_ONCE(!hlist_empty(&vn->sock_list[h])); + } } static struct pernet_operations vxlan_net_ops = { diff --git a/drivers/net/wireguard/device.c b/drivers/net/wireguard/device.c index a3ed49cd95c3..551ddaaaf540 100644 --- a/drivers/net/wireguard/device.c +++ b/drivers/net/wireguard/device.c @@ -138,7 +138,7 @@ static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) else if (skb->protocol == htons(ETH_P_IPV6)) net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n", dev->name, &ipv6_hdr(skb)->daddr); - goto err; + goto err_icmp; } family = READ_ONCE(peer->endpoint.addr.sa_family); @@ -157,7 +157,7 @@ static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) } else { struct sk_buff *segs = skb_gso_segment(skb, 0); - if (unlikely(IS_ERR(segs))) { + if (IS_ERR(segs)) { ret = PTR_ERR(segs); goto err_peer; } @@ -201,12 +201,13 @@ static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) err_peer: wg_peer_put(peer); -err: - ++dev->stats.tx_errors; +err_icmp: if (skb->protocol == htons(ETH_P_IP)) icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); else if (skb->protocol == htons(ETH_P_IPV6)) icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); +err: + ++dev->stats.tx_errors; kfree_skb(skb); return ret; } @@ -234,8 +235,8 @@ static void wg_destruct(struct net_device *dev) destroy_workqueue(wg->handshake_receive_wq); destroy_workqueue(wg->handshake_send_wq); destroy_workqueue(wg->packet_crypt_wq); - wg_packet_queue_free(&wg->decrypt_queue, true); - wg_packet_queue_free(&wg->encrypt_queue, true); + wg_packet_queue_free(&wg->decrypt_queue); + wg_packet_queue_free(&wg->encrypt_queue); rcu_barrier(); /* Wait for all the peers to be actually freed. */ wg_ratelimiter_uninit(); memzero_explicit(&wg->static_identity, sizeof(wg->static_identity)); @@ -337,12 +338,12 @@ static int wg_newlink(struct net *src_net, struct net_device *dev, goto err_destroy_handshake_send; ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker, - true, MAX_QUEUED_PACKETS); + MAX_QUEUED_PACKETS); if (ret < 0) goto err_destroy_packet_crypt; ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker, - true, MAX_QUEUED_PACKETS); + MAX_QUEUED_PACKETS); if (ret < 0) goto err_free_encrypt_queue; @@ -367,9 +368,9 @@ static int wg_newlink(struct net *src_net, struct net_device *dev, err_uninit_ratelimiter: wg_ratelimiter_uninit(); err_free_decrypt_queue: - wg_packet_queue_free(&wg->decrypt_queue, true); + wg_packet_queue_free(&wg->decrypt_queue); err_free_encrypt_queue: - wg_packet_queue_free(&wg->encrypt_queue, true); + wg_packet_queue_free(&wg->encrypt_queue); err_destroy_packet_crypt: destroy_workqueue(wg->packet_crypt_wq); err_destroy_handshake_send: diff --git a/drivers/net/wireguard/device.h b/drivers/net/wireguard/device.h index 4d0144e16947..854bc3d97150 100644 --- a/drivers/net/wireguard/device.h +++ b/drivers/net/wireguard/device.h @@ -27,13 +27,14 @@ struct multicore_worker { struct crypt_queue { struct ptr_ring ring; - union { - struct { - struct multicore_worker __percpu *worker; - int last_cpu; - }; - struct work_struct work; - }; + struct multicore_worker __percpu *worker; + int last_cpu; +}; + +struct prev_queue { + struct sk_buff *head, *tail, *peeked; + struct { struct sk_buff *next, *prev; } empty; // Match first 2 members of struct sk_buff. + atomic_t count; }; struct wg_device { diff --git a/drivers/net/wireguard/peer.c b/drivers/net/wireguard/peer.c index b3b6370e6b95..cd5cb0292cb6 100644 --- a/drivers/net/wireguard/peer.c +++ b/drivers/net/wireguard/peer.c @@ -32,27 +32,22 @@ struct wg_peer *wg_peer_create(struct wg_device *wg, peer = kzalloc(sizeof(*peer), GFP_KERNEL); if (unlikely(!peer)) return ERR_PTR(ret); - peer->device = wg; + if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)) + goto err; + peer->device = wg; wg_noise_handshake_init(&peer->handshake, &wg->static_identity, public_key, preshared_key, peer); - if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)) - goto err_1; - if (wg_packet_queue_init(&peer->tx_queue, wg_packet_tx_worker, false, - MAX_QUEUED_PACKETS)) - goto err_2; - if (wg_packet_queue_init(&peer->rx_queue, NULL, false, - MAX_QUEUED_PACKETS)) - goto err_3; - peer->internal_id = atomic64_inc_return(&peer_counter); peer->serial_work_cpu = nr_cpumask_bits; wg_cookie_init(&peer->latest_cookie); wg_timers_init(peer); wg_cookie_checker_precompute_peer_keys(peer); spin_lock_init(&peer->keypairs.keypair_update_lock); - INIT_WORK(&peer->transmit_handshake_work, - wg_packet_handshake_send_worker); + INIT_WORK(&peer->transmit_handshake_work, wg_packet_handshake_send_worker); + INIT_WORK(&peer->transmit_packet_work, wg_packet_tx_worker); + wg_prev_queue_init(&peer->tx_queue); + wg_prev_queue_init(&peer->rx_queue); rwlock_init(&peer->endpoint_lock); kref_init(&peer->refcount); skb_queue_head_init(&peer->staged_packet_queue); @@ -68,11 +63,7 @@ struct wg_peer *wg_peer_create(struct wg_device *wg, pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id); return peer; -err_3: - wg_packet_queue_free(&peer->tx_queue, false); -err_2: - dst_cache_destroy(&peer->endpoint_cache); -err_1: +err: kfree(peer); return ERR_PTR(ret); } @@ -197,8 +188,7 @@ static void rcu_release(struct rcu_head *rcu) struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu); dst_cache_destroy(&peer->endpoint_cache); - wg_packet_queue_free(&peer->rx_queue, false); - wg_packet_queue_free(&peer->tx_queue, false); + WARN_ON(wg_prev_queue_peek(&peer->tx_queue) || wg_prev_queue_peek(&peer->rx_queue)); /* The final zeroing takes care of clearing any remaining handshake key * material and other potentially sensitive information. diff --git a/drivers/net/wireguard/peer.h b/drivers/net/wireguard/peer.h index 23af40922997..8d53b687a1d1 100644 --- a/drivers/net/wireguard/peer.h +++ b/drivers/net/wireguard/peer.h @@ -36,16 +36,17 @@ struct endpoint { struct wg_peer { struct wg_device *device; - struct crypt_queue tx_queue, rx_queue; + struct prev_queue tx_queue, rx_queue; struct sk_buff_head staged_packet_queue; int serial_work_cpu; + bool is_dead; struct noise_keypairs keypairs; struct endpoint endpoint; struct dst_cache endpoint_cache; rwlock_t endpoint_lock; struct noise_handshake handshake; atomic64_t last_sent_handshake; - struct work_struct transmit_handshake_work, clear_peer_work; + struct work_struct transmit_handshake_work, clear_peer_work, transmit_packet_work; struct cookie latest_cookie; struct hlist_node pubkey_hash; u64 rx_bytes, tx_bytes; @@ -61,9 +62,8 @@ struct wg_peer { struct rcu_head rcu; struct list_head peer_list; struct list_head allowedips_list; - u64 internal_id; struct napi_struct napi; - bool is_dead; + u64 internal_id; }; struct wg_peer *wg_peer_create(struct wg_device *wg, diff --git a/drivers/net/wireguard/queueing.c b/drivers/net/wireguard/queueing.c index 71b8e80b58e1..48e7b982a307 100644 --- a/drivers/net/wireguard/queueing.c +++ b/drivers/net/wireguard/queueing.c @@ -9,8 +9,7 @@ struct multicore_worker __percpu * wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr) { int cpu; - struct multicore_worker __percpu *worker = - alloc_percpu(struct multicore_worker); + struct multicore_worker __percpu *worker = alloc_percpu(struct multicore_worker); if (!worker) return NULL; @@ -23,7 +22,7 @@ wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr) } int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, - bool multicore, unsigned int len) + unsigned int len) { int ret; @@ -31,25 +30,78 @@ int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, ret = ptr_ring_init(&queue->ring, len, GFP_KERNEL); if (ret) return ret; - if (function) { - if (multicore) { - queue->worker = wg_packet_percpu_multicore_worker_alloc( - function, queue); - if (!queue->worker) { - ptr_ring_cleanup(&queue->ring, NULL); - return -ENOMEM; - } - } else { - INIT_WORK(&queue->work, function); - } + queue->worker = wg_packet_percpu_multicore_worker_alloc(function, queue); + if (!queue->worker) { + ptr_ring_cleanup(&queue->ring, NULL); + return -ENOMEM; } return 0; } -void wg_packet_queue_free(struct crypt_queue *queue, bool multicore) +void wg_packet_queue_free(struct crypt_queue *queue) { - if (multicore) - free_percpu(queue->worker); + free_percpu(queue->worker); WARN_ON(!__ptr_ring_empty(&queue->ring)); ptr_ring_cleanup(&queue->ring, NULL); } + +#define NEXT(skb) ((skb)->prev) +#define STUB(queue) ((struct sk_buff *)&queue->empty) + +void wg_prev_queue_init(struct prev_queue *queue) +{ + NEXT(STUB(queue)) = NULL; + queue->head = queue->tail = STUB(queue); + queue->peeked = NULL; + atomic_set(&queue->count, 0); + BUILD_BUG_ON( + offsetof(struct sk_buff, next) != offsetof(struct prev_queue, empty.next) - + offsetof(struct prev_queue, empty) || + offsetof(struct sk_buff, prev) != offsetof(struct prev_queue, empty.prev) - + offsetof(struct prev_queue, empty)); +} + +static void __wg_prev_queue_enqueue(struct prev_queue *queue, struct sk_buff *skb) +{ + WRITE_ONCE(NEXT(skb), NULL); + WRITE_ONCE(NEXT(xchg_release(&queue->head, skb)), skb); +} + +bool wg_prev_queue_enqueue(struct prev_queue *queue, struct sk_buff *skb) +{ + if (!atomic_add_unless(&queue->count, 1, MAX_QUEUED_PACKETS)) + return false; + __wg_prev_queue_enqueue(queue, skb); + return true; +} + +struct sk_buff *wg_prev_queue_dequeue(struct prev_queue *queue) +{ + struct sk_buff *tail = queue->tail, *next = smp_load_acquire(&NEXT(tail)); + + if (tail == STUB(queue)) { + if (!next) + return NULL; + queue->tail = next; + tail = next; + next = smp_load_acquire(&NEXT(next)); + } + if (next) { + queue->tail = next; + atomic_dec(&queue->count); + return tail; + } + if (tail != READ_ONCE(queue->head)) + return NULL; + __wg_prev_queue_enqueue(queue, STUB(queue)); + next = smp_load_acquire(&NEXT(tail)); + if (next) { + queue->tail = next; + atomic_dec(&queue->count); + return tail; + } + return NULL; +} + +#undef NEXT +#undef STUB diff --git a/drivers/net/wireguard/queueing.h b/drivers/net/wireguard/queueing.h index dfb674e03076..4ef2944a68bc 100644 --- a/drivers/net/wireguard/queueing.h +++ b/drivers/net/wireguard/queueing.h @@ -17,12 +17,13 @@ struct wg_device; struct wg_peer; struct multicore_worker; struct crypt_queue; +struct prev_queue; struct sk_buff; /* queueing.c APIs: */ int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, - bool multicore, unsigned int len); -void wg_packet_queue_free(struct crypt_queue *queue, bool multicore); + unsigned int len); +void wg_packet_queue_free(struct crypt_queue *queue); struct multicore_worker __percpu * wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr); @@ -135,8 +136,31 @@ static inline int wg_cpumask_next_online(int *next) return cpu; } +void wg_prev_queue_init(struct prev_queue *queue); + +/* Multi producer */ +bool wg_prev_queue_enqueue(struct prev_queue *queue, struct sk_buff *skb); + +/* Single consumer */ +struct sk_buff *wg_prev_queue_dequeue(struct prev_queue *queue); + +/* Single consumer */ +static inline struct sk_buff *wg_prev_queue_peek(struct prev_queue *queue) +{ + if (queue->peeked) + return queue->peeked; + queue->peeked = wg_prev_queue_dequeue(queue); + return queue->peeked; +} + +/* Single consumer */ +static inline void wg_prev_queue_drop_peeked(struct prev_queue *queue) +{ + queue->peeked = NULL; +} + static inline int wg_queue_enqueue_per_device_and_peer( - struct crypt_queue *device_queue, struct crypt_queue *peer_queue, + struct crypt_queue *device_queue, struct prev_queue *peer_queue, struct sk_buff *skb, struct workqueue_struct *wq, int *next_cpu) { int cpu; @@ -145,8 +169,9 @@ static inline int wg_queue_enqueue_per_device_and_peer( /* We first queue this up for the peer ingestion, but the consumer * will wait for the state to change to CRYPTED or DEAD before. */ - if (unlikely(ptr_ring_produce_bh(&peer_queue->ring, skb))) + if (unlikely(!wg_prev_queue_enqueue(peer_queue, skb))) return -ENOSPC; + /* Then we queue it up in the device queue, which consumes the * packet as soon as it can. */ @@ -157,9 +182,7 @@ static inline int wg_queue_enqueue_per_device_and_peer( return 0; } -static inline void wg_queue_enqueue_per_peer(struct crypt_queue *queue, - struct sk_buff *skb, - enum packet_state state) +static inline void wg_queue_enqueue_per_peer_tx(struct sk_buff *skb, enum packet_state state) { /* We take a reference, because as soon as we call atomic_set, the * peer can be freed from below us. @@ -167,14 +190,12 @@ static inline void wg_queue_enqueue_per_peer(struct crypt_queue *queue, struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb)); atomic_set_release(&PACKET_CB(skb)->state, state); - queue_work_on(wg_cpumask_choose_online(&peer->serial_work_cpu, - peer->internal_id), - peer->device->packet_crypt_wq, &queue->work); + queue_work_on(wg_cpumask_choose_online(&peer->serial_work_cpu, peer->internal_id), + peer->device->packet_crypt_wq, &peer->transmit_packet_work); wg_peer_put(peer); } -static inline void wg_queue_enqueue_per_peer_napi(struct sk_buff *skb, - enum packet_state state) +static inline void wg_queue_enqueue_per_peer_rx(struct sk_buff *skb, enum packet_state state) { /* We take a reference, because as soon as we call atomic_set, the * peer can be freed from below us. diff --git a/drivers/net/wireguard/receive.c b/drivers/net/wireguard/receive.c index 2c9551ea6dc7..7dc84bcca261 100644 --- a/drivers/net/wireguard/receive.c +++ b/drivers/net/wireguard/receive.c @@ -444,7 +444,6 @@ packet_processed: int wg_packet_rx_poll(struct napi_struct *napi, int budget) { struct wg_peer *peer = container_of(napi, struct wg_peer, napi); - struct crypt_queue *queue = &peer->rx_queue; struct noise_keypair *keypair; struct endpoint endpoint; enum packet_state state; @@ -455,11 +454,10 @@ int wg_packet_rx_poll(struct napi_struct *napi, int budget) if (unlikely(budget <= 0)) return 0; - while ((skb = __ptr_ring_peek(&queue->ring)) != NULL && + while ((skb = wg_prev_queue_peek(&peer->rx_queue)) != NULL && (state = atomic_read_acquire(&PACKET_CB(skb)->state)) != PACKET_STATE_UNCRYPTED) { - __ptr_ring_discard_one(&queue->ring); - peer = PACKET_PEER(skb); + wg_prev_queue_drop_peeked(&peer->rx_queue); keypair = PACKET_CB(skb)->keypair; free = true; @@ -508,7 +506,7 @@ void wg_packet_decrypt_worker(struct work_struct *work) enum packet_state state = likely(decrypt_packet(skb, PACKET_CB(skb)->keypair)) ? PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; - wg_queue_enqueue_per_peer_napi(skb, state); + wg_queue_enqueue_per_peer_rx(skb, state); if (need_resched()) cond_resched(); } @@ -531,12 +529,10 @@ static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb) if (unlikely(READ_ONCE(peer->is_dead))) goto err; - ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, - &peer->rx_queue, skb, - wg->packet_crypt_wq, - &wg->decrypt_queue.last_cpu); + ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, &peer->rx_queue, skb, + wg->packet_crypt_wq, &wg->decrypt_queue.last_cpu); if (unlikely(ret == -EPIPE)) - wg_queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD); + wg_queue_enqueue_per_peer_rx(skb, PACKET_STATE_DEAD); if (likely(!ret || ret == -EPIPE)) { rcu_read_unlock_bh(); return; diff --git a/drivers/net/wireguard/send.c b/drivers/net/wireguard/send.c index f74b9341ab0f..5368f7c35b4b 100644 --- a/drivers/net/wireguard/send.c +++ b/drivers/net/wireguard/send.c @@ -239,8 +239,7 @@ void wg_packet_send_keepalive(struct wg_peer *peer) wg_packet_send_staged_packets(peer); } -static void wg_packet_create_data_done(struct sk_buff *first, - struct wg_peer *peer) +static void wg_packet_create_data_done(struct wg_peer *peer, struct sk_buff *first) { struct sk_buff *skb, *next; bool is_keepalive, data_sent = false; @@ -262,22 +261,19 @@ static void wg_packet_create_data_done(struct sk_buff *first, void wg_packet_tx_worker(struct work_struct *work) { - struct crypt_queue *queue = container_of(work, struct crypt_queue, - work); + struct wg_peer *peer = container_of(work, struct wg_peer, transmit_packet_work); struct noise_keypair *keypair; enum packet_state state; struct sk_buff *first; - struct wg_peer *peer; - while ((first = __ptr_ring_peek(&queue->ring)) != NULL && + while ((first = wg_prev_queue_peek(&peer->tx_queue)) != NULL && (state = atomic_read_acquire(&PACKET_CB(first)->state)) != PACKET_STATE_UNCRYPTED) { - __ptr_ring_discard_one(&queue->ring); - peer = PACKET_PEER(first); + wg_prev_queue_drop_peeked(&peer->tx_queue); keypair = PACKET_CB(first)->keypair; if (likely(state == PACKET_STATE_CRYPTED)) - wg_packet_create_data_done(first, peer); + wg_packet_create_data_done(peer, first); else kfree_skb_list(first); @@ -306,16 +302,14 @@ void wg_packet_encrypt_worker(struct work_struct *work) break; } } - wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first, - state); + wg_queue_enqueue_per_peer_tx(first, state); if (need_resched()) cond_resched(); } } -static void wg_packet_create_data(struct sk_buff *first) +static void wg_packet_create_data(struct wg_peer *peer, struct sk_buff *first) { - struct wg_peer *peer = PACKET_PEER(first); struct wg_device *wg = peer->device; int ret = -EINVAL; @@ -323,13 +317,10 @@ static void wg_packet_create_data(struct sk_buff *first) if (unlikely(READ_ONCE(peer->is_dead))) goto err; - ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, - &peer->tx_queue, first, - wg->packet_crypt_wq, - &wg->encrypt_queue.last_cpu); + ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, &peer->tx_queue, first, + wg->packet_crypt_wq, &wg->encrypt_queue.last_cpu); if (unlikely(ret == -EPIPE)) - wg_queue_enqueue_per_peer(&peer->tx_queue, first, - PACKET_STATE_DEAD); + wg_queue_enqueue_per_peer_tx(first, PACKET_STATE_DEAD); err: rcu_read_unlock_bh(); if (likely(!ret || ret == -EPIPE)) @@ -393,7 +384,7 @@ void wg_packet_send_staged_packets(struct wg_peer *peer) packets.prev->next = NULL; wg_peer_get(keypair->entry.peer); PACKET_CB(packets.next)->keypair = keypair; - wg_packet_create_data(packets.next); + wg_packet_create_data(peer, packets.next); return; out_invalid: diff --git a/drivers/net/wireguard/socket.c b/drivers/net/wireguard/socket.c index 410b318e57fb..d9ad850daa79 100644 --- a/drivers/net/wireguard/socket.c +++ b/drivers/net/wireguard/socket.c @@ -53,7 +53,7 @@ static int send4(struct wg_device *wg, struct sk_buff *skb, if (unlikely(!inet_confirm_addr(sock_net(sock), NULL, 0, fl.saddr, RT_SCOPE_HOST))) { endpoint->src4.s_addr = 0; - *(__force __be32 *)&endpoint->src_if4 = 0; + endpoint->src_if4 = 0; fl.saddr = 0; if (cache) dst_cache_reset(cache); @@ -63,7 +63,7 @@ static int send4(struct wg_device *wg, struct sk_buff *skb, PTR_ERR(rt) == -EINVAL) || (!IS_ERR(rt) && rt->dst.dev->ifindex != endpoint->src_if4)))) { endpoint->src4.s_addr = 0; - *(__force __be32 *)&endpoint->src_if4 = 0; + endpoint->src_if4 = 0; fl.saddr = 0; if (cache) dst_cache_reset(cache); @@ -71,7 +71,7 @@ static int send4(struct wg_device *wg, struct sk_buff *skb, ip_rt_put(rt); rt = ip_route_output_flow(sock_net(sock), &fl, sock); } - if (unlikely(IS_ERR(rt))) { + if (IS_ERR(rt)) { ret = PTR_ERR(rt); net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", wg->dev->name, &endpoint->addr, ret); @@ -138,7 +138,7 @@ static int send6(struct wg_device *wg, struct sk_buff *skb, } dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sock), sock, &fl, NULL); - if (unlikely(IS_ERR(dst))) { + if (IS_ERR(dst)) { ret = PTR_ERR(dst); net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", wg->dev->name, &endpoint->addr, ret); diff --git a/drivers/net/wireless/intel/iwlwifi/fw/file.h b/drivers/net/wireless/intel/iwlwifi/fw/file.h index f2e7b735d211..35dffcaf5aba 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/file.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/file.h @@ -870,7 +870,7 @@ struct iwl_fw_dbg_trigger_time_event { * tx_bar: tid bitmap to configure on what tid the trigger should occur * when a BAR is send (for an Rx BlocAck session). * frame_timeout: tid bitmap to configure on what tid the trigger should occur - * when a frame times out in the reodering buffer. + * when a frame times out in the reordering buffer. */ struct iwl_fw_dbg_trigger_ba { __le16 rx_ba_start; diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index e5c73f819662..6afb5ca999c2 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c @@ -1343,11 +1343,21 @@ int xenvif_tx_action(struct xenvif_queue *queue, int budget) return 0; gnttab_batch_copy(queue->tx_copy_ops, nr_cops); - if (nr_mops != 0) + if (nr_mops != 0) { ret = gnttab_map_refs(queue->tx_map_ops, NULL, queue->pages_to_map, nr_mops); + if (ret) { + unsigned int i; + + netdev_err(queue->vif->dev, "Map fail: nr %u ret %d\n", + nr_mops, ret); + for (i = 0; i < nr_mops; ++i) + WARN_ON_ONCE(queue->tx_map_ops[i].status == + GNTST_okay); + } + } work_done = xenvif_tx_submit(queue); diff --git a/drivers/nfc/microread/mei.c b/drivers/nfc/microread/mei.c index 5dad8847a9b3..8fa7771085eb 100644 --- a/drivers/nfc/microread/mei.c +++ b/drivers/nfc/microread/mei.c @@ -44,15 +44,13 @@ static int microread_mei_probe(struct mei_cl_device *cldev, return 0; } -static int microread_mei_remove(struct mei_cl_device *cldev) +static void microread_mei_remove(struct mei_cl_device *cldev) { struct nfc_mei_phy *phy = mei_cldev_get_drvdata(cldev); microread_remove(phy->hdev); nfc_mei_phy_free(phy); - - return 0; } static struct mei_cl_device_id microread_mei_tbl[] = { diff --git a/drivers/nfc/pn544/mei.c b/drivers/nfc/pn544/mei.c index 579bc599f545..5c10aac085a4 100644 --- a/drivers/nfc/pn544/mei.c +++ b/drivers/nfc/pn544/mei.c @@ -42,7 +42,7 @@ static int pn544_mei_probe(struct mei_cl_device *cldev, return 0; } -static int pn544_mei_remove(struct mei_cl_device *cldev) +static void pn544_mei_remove(struct mei_cl_device *cldev) { struct nfc_mei_phy *phy = mei_cldev_get_drvdata(cldev); @@ -51,8 +51,6 @@ static int pn544_mei_remove(struct mei_cl_device *cldev) pn544_hci_remove(phy->hdev); nfc_mei_phy_free(phy); - - return 0; } static struct mei_cl_device_id pn544_mei_tbl[] = { diff --git a/drivers/ntb/hw/Kconfig b/drivers/ntb/hw/Kconfig index e77c587060ff..c325be526b80 100644 --- a/drivers/ntb/hw/Kconfig +++ b/drivers/ntb/hw/Kconfig @@ -2,4 +2,5 @@ source "drivers/ntb/hw/amd/Kconfig" source "drivers/ntb/hw/idt/Kconfig" source "drivers/ntb/hw/intel/Kconfig" +source "drivers/ntb/hw/epf/Kconfig" source "drivers/ntb/hw/mscc/Kconfig" diff --git a/drivers/ntb/hw/Makefile b/drivers/ntb/hw/Makefile index 4714d6238845..223ca592b5f9 100644 --- a/drivers/ntb/hw/Makefile +++ b/drivers/ntb/hw/Makefile @@ -2,4 +2,5 @@ obj-$(CONFIG_NTB_AMD) += amd/ obj-$(CONFIG_NTB_IDT) += idt/ obj-$(CONFIG_NTB_INTEL) += intel/ +obj-$(CONFIG_NTB_EPF) += epf/ obj-$(CONFIG_NTB_SWITCHTEC) += mscc/ diff --git a/drivers/ntb/hw/epf/Kconfig b/drivers/ntb/hw/epf/Kconfig new file mode 100644 index 000000000000..6197d1aab344 --- /dev/null +++ b/drivers/ntb/hw/epf/Kconfig @@ -0,0 +1,6 @@ +config NTB_EPF + tristate "Generic EPF Non-Transparent Bridge support" + depends on m + help + This driver supports EPF NTB on configurable endpoint. + If unsure, say N. diff --git a/drivers/ntb/hw/epf/Makefile b/drivers/ntb/hw/epf/Makefile new file mode 100644 index 000000000000..2f560a422bc6 --- /dev/null +++ b/drivers/ntb/hw/epf/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_NTB_EPF) += ntb_hw_epf.o diff --git a/drivers/ntb/hw/epf/ntb_hw_epf.c b/drivers/ntb/hw/epf/ntb_hw_epf.c new file mode 100644 index 000000000000..b019755e4e21 --- /dev/null +++ b/drivers/ntb/hw/epf/ntb_hw_epf.c @@ -0,0 +1,753 @@ +// SPDX-License-Identifier: GPL-2.0 +/** + * Host side endpoint driver to implement Non-Transparent Bridge functionality + * + * Copyright (C) 2020 Texas Instruments + * Author: Kishon Vijay Abraham I <kishon@ti.com> + */ + +#include <linux/delay.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include <linux/ntb.h> + +#define NTB_EPF_COMMAND 0x0 +#define CMD_CONFIGURE_DOORBELL 1 +#define CMD_TEARDOWN_DOORBELL 2 +#define CMD_CONFIGURE_MW 3 +#define CMD_TEARDOWN_MW 4 +#define CMD_LINK_UP 5 +#define CMD_LINK_DOWN 6 + +#define NTB_EPF_ARGUMENT 0x4 +#define MSIX_ENABLE BIT(16) + +#define NTB_EPF_CMD_STATUS 0x8 +#define COMMAND_STATUS_OK 1 +#define COMMAND_STATUS_ERROR 2 + +#define NTB_EPF_LINK_STATUS 0x0A +#define LINK_STATUS_UP BIT(0) + +#define NTB_EPF_TOPOLOGY 0x0C +#define NTB_EPF_LOWER_ADDR 0x10 +#define NTB_EPF_UPPER_ADDR 0x14 +#define NTB_EPF_LOWER_SIZE 0x18 +#define NTB_EPF_UPPER_SIZE 0x1C +#define NTB_EPF_MW_COUNT 0x20 +#define NTB_EPF_MW1_OFFSET 0x24 +#define NTB_EPF_SPAD_OFFSET 0x28 +#define NTB_EPF_SPAD_COUNT 0x2C +#define NTB_EPF_DB_ENTRY_SIZE 0x30 +#define NTB_EPF_DB_DATA(n) (0x34 + (n) * 4) +#define NTB_EPF_DB_OFFSET(n) (0xB4 + (n) * 4) + +#define NTB_EPF_MIN_DB_COUNT 3 +#define NTB_EPF_MAX_DB_COUNT 31 +#define NTB_EPF_MW_OFFSET 2 + +#define NTB_EPF_COMMAND_TIMEOUT 1000 /* 1 Sec */ + +enum pci_barno { + BAR_0, + BAR_1, + BAR_2, + BAR_3, + BAR_4, + BAR_5, +}; + +struct ntb_epf_dev { + struct ntb_dev ntb; + struct device *dev; + /* Mutex to protect providing commands to NTB EPF */ + struct mutex cmd_lock; + + enum pci_barno ctrl_reg_bar; + enum pci_barno peer_spad_reg_bar; + enum pci_barno db_reg_bar; + + unsigned int mw_count; + unsigned int spad_count; + unsigned int db_count; + + void __iomem *ctrl_reg; + void __iomem *db_reg; + void __iomem *peer_spad_reg; + + unsigned int self_spad; + unsigned int peer_spad; + + int db_val; + u64 db_valid_mask; +}; + +#define ntb_ndev(__ntb) container_of(__ntb, struct ntb_epf_dev, ntb) + +struct ntb_epf_data { + /* BAR that contains both control region and self spad region */ + enum pci_barno ctrl_reg_bar; + /* BAR that contains peer spad region */ + enum pci_barno peer_spad_reg_bar; + /* BAR that contains Doorbell region and Memory window '1' */ + enum pci_barno db_reg_bar; +}; + +static int ntb_epf_send_command(struct ntb_epf_dev *ndev, u32 command, + u32 argument) +{ + ktime_t timeout; + bool timedout; + int ret = 0; + u32 status; + + mutex_lock(&ndev->cmd_lock); + writel(argument, ndev->ctrl_reg + NTB_EPF_ARGUMENT); + writel(command, ndev->ctrl_reg + NTB_EPF_COMMAND); + + timeout = ktime_add_ms(ktime_get(), NTB_EPF_COMMAND_TIMEOUT); + while (1) { + timedout = ktime_after(ktime_get(), timeout); + status = readw(ndev->ctrl_reg + NTB_EPF_CMD_STATUS); + + if (status == COMMAND_STATUS_ERROR) { + ret = -EINVAL; + break; + } + + if (status == COMMAND_STATUS_OK) + break; + + if (WARN_ON(timedout)) { + ret = -ETIMEDOUT; + break; + } + + usleep_range(5, 10); + } + + writew(0, ndev->ctrl_reg + NTB_EPF_CMD_STATUS); + mutex_unlock(&ndev->cmd_lock); + + return ret; +} + +static int ntb_epf_mw_to_bar(struct ntb_epf_dev *ndev, int idx) +{ + struct device *dev = ndev->dev; + + if (idx < 0 || idx > ndev->mw_count) { + dev_err(dev, "Unsupported Memory Window index %d\n", idx); + return -EINVAL; + } + + return idx + 2; +} + +static int ntb_epf_mw_count(struct ntb_dev *ntb, int pidx) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + + if (pidx != NTB_DEF_PEER_IDX) { + dev_err(dev, "Unsupported Peer ID %d\n", pidx); + return -EINVAL; + } + + return ndev->mw_count; +} + +static int ntb_epf_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, + resource_size_t *addr_align, + resource_size_t *size_align, + resource_size_t *size_max) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + int bar; + + if (pidx != NTB_DEF_PEER_IDX) { + dev_err(dev, "Unsupported Peer ID %d\n", pidx); + return -EINVAL; + } + + bar = ntb_epf_mw_to_bar(ndev, idx); + if (bar < 0) + return bar; + + if (addr_align) + *addr_align = SZ_4K; + + if (size_align) + *size_align = 1; + + if (size_max) + *size_max = pci_resource_len(ndev->ntb.pdev, bar); + + return 0; +} + +static u64 ntb_epf_link_is_up(struct ntb_dev *ntb, + enum ntb_speed *speed, + enum ntb_width *width) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + u32 status; + + status = readw(ndev->ctrl_reg + NTB_EPF_LINK_STATUS); + + return status & LINK_STATUS_UP; +} + +static u32 ntb_epf_spad_read(struct ntb_dev *ntb, int idx) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + u32 offset; + + if (idx < 0 || idx >= ndev->spad_count) { + dev_err(dev, "READ: Invalid ScratchPad Index %d\n", idx); + return 0; + } + + offset = readl(ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET); + offset += (idx << 2); + + return readl(ndev->ctrl_reg + offset); +} + +static int ntb_epf_spad_write(struct ntb_dev *ntb, + int idx, u32 val) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + u32 offset; + + if (idx < 0 || idx >= ndev->spad_count) { + dev_err(dev, "WRITE: Invalid ScratchPad Index %d\n", idx); + return -EINVAL; + } + + offset = readl(ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET); + offset += (idx << 2); + writel(val, ndev->ctrl_reg + offset); + + return 0; +} + +static u32 ntb_epf_peer_spad_read(struct ntb_dev *ntb, int pidx, int idx) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + u32 offset; + + if (pidx != NTB_DEF_PEER_IDX) { + dev_err(dev, "Unsupported Peer ID %d\n", pidx); + return -EINVAL; + } + + if (idx < 0 || idx >= ndev->spad_count) { + dev_err(dev, "WRITE: Invalid Peer ScratchPad Index %d\n", idx); + return -EINVAL; + } + + offset = (idx << 2); + return readl(ndev->peer_spad_reg + offset); +} + +static int ntb_epf_peer_spad_write(struct ntb_dev *ntb, int pidx, + int idx, u32 val) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + u32 offset; + + if (pidx != NTB_DEF_PEER_IDX) { + dev_err(dev, "Unsupported Peer ID %d\n", pidx); + return -EINVAL; + } + + if (idx < 0 || idx >= ndev->spad_count) { + dev_err(dev, "WRITE: Invalid Peer ScratchPad Index %d\n", idx); + return -EINVAL; + } + + offset = (idx << 2); + writel(val, ndev->peer_spad_reg + offset); + + return 0; +} + +static int ntb_epf_link_enable(struct ntb_dev *ntb, + enum ntb_speed max_speed, + enum ntb_width max_width) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + int ret; + + ret = ntb_epf_send_command(ndev, CMD_LINK_UP, 0); + if (ret) { + dev_err(dev, "Fail to enable link\n"); + return ret; + } + + return 0; +} + +static int ntb_epf_link_disable(struct ntb_dev *ntb) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + int ret; + + ret = ntb_epf_send_command(ndev, CMD_LINK_DOWN, 0); + if (ret) { + dev_err(dev, "Fail to disable link\n"); + return ret; + } + + return 0; +} + +static irqreturn_t ntb_epf_vec_isr(int irq, void *dev) +{ + struct ntb_epf_dev *ndev = dev; + int irq_no; + + irq_no = irq - pci_irq_vector(ndev->ntb.pdev, 0); + ndev->db_val = irq_no + 1; + + if (irq_no == 0) + ntb_link_event(&ndev->ntb); + else + ntb_db_event(&ndev->ntb, irq_no); + + return IRQ_HANDLED; +} + +static int ntb_epf_init_isr(struct ntb_epf_dev *ndev, int msi_min, int msi_max) +{ + struct pci_dev *pdev = ndev->ntb.pdev; + struct device *dev = ndev->dev; + u32 argument = MSIX_ENABLE; + int irq; + int ret; + int i; + + irq = pci_alloc_irq_vectors(pdev, msi_min, msi_max, PCI_IRQ_MSIX); + if (irq < 0) { + dev_dbg(dev, "Failed to get MSIX interrupts\n"); + irq = pci_alloc_irq_vectors(pdev, msi_min, msi_max, + PCI_IRQ_MSI); + if (irq < 0) { + dev_err(dev, "Failed to get MSI interrupts\n"); + return irq; + } + argument &= ~MSIX_ENABLE; + } + + for (i = 0; i < irq; i++) { + ret = request_irq(pci_irq_vector(pdev, i), ntb_epf_vec_isr, + 0, "ntb_epf", ndev); + if (ret) { + dev_err(dev, "Failed to request irq\n"); + goto err_request_irq; + } + } + + ndev->db_count = irq - 1; + + ret = ntb_epf_send_command(ndev, CMD_CONFIGURE_DOORBELL, + argument | irq); + if (ret) { + dev_err(dev, "Failed to configure doorbell\n"); + goto err_configure_db; + } + + return 0; + +err_configure_db: + for (i = 0; i < ndev->db_count + 1; i++) + free_irq(pci_irq_vector(pdev, i), ndev); + +err_request_irq: + pci_free_irq_vectors(pdev); + + return ret; +} + +static int ntb_epf_peer_mw_count(struct ntb_dev *ntb) +{ + return ntb_ndev(ntb)->mw_count; +} + +static int ntb_epf_spad_count(struct ntb_dev *ntb) +{ + return ntb_ndev(ntb)->spad_count; +} + +static u64 ntb_epf_db_valid_mask(struct ntb_dev *ntb) +{ + return ntb_ndev(ntb)->db_valid_mask; +} + +static int ntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits) +{ + return 0; +} + +static int ntb_epf_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, + dma_addr_t addr, resource_size_t size) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + resource_size_t mw_size; + int bar; + + if (pidx != NTB_DEF_PEER_IDX) { + dev_err(dev, "Unsupported Peer ID %d\n", pidx); + return -EINVAL; + } + + bar = idx + NTB_EPF_MW_OFFSET; + + mw_size = pci_resource_len(ntb->pdev, bar); + + if (size > mw_size) { + dev_err(dev, "Size:%pa is greater than the MW size %pa\n", + &size, &mw_size); + return -EINVAL; + } + + writel(lower_32_bits(addr), ndev->ctrl_reg + NTB_EPF_LOWER_ADDR); + writel(upper_32_bits(addr), ndev->ctrl_reg + NTB_EPF_UPPER_ADDR); + writel(lower_32_bits(size), ndev->ctrl_reg + NTB_EPF_LOWER_SIZE); + writel(upper_32_bits(size), ndev->ctrl_reg + NTB_EPF_UPPER_SIZE); + ntb_epf_send_command(ndev, CMD_CONFIGURE_MW, idx); + + return 0; +} + +static int ntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + int ret = 0; + + ntb_epf_send_command(ndev, CMD_TEARDOWN_MW, idx); + if (ret) + dev_err(dev, "Failed to teardown memory window\n"); + + return ret; +} + +static int ntb_epf_peer_mw_get_addr(struct ntb_dev *ntb, int idx, + phys_addr_t *base, resource_size_t *size) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + u32 offset = 0; + int bar; + + if (idx == 0) + offset = readl(ndev->ctrl_reg + NTB_EPF_MW1_OFFSET); + + bar = idx + NTB_EPF_MW_OFFSET; + + if (base) + *base = pci_resource_start(ndev->ntb.pdev, bar) + offset; + + if (size) + *size = pci_resource_len(ndev->ntb.pdev, bar) - offset; + + return 0; +} + +static int ntb_epf_peer_db_set(struct ntb_dev *ntb, u64 db_bits) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + u32 interrupt_num = ffs(db_bits) + 1; + struct device *dev = ndev->dev; + u32 db_entry_size; + u32 db_offset; + u32 db_data; + + if (interrupt_num > ndev->db_count) { + dev_err(dev, "DB interrupt %d greater than Max Supported %d\n", + interrupt_num, ndev->db_count); + return -EINVAL; + } + + db_entry_size = readl(ndev->ctrl_reg + NTB_EPF_DB_ENTRY_SIZE); + + db_data = readl(ndev->ctrl_reg + NTB_EPF_DB_DATA(interrupt_num)); + db_offset = readl(ndev->ctrl_reg + NTB_EPF_DB_OFFSET(interrupt_num)); + writel(db_data, ndev->db_reg + (db_entry_size * interrupt_num) + + db_offset); + + return 0; +} + +static u64 ntb_epf_db_read(struct ntb_dev *ntb) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + + return ndev->db_val; +} + +static int ntb_epf_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) +{ + return 0; +} + +static int ntb_epf_db_clear(struct ntb_dev *ntb, u64 db_bits) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + + ndev->db_val = 0; + + return 0; +} + +static const struct ntb_dev_ops ntb_epf_ops = { + .mw_count = ntb_epf_mw_count, + .spad_count = ntb_epf_spad_count, + .peer_mw_count = ntb_epf_peer_mw_count, + .db_valid_mask = ntb_epf_db_valid_mask, + .db_set_mask = ntb_epf_db_set_mask, + .mw_set_trans = ntb_epf_mw_set_trans, + .mw_clear_trans = ntb_epf_mw_clear_trans, + .peer_mw_get_addr = ntb_epf_peer_mw_get_addr, + .link_enable = ntb_epf_link_enable, + .spad_read = ntb_epf_spad_read, + .spad_write = ntb_epf_spad_write, + .peer_spad_read = ntb_epf_peer_spad_read, + .peer_spad_write = ntb_epf_peer_spad_write, + .peer_db_set = ntb_epf_peer_db_set, + .db_read = ntb_epf_db_read, + .mw_get_align = ntb_epf_mw_get_align, + .link_is_up = ntb_epf_link_is_up, + .db_clear_mask = ntb_epf_db_clear_mask, + .db_clear = ntb_epf_db_clear, + .link_disable = ntb_epf_link_disable, +}; + +static inline void ntb_epf_init_struct(struct ntb_epf_dev *ndev, + struct pci_dev *pdev) +{ + ndev->ntb.pdev = pdev; + ndev->ntb.topo = NTB_TOPO_NONE; + ndev->ntb.ops = &ntb_epf_ops; +} + +static int ntb_epf_init_dev(struct ntb_epf_dev *ndev) +{ + struct device *dev = ndev->dev; + int ret; + + /* One Link interrupt and rest doorbell interrupt */ + ret = ntb_epf_init_isr(ndev, NTB_EPF_MIN_DB_COUNT + 1, + NTB_EPF_MAX_DB_COUNT + 1); + if (ret) { + dev_err(dev, "Failed to init ISR\n"); + return ret; + } + + ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; + ndev->mw_count = readl(ndev->ctrl_reg + NTB_EPF_MW_COUNT); + ndev->spad_count = readl(ndev->ctrl_reg + NTB_EPF_SPAD_COUNT); + + return 0; +} + +static int ntb_epf_init_pci(struct ntb_epf_dev *ndev, + struct pci_dev *pdev) +{ + struct device *dev = ndev->dev; + int ret; + + pci_set_drvdata(pdev, ndev); + + ret = pci_enable_device(pdev); + if (ret) { + dev_err(dev, "Cannot enable PCI device\n"); + goto err_pci_enable; + } + + ret = pci_request_regions(pdev, "ntb"); + if (ret) { + dev_err(dev, "Cannot obtain PCI resources\n"); + goto err_pci_regions; + } + + pci_set_master(pdev); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) { + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(dev, "Cannot set DMA mask\n"); + goto err_dma_mask; + } + dev_warn(&pdev->dev, "Cannot DMA highmem\n"); + } + + ndev->ctrl_reg = pci_iomap(pdev, ndev->ctrl_reg_bar, 0); + if (!ndev->ctrl_reg) { + ret = -EIO; + goto err_dma_mask; + } + + ndev->peer_spad_reg = pci_iomap(pdev, ndev->peer_spad_reg_bar, 0); + if (!ndev->peer_spad_reg) { + ret = -EIO; + goto err_dma_mask; + } + + ndev->db_reg = pci_iomap(pdev, ndev->db_reg_bar, 0); + if (!ndev->db_reg) { + ret = -EIO; + goto err_dma_mask; + } + + return 0; + +err_dma_mask: + pci_clear_master(pdev); + +err_pci_regions: + pci_disable_device(pdev); + +err_pci_enable: + pci_set_drvdata(pdev, NULL); + + return ret; +} + +static void ntb_epf_deinit_pci(struct ntb_epf_dev *ndev) +{ + struct pci_dev *pdev = ndev->ntb.pdev; + + pci_iounmap(pdev, ndev->ctrl_reg); + pci_iounmap(pdev, ndev->peer_spad_reg); + pci_iounmap(pdev, ndev->db_reg); + + pci_clear_master(pdev); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +static void ntb_epf_cleanup_isr(struct ntb_epf_dev *ndev) +{ + struct pci_dev *pdev = ndev->ntb.pdev; + int i; + + ntb_epf_send_command(ndev, CMD_TEARDOWN_DOORBELL, ndev->db_count + 1); + + for (i = 0; i < ndev->db_count + 1; i++) + free_irq(pci_irq_vector(pdev, i), ndev); + pci_free_irq_vectors(pdev); +} + +static int ntb_epf_pci_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + enum pci_barno peer_spad_reg_bar = BAR_1; + enum pci_barno ctrl_reg_bar = BAR_0; + enum pci_barno db_reg_bar = BAR_2; + struct device *dev = &pdev->dev; + struct ntb_epf_data *data; + struct ntb_epf_dev *ndev; + int ret; + + if (pci_is_bridge(pdev)) + return -ENODEV; + + ndev = devm_kzalloc(dev, sizeof(*ndev), GFP_KERNEL); + if (!ndev) + return -ENOMEM; + + data = (struct ntb_epf_data *)id->driver_data; + if (data) { + if (data->peer_spad_reg_bar) + peer_spad_reg_bar = data->peer_spad_reg_bar; + if (data->ctrl_reg_bar) + ctrl_reg_bar = data->ctrl_reg_bar; + if (data->db_reg_bar) + db_reg_bar = data->db_reg_bar; + } + + ndev->peer_spad_reg_bar = peer_spad_reg_bar; + ndev->ctrl_reg_bar = ctrl_reg_bar; + ndev->db_reg_bar = db_reg_bar; + ndev->dev = dev; + + ntb_epf_init_struct(ndev, pdev); + mutex_init(&ndev->cmd_lock); + + ret = ntb_epf_init_pci(ndev, pdev); + if (ret) { + dev_err(dev, "Failed to init PCI\n"); + return ret; + } + + ret = ntb_epf_init_dev(ndev); + if (ret) { + dev_err(dev, "Failed to init device\n"); + goto err_init_dev; + } + + ret = ntb_register_device(&ndev->ntb); + if (ret) { + dev_err(dev, "Failed to register NTB device\n"); + goto err_register_dev; + } + + return 0; + +err_register_dev: + ntb_epf_cleanup_isr(ndev); + +err_init_dev: + ntb_epf_deinit_pci(ndev); + + return ret; +} + +static void ntb_epf_pci_remove(struct pci_dev *pdev) +{ + struct ntb_epf_dev *ndev = pci_get_drvdata(pdev); + + ntb_unregister_device(&ndev->ntb); + ntb_epf_cleanup_isr(ndev); + ntb_epf_deinit_pci(ndev); +} + +static const struct ntb_epf_data j721e_data = { + .ctrl_reg_bar = BAR_0, + .peer_spad_reg_bar = BAR_1, + .db_reg_bar = BAR_2, +}; + +static const struct pci_device_id ntb_epf_pci_tbl[] = { + { + PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E), + .class = PCI_CLASS_MEMORY_RAM << 8, .class_mask = 0xffff00, + .driver_data = (kernel_ulong_t)&j721e_data, + }, + { }, +}; + +static struct pci_driver ntb_epf_pci_driver = { + .name = KBUILD_MODNAME, + .id_table = ntb_epf_pci_tbl, + .probe = ntb_epf_pci_probe, + .remove = ntb_epf_pci_remove, +}; +module_pci_driver(ntb_epf_pci_driver); + +MODULE_DESCRIPTION("PCI ENDPOINT NTB HOST DRIVER"); +MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c index e03a1f38d750..7b9556291eb1 100644 --- a/drivers/nvdimm/blk.c +++ b/drivers/nvdimm/blk.c @@ -310,11 +310,10 @@ static int nd_blk_probe(struct device *dev) return nsblk_attach_disk(nsblk); } -static int nd_blk_remove(struct device *dev) +static void nd_blk_remove(struct device *dev) { if (is_nd_btt(dev)) nvdimm_namespace_detach_btt(to_nd_btt(dev)); - return 0; } static struct nd_device_driver nd_blk_driver = { diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index 2304c6183822..48f0985ca8a0 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -113,18 +113,17 @@ static int nvdimm_bus_remove(struct device *dev) struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver); struct module *provider = to_bus_provider(dev); struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); - int rc = 0; if (nd_drv->remove) { debug_nvdimm_lock(dev); - rc = nd_drv->remove(dev); + nd_drv->remove(dev); debug_nvdimm_unlock(dev); } - dev_dbg(&nvdimm_bus->dev, "%s.remove(%s) = %d\n", dev->driver->name, - dev_name(dev), rc); + dev_dbg(&nvdimm_bus->dev, "%s.remove(%s)\n", dev->driver->name, + dev_name(dev)); module_put(provider); - return rc; + return 0; } static void nvdimm_bus_shutdown(struct device *dev) @@ -427,7 +426,7 @@ static void free_badrange_list(struct list_head *badrange_list) list_del_init(badrange_list); } -static int nd_bus_remove(struct device *dev) +static void nd_bus_remove(struct device *dev) { struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); @@ -446,8 +445,6 @@ static int nd_bus_remove(struct device *dev) spin_unlock(&nvdimm_bus->badrange.lock); nvdimm_bus_destroy_ndctl(nvdimm_bus); - - return 0; } static int nd_bus_probe(struct device *dev) diff --git a/drivers/nvdimm/dimm.c b/drivers/nvdimm/dimm.c index 7d4ddc4d9322..91d9163ee303 100644 --- a/drivers/nvdimm/dimm.c +++ b/drivers/nvdimm/dimm.c @@ -113,19 +113,14 @@ static int nvdimm_probe(struct device *dev) return rc; } -static int nvdimm_remove(struct device *dev) +static void nvdimm_remove(struct device *dev) { struct nvdimm_drvdata *ndd = dev_get_drvdata(dev); - if (!ndd) - return 0; - nvdimm_bus_lock(dev); dev_set_drvdata(dev, NULL); nvdimm_bus_unlock(dev); put_ndd(ndd); - - return 0; } static struct nd_device_driver nvdimm_driver = { diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 281fedb4dc4d..b8a85bfb2e95 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -563,7 +563,7 @@ static int nd_pmem_probe(struct device *dev) return pmem_attach_disk(dev, ndns); } -static int nd_pmem_remove(struct device *dev) +static void nd_pmem_remove(struct device *dev) { struct pmem_device *pmem = dev_get_drvdata(dev); @@ -578,8 +578,6 @@ static int nd_pmem_remove(struct device *dev) pmem->bb_state = NULL; } nvdimm_flush(to_nd_region(dev->parent), NULL); - - return 0; } static void nd_pmem_shutdown(struct device *dev) diff --git a/drivers/nvdimm/region.c b/drivers/nvdimm/region.c index bfce87ed72ab..e0c34120df37 100644 --- a/drivers/nvdimm/region.c +++ b/drivers/nvdimm/region.c @@ -87,7 +87,7 @@ static int child_unregister(struct device *dev, void *data) return 0; } -static int nd_region_remove(struct device *dev) +static void nd_region_remove(struct device *dev) { struct nd_region *nd_region = to_nd_region(dev); @@ -108,8 +108,6 @@ static int nd_region_remove(struct device *dev) */ sysfs_put(nd_region->bb_state); nd_region->bb_state = NULL; - - return 0; } static int child_notify(struct device *dev, void *data) diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index 5dfd806fc2d2..604ab0e5a2ad 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -630,7 +630,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->queue_size = NVMF_DEF_QUEUE_SIZE; opts->nr_io_queues = num_online_cpus(); opts->reconnect_delay = NVMF_DEF_RECONNECT_DELAY; - opts->kato = NVME_DEFAULT_KATO; + opts->kato = 0; opts->duplicate_connect = false; opts->fast_io_fail_tmo = NVMF_DEF_FAIL_FAST_TMO; opts->hdr_digest = false; @@ -893,6 +893,9 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->nr_write_queues = 0; opts->nr_poll_queues = 0; opts->duplicate_connect = true; + } else { + if (!opts->kato) + opts->kato = NVME_DEFAULT_KATO; } if (ctrl_loss_tmo < 0) { opts->max_reconnects = -1; diff --git a/drivers/nvme/host/hwmon.c b/drivers/nvme/host/hwmon.c index 8f9e96986780..0a586d712920 100644 --- a/drivers/nvme/host/hwmon.c +++ b/drivers/nvme/host/hwmon.c @@ -248,6 +248,7 @@ int nvme_hwmon_init(struct nvme_ctrl *ctrl) if (IS_ERR(hwmon)) { dev_warn(dev, "Failed to instantiate hwmon device\n"); kfree(data); + return PTR_ERR(hwmon); } ctrl->hwmon_device = hwmon; return 0; diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 7b6632c00ffd..17ab3320d28b 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -2632,6 +2632,7 @@ static void nvme_reset_work(struct work_struct *work) * Don't limit the IOMMU merged segment size. */ dma_set_max_seg_size(dev->dev, 0xffffffff); + dma_set_min_align_mask(dev->dev, NVME_CTRL_PAGE_SIZE - 1); mutex_unlock(&dev->shutdown_lock); @@ -3233,7 +3234,8 @@ static const struct pci_device_id nvme_id_table[] = { { PCI_DEVICE(0x126f, 0x2263), /* Silicon Motion unidentified */ .driver_data = NVME_QUIRK_NO_NS_DESC_LIST, }, { PCI_DEVICE(0x1bb1, 0x0100), /* Seagate Nytro Flash Storage */ - .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, }, + .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY | + NVME_QUIRK_NO_NS_DESC_LIST, }, { PCI_DEVICE(0x1c58, 0x0003), /* HGST adapter */ .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, }, { PCI_DEVICE(0x1c58, 0x0023), /* WDC SN200 adapter */ @@ -3247,6 +3249,9 @@ static const struct pci_device_id nvme_id_table[] = { NVME_QUIRK_IGNORE_DEV_SUBNQN, }, { PCI_DEVICE(0x1987, 0x5016), /* Phison E16 */ .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN, }, + { PCI_DEVICE(0x1b4b, 0x1092), /* Lexar 256 GB SSD */ + .driver_data = NVME_QUIRK_NO_NS_DESC_LIST | + NVME_QUIRK_IGNORE_DEV_SUBNQN, }, { PCI_DEVICE(0x1d1d, 0x1f1f), /* LighNVM qemu device */ .driver_data = NVME_QUIRK_LIGHTNVM, }, { PCI_DEVICE(0x1d1d, 0x2807), /* CNEX WL */ @@ -3264,6 +3269,8 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, { PCI_DEVICE(0x1d97, 0x2263), /* SPCC */ .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, + { PCI_DEVICE(0x2646, 0x2262), /* KINGSTON SKC2000 NVMe SSD */ + .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, { PCI_DEVICE(0x2646, 0x2263), /* KINGSTON A2000 NVMe SSD */ .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0061), diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index bc6a774f2124..fe6b8aa90b53 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -313,27 +313,40 @@ static void nvmet_execute_get_log_page(struct nvmet_req *req) nvmet_req_complete(req, NVME_SC_INVALID_FIELD | NVME_SC_DNR); } -static void nvmet_id_set_model_number(struct nvme_id_ctrl *id, - struct nvmet_subsys *subsys) +static u16 nvmet_set_model_number(struct nvmet_subsys *subsys) { - const char *model = NVMET_DEFAULT_CTRL_MODEL; - struct nvmet_subsys_model *subsys_model; + u16 status = 0; + + mutex_lock(&subsys->lock); + if (!subsys->model_number) { + subsys->model_number = + kstrdup(NVMET_DEFAULT_CTRL_MODEL, GFP_KERNEL); + if (!subsys->model_number) + status = NVME_SC_INTERNAL; + } + mutex_unlock(&subsys->lock); - rcu_read_lock(); - subsys_model = rcu_dereference(subsys->model); - if (subsys_model) - model = subsys_model->number; - memcpy_and_pad(id->mn, sizeof(id->mn), model, strlen(model), ' '); - rcu_read_unlock(); + return status; } static void nvmet_execute_identify_ctrl(struct nvmet_req *req) { struct nvmet_ctrl *ctrl = req->sq->ctrl; + struct nvmet_subsys *subsys = ctrl->subsys; struct nvme_id_ctrl *id; u32 cmd_capsule_size; u16 status = 0; + /* + * If there is no model number yet, set it now. It will then remain + * stable for the life time of the subsystem. + */ + if (!subsys->model_number) { + status = nvmet_set_model_number(subsys); + if (status) + goto out; + } + id = kzalloc(sizeof(*id), GFP_KERNEL); if (!id) { status = NVME_SC_INTERNAL; @@ -347,7 +360,8 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req) memset(id->sn, ' ', sizeof(id->sn)); bin2hex(id->sn, &ctrl->subsys->serial, min(sizeof(ctrl->subsys->serial), sizeof(id->sn) / 2)); - nvmet_id_set_model_number(id, ctrl->subsys); + memcpy_and_pad(id->mn, sizeof(id->mn), subsys->model_number, + strlen(subsys->model_number), ' '); memcpy_and_pad(id->fr, sizeof(id->fr), UTS_RELEASE, strlen(UTS_RELEASE), ' '); diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 635a7cb45d0b..e5dbd1923b7b 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -1118,16 +1118,12 @@ static ssize_t nvmet_subsys_attr_model_show(struct config_item *item, char *page) { struct nvmet_subsys *subsys = to_subsys(item); - struct nvmet_subsys_model *subsys_model; - char *model = NVMET_DEFAULT_CTRL_MODEL; int ret; - rcu_read_lock(); - subsys_model = rcu_dereference(subsys->model); - if (subsys_model) - model = subsys_model->number; - ret = snprintf(page, PAGE_SIZE, "%s\n", model); - rcu_read_unlock(); + mutex_lock(&subsys->lock); + ret = snprintf(page, PAGE_SIZE, "%s\n", subsys->model_number ? + subsys->model_number : NVMET_DEFAULT_CTRL_MODEL); + mutex_unlock(&subsys->lock); return ret; } @@ -1138,14 +1134,17 @@ static bool nvmet_is_ascii(const char c) return c >= 0x20 && c <= 0x7e; } -static ssize_t nvmet_subsys_attr_model_store(struct config_item *item, - const char *page, size_t count) +static ssize_t nvmet_subsys_attr_model_store_locked(struct nvmet_subsys *subsys, + const char *page, size_t count) { - struct nvmet_subsys *subsys = to_subsys(item); - struct nvmet_subsys_model *new_model; - char *new_model_number; int pos = 0, len; + if (subsys->model_number) { + pr_err("Can't set model number. %s is already assigned\n", + subsys->model_number); + return -EINVAL; + } + len = strcspn(page, "\n"); if (!len) return -EINVAL; @@ -1155,28 +1154,25 @@ static ssize_t nvmet_subsys_attr_model_store(struct config_item *item, return -EINVAL; } - new_model_number = kmemdup_nul(page, len, GFP_KERNEL); - if (!new_model_number) + subsys->model_number = kmemdup_nul(page, len, GFP_KERNEL); + if (!subsys->model_number) return -ENOMEM; + return count; +} - new_model = kzalloc(sizeof(*new_model) + len + 1, GFP_KERNEL); - if (!new_model) { - kfree(new_model_number); - return -ENOMEM; - } - memcpy(new_model->number, new_model_number, len); +static ssize_t nvmet_subsys_attr_model_store(struct config_item *item, + const char *page, size_t count) +{ + struct nvmet_subsys *subsys = to_subsys(item); + ssize_t ret; down_write(&nvmet_config_sem); mutex_lock(&subsys->lock); - new_model = rcu_replace_pointer(subsys->model, new_model, - mutex_is_locked(&subsys->lock)); + ret = nvmet_subsys_attr_model_store_locked(subsys, page, count); mutex_unlock(&subsys->lock); up_write(&nvmet_config_sem); - kfree_rcu(new_model, rcuhead); - kfree(new_model_number); - - return count; + return ret; } CONFIGFS_ATTR(nvmet_subsys_, attr_model); diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 67bbf0e3b507..be6fcdaf51a7 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -1532,7 +1532,7 @@ static void nvmet_subsys_free(struct kref *ref) nvmet_passthru_subsys_free(subsys); kfree(subsys->subsysnqn); - kfree_rcu(subsys->model, rcuhead); + kfree(subsys->model_number); kfree(subsys); } diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index 3d9a5d3ed9cd..9a8b3726a37c 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -185,7 +185,7 @@ static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio, } bip = bio_integrity_alloc(bio, GFP_NOIO, - min_t(unsigned int, req->metadata_sg_cnt, BIO_MAX_PAGES)); + bio_max_segs(req->metadata_sg_cnt)); if (IS_ERR(bip)) { pr_err("Unable to allocate bio_integrity_payload\n"); return PTR_ERR(bip); @@ -225,7 +225,7 @@ static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio, static void nvmet_bdev_execute_rw(struct nvmet_req *req) { - int sg_cnt = req->sg_cnt; + unsigned int sg_cnt = req->sg_cnt; struct bio *bio; struct scatterlist *sg; struct blk_plug plug; @@ -262,7 +262,7 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) bio = &req->b.inline_bio; bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec)); } else { - bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES)); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(sg_cnt)); } bio_set_dev(bio, req->ns->bdev); bio->bi_iter.bi_sector = sector; @@ -289,7 +289,7 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) } } - bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES)); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(sg_cnt)); bio_set_dev(bio, req->ns->bdev); bio->bi_iter.bi_sector = sector; bio->bi_opf = op; diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index cdfa537b1c0a..4b84edb49f22 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -208,11 +208,6 @@ struct nvmet_ctrl { bool pi_support; }; -struct nvmet_subsys_model { - struct rcu_head rcuhead; - char number[]; -}; - struct nvmet_subsys { enum nvme_subsys_type type; @@ -242,7 +237,7 @@ struct nvmet_subsys { struct config_group namespaces_group; struct config_group allowed_hosts_group; - struct nvmet_subsys_model __rcu *model; + char *model_number; #ifdef CONFIG_NVME_TARGET_PASSTHRU struct nvme_ctrl *passthru_ctrl; diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c index f50c7b2bf21c..26c587ccd152 100644 --- a/drivers/nvme/target/passthru.c +++ b/drivers/nvme/target/passthru.c @@ -26,7 +26,7 @@ static u16 nvmet_passthru_override_id_ctrl(struct nvmet_req *req) struct nvme_ctrl *pctrl = ctrl->subsys->passthru_ctrl; u16 status = NVME_SC_SUCCESS; struct nvme_id_ctrl *id; - int max_hw_sectors; + unsigned int max_hw_sectors; int page_shift; id = kzalloc(sizeof(*id), GFP_KERNEL); @@ -198,7 +198,7 @@ static int nvmet_passthru_map_sg(struct nvmet_req *req, struct request *rq) bio = &req->p.inline_bio; bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec)); } else { - bio = bio_alloc(GFP_KERNEL, min(req->sg_cnt, BIO_MAX_PAGES)); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(req->sg_cnt)); bio->bi_end_io = bio_put; } bio->bi_opf = req_op(rq); diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig index 954d3b4a52ab..75d2594c16e1 100644 --- a/drivers/nvmem/Kconfig +++ b/drivers/nvmem/Kconfig @@ -270,4 +270,12 @@ config SPRD_EFUSE This driver can also be built as a module. If so, the module will be called nvmem-sprd-efuse. +config NVMEM_RMEM + tristate "Reserved Memory Based Driver Support" + help + This driver maps reserved memory into an nvmem device. It might be + useful to expose information left by firmware in memory. + + This driver can also be built as a module. If so, the module + will be called nvmem-rmem. endif diff --git a/drivers/nvmem/Makefile b/drivers/nvmem/Makefile index a7c377218341..5376b8e0dae5 100644 --- a/drivers/nvmem/Makefile +++ b/drivers/nvmem/Makefile @@ -55,3 +55,5 @@ obj-$(CONFIG_NVMEM_ZYNQMP) += nvmem_zynqmp_nvmem.o nvmem_zynqmp_nvmem-y := zynqmp_nvmem.o obj-$(CONFIG_SPRD_EFUSE) += nvmem_sprd_efuse.o nvmem_sprd_efuse-y := sprd-efuse.o +obj-$(CONFIG_NVMEM_RMEM) += nvmem-rmem.o +nvmem-rmem-y := rmem.o diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 177f5bf27c6d..a5ab1e0c74cf 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -682,7 +682,9 @@ static int nvmem_add_cells_from_of(struct nvmem_device *nvmem) for_each_child_of_node(parent, child) { addr = of_get_property(child, "reg", &len); - if (!addr || (len < 2 * sizeof(u32))) { + if (!addr) + continue; + if (len < 2 * sizeof(u32)) { dev_err(dev, "nvmem: invalid reg on %pOF\n", child); return -EINVAL; } @@ -713,6 +715,7 @@ static int nvmem_add_cells_from_of(struct nvmem_device *nvmem) cell->name, nvmem->stride); /* Cells already added will be freed later. */ kfree_const(cell->name); + of_node_put(cell->np); kfree(cell); return -EINVAL; } diff --git a/drivers/nvmem/imx-iim.c b/drivers/nvmem/imx-iim.c index 701704b87dc9..c86339a7f583 100644 --- a/drivers/nvmem/imx-iim.c +++ b/drivers/nvmem/imx-iim.c @@ -96,7 +96,6 @@ MODULE_DEVICE_TABLE(of, imx_iim_dt_ids); static int imx_iim_probe(struct platform_device *pdev) { - const struct of_device_id *of_id; struct device *dev = &pdev->dev; struct iim_priv *iim; struct nvmem_device *nvmem; @@ -111,11 +110,7 @@ static int imx_iim_probe(struct platform_device *pdev) if (IS_ERR(iim->base)) return PTR_ERR(iim->base); - of_id = of_match_device(imx_iim_dt_ids, dev); - if (!of_id) - return -ENODEV; - - drvdata = of_id->data; + drvdata = of_device_get_match_data(&pdev->dev); iim->clk = devm_clk_get(dev, NULL); if (IS_ERR(iim->clk)) diff --git a/drivers/nvmem/qcom-spmi-sdam.c b/drivers/nvmem/qcom-spmi-sdam.c index a72704cd0468..f6e9f96933ca 100644 --- a/drivers/nvmem/qcom-spmi-sdam.c +++ b/drivers/nvmem/qcom-spmi-sdam.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2017, 2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2017, 2020-2021, The Linux Foundation. All rights reserved. */ #include <linux/device.h> @@ -18,7 +18,6 @@ #define SDAM_PBS_TRIG_CLR 0xE6 struct sdam_chip { - struct platform_device *pdev; struct regmap *regmap; struct nvmem_config sdam_config; unsigned int base; @@ -65,7 +64,7 @@ static int sdam_read(void *priv, unsigned int offset, void *val, size_t bytes) { struct sdam_chip *sdam = priv; - struct device *dev = &sdam->pdev->dev; + struct device *dev = sdam->sdam_config.dev; int rc; if (!sdam_is_valid(sdam, offset, bytes)) { @@ -86,7 +85,7 @@ static int sdam_write(void *priv, unsigned int offset, void *val, size_t bytes) { struct sdam_chip *sdam = priv; - struct device *dev = &sdam->pdev->dev; + struct device *dev = sdam->sdam_config.dev; int rc; if (!sdam_is_valid(sdam, offset, bytes)) { diff --git a/drivers/nvmem/rmem.c b/drivers/nvmem/rmem.c new file mode 100644 index 000000000000..b11c3c974b3d --- /dev/null +++ b/drivers/nvmem/rmem.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2020 Nicolas Saenz Julienne <nsaenzjulienne@suse.de> + */ + +#include <linux/io.h> +#include <linux/module.h> +#include <linux/nvmem-provider.h> +#include <linux/of_reserved_mem.h> +#include <linux/platform_device.h> + +struct rmem { + struct device *dev; + struct nvmem_device *nvmem; + struct reserved_mem *mem; + + phys_addr_t size; +}; + +static int rmem_read(void *context, unsigned int offset, + void *val, size_t bytes) +{ + struct rmem *priv = context; + size_t available = priv->mem->size; + loff_t off = offset; + void *addr; + int count; + + /* + * Only map the reserved memory at this point to avoid potential rogue + * kernel threads inadvertently modifying it. Based on the current + * uses-cases for this driver, the performance hit isn't a concern. + * Nor is likely to be, given the nature of the subsystem. Most nvmem + * devices operate over slow buses to begin with. + * + * An alternative would be setting the memory as RO, set_memory_ro(), + * but as of Dec 2020 this isn't possible on arm64. + */ + addr = memremap(priv->mem->base, available, MEMREMAP_WB); + if (IS_ERR(addr)) { + dev_err(priv->dev, "Failed to remap memory region\n"); + return PTR_ERR(addr); + } + + count = memory_read_from_buffer(val, bytes, &off, addr, available); + + memunmap(addr); + + return count; +} + +static int rmem_probe(struct platform_device *pdev) +{ + struct nvmem_config config = { }; + struct device *dev = &pdev->dev; + struct reserved_mem *mem; + struct rmem *priv; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + priv->dev = dev; + + mem = of_reserved_mem_lookup(dev->of_node); + if (!mem) { + dev_err(dev, "Failed to lookup reserved memory\n"); + return -EINVAL; + } + priv->mem = mem; + + config.dev = dev; + config.priv = priv; + config.name = "rmem"; + config.size = mem->size; + config.reg_read = rmem_read; + + return PTR_ERR_OR_ZERO(devm_nvmem_register(dev, &config)); +} + +static const struct of_device_id rmem_match[] = { + { .compatible = "nvmem-rmem", }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, rmem_match); + +static struct platform_driver rmem_driver = { + .probe = rmem_probe, + .driver = { + .name = "rmem", + .of_match_table = rmem_match, + }, +}; +module_platform_driver(rmem_driver); + +MODULE_AUTHOR("Nicolas Saenz Julienne <nsaenzjulienne@suse.de>"); +MODULE_DESCRIPTION("Reserved Memory Based nvmem Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/of/platform.c b/drivers/of/platform.c index 020bf860c72c..0da86209ddaa 100644 --- a/drivers/of/platform.c +++ b/drivers/of/platform.c @@ -511,6 +511,7 @@ static const struct of_device_id reserved_mem_matches[] = { { .compatible = "qcom,rmtfs-mem" }, { .compatible = "qcom,cmd-db" }, { .compatible = "ramoops" }, + { .compatible = "nvmem-rmem" }, {} }; diff --git a/drivers/of/property.c b/drivers/of/property.c index 5f9eed79a8aa..5036a362f52e 100644 --- a/drivers/of/property.c +++ b/drivers/of/property.c @@ -24,6 +24,7 @@ #include <linux/of.h> #include <linux/of_device.h> #include <linux/of_graph.h> +#include <linux/of_irq.h> #include <linux/string.h> #include <linux/moduleparam.h> @@ -1102,7 +1103,9 @@ static int of_link_to_phandle(struct device_node *con_np, * created for them. */ sup_dev = get_dev_from_fwnode(&sup_np->fwnode); - if (!sup_dev && of_node_check_flag(sup_np, OF_POPULATED)) { + if (!sup_dev && + (of_node_check_flag(sup_np, OF_POPULATED) || + sup_np->fwnode.flags & FWNODE_FLAG_NOT_DEVICE)) { pr_debug("Not linking %pOFP to %pOFP - No struct device\n", con_np, sup_np); of_node_put(sup_np); @@ -1232,6 +1235,7 @@ static struct device_node *parse_##fname(struct device_node *np, \ struct supplier_bindings { struct device_node *(*parse_prop)(struct device_node *np, const char *prop_name, int index); + bool optional; }; DEFINE_SIMPLE_PROP(clocks, "clocks", "#clock-cells") @@ -1244,8 +1248,6 @@ DEFINE_SIMPLE_PROP(dmas, "dmas", "#dma-cells") DEFINE_SIMPLE_PROP(power_domains, "power-domains", "#power-domain-cells") DEFINE_SIMPLE_PROP(hwlocks, "hwlocks", "#hwlock-cells") DEFINE_SIMPLE_PROP(extcon, "extcon", NULL) -DEFINE_SIMPLE_PROP(interrupts_extended, "interrupts-extended", - "#interrupt-cells") DEFINE_SIMPLE_PROP(nvmem_cells, "nvmem-cells", NULL) DEFINE_SIMPLE_PROP(phys, "phys", "#phy-cells") DEFINE_SIMPLE_PROP(wakeup_parent, "wakeup-parent", NULL) @@ -1271,19 +1273,55 @@ static struct device_node *parse_iommu_maps(struct device_node *np, return of_parse_phandle(np, prop_name, (index * 4) + 1); } +static struct device_node *parse_gpio_compat(struct device_node *np, + const char *prop_name, int index) +{ + struct of_phandle_args sup_args; + + if (strcmp(prop_name, "gpio") && strcmp(prop_name, "gpios")) + return NULL; + + /* + * Ignore node with gpio-hog property since its gpios are all provided + * by its parent. + */ + if (of_find_property(np, "gpio-hog", NULL)) + return NULL; + + if (of_parse_phandle_with_args(np, prop_name, "#gpio-cells", index, + &sup_args)) + return NULL; + + return sup_args.np; +} + +static struct device_node *parse_interrupts(struct device_node *np, + const char *prop_name, int index) +{ + struct of_phandle_args sup_args; + + if (!IS_ENABLED(CONFIG_OF_IRQ) || IS_ENABLED(CONFIG_PPC)) + return NULL; + + if (strcmp(prop_name, "interrupts") && + strcmp(prop_name, "interrupts-extended")) + return NULL; + + return of_irq_parse_one(np, index, &sup_args) ? NULL : sup_args.np; +} + static const struct supplier_bindings of_supplier_bindings[] = { { .parse_prop = parse_clocks, }, { .parse_prop = parse_interconnects, }, - { .parse_prop = parse_iommus, }, - { .parse_prop = parse_iommu_maps, }, + { .parse_prop = parse_iommus, .optional = true, }, + { .parse_prop = parse_iommu_maps, .optional = true, }, { .parse_prop = parse_mboxes, }, { .parse_prop = parse_io_channels, }, { .parse_prop = parse_interrupt_parent, }, - { .parse_prop = parse_dmas, }, + { .parse_prop = parse_dmas, .optional = true, }, { .parse_prop = parse_power_domains, }, { .parse_prop = parse_hwlocks, }, { .parse_prop = parse_extcon, }, - { .parse_prop = parse_interrupts_extended, }, { .parse_prop = parse_nvmem_cells, }, { .parse_prop = parse_phys, }, { .parse_prop = parse_wakeup_parent, }, @@ -1296,6 +1334,8 @@ static const struct supplier_bindings of_supplier_bindings[] = { { .parse_prop = parse_pinctrl6, }, { .parse_prop = parse_pinctrl7, }, { .parse_prop = parse_pinctrl8, }, + { .parse_prop = parse_gpio_compat, }, + { .parse_prop = parse_interrupts, }, { .parse_prop = parse_regulators, }, { .parse_prop = parse_gpio, }, { .parse_prop = parse_gpios, }, @@ -1332,6 +1372,11 @@ static int of_link_property(struct device_node *con_np, const char *prop_name) /* Do not stop at first failed link, link all available suppliers. */ while (!matched && s->parse_prop) { + if (s->optional && !fw_devlink_is_strict()) { + s++; + continue; + } + while ((phandle = s->parse_prop(con_np, prop_name, i))) { matched = true; i++; diff --git a/drivers/opp/core.c b/drivers/opp/core.c index c3f3d9249cc5..c2689386a906 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -998,14 +998,15 @@ static int _set_opp(struct device *dev, struct opp_table *opp_table, old_opp = opp_table->current_opp; /* Return early if nothing to do */ - if (opp_table->enabled && old_opp == opp) { + if (old_opp == opp && opp_table->current_rate == freq && + opp_table->enabled) { dev_dbg(dev, "%s: OPPs are same, nothing to do\n", __func__); return 0; } dev_dbg(dev, "%s: switching OPP: Freq %lu -> %lu Hz, Level %u -> %u, Bw %u -> %u\n", - __func__, old_opp->rate, freq, old_opp->level, opp->level, - old_opp->bandwidth ? old_opp->bandwidth[0].peak : 0, + __func__, opp_table->current_rate, freq, old_opp->level, + opp->level, old_opp->bandwidth ? old_opp->bandwidth[0].peak : 0, opp->bandwidth ? opp->bandwidth[0].peak : 0); scaling_down = _opp_compare_key(old_opp, opp); @@ -1061,6 +1062,7 @@ static int _set_opp(struct device *dev, struct opp_table *opp_table, /* Make sure current_opp doesn't get freed */ dev_pm_opp_get(opp); opp_table->current_opp = opp; + opp_table->current_rate = freq; return ret; } diff --git a/drivers/opp/opp.h b/drivers/opp/opp.h index 9b9daf83b074..50fb9dced3c5 100644 --- a/drivers/opp/opp.h +++ b/drivers/opp/opp.h @@ -135,6 +135,7 @@ enum opp_table_access { * @clock_latency_ns_max: Max clock latency in nanoseconds. * @parsed_static_opps: Count of devices for which OPPs are initialized from DT. * @shared_opp: OPP is shared between multiple devices. + * @current_rate: Currently configured frequency. * @current_opp: Currently configured OPP for the table. * @suspend_opp: Pointer to OPP to be used during device suspend. * @genpd_virt_dev_lock: Mutex protecting the genpd virtual device pointers. @@ -184,6 +185,7 @@ struct opp_table { unsigned int parsed_static_opps; enum opp_table_access shared_opp; + unsigned long current_rate; struct dev_pm_opp *current_opp; struct dev_pm_opp *suspend_opp; diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile index 11cc79411e2d..d62c4ac4ae1b 100644 --- a/drivers/pci/Makefile +++ b/drivers/pci/Makefile @@ -36,4 +36,4 @@ obj-$(CONFIG_PCI_ENDPOINT) += endpoint/ obj-y += controller/ obj-y += switch/ -ccflags-$(CONFIG_PCI_DEBUG) := -DDEBUG +subdir-ccflags-$(CONFIG_PCI_DEBUG) := -DDEBUG diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 64e2f5e379aa..5aa8977d7b0f 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -55,15 +55,6 @@ config PCI_RCAR_GEN2 There are 3 internal PCI controllers available with a single built-in EHCI/OHCI host controller present on each one. -config PCIE_RCAR - bool "Renesas R-Car PCIe controller" - depends on ARCH_RENESAS || COMPILE_TEST - depends on PCI_MSI_IRQ_DOMAIN - select PCIE_RCAR_HOST - help - Say Y here if you want PCIe controller support on R-Car SoCs. - This option will be removed after arm64 defconfig is updated. - config PCIE_RCAR_HOST bool "Renesas R-Car PCIe host controller" depends on ARCH_RENESAS || COMPILE_TEST @@ -242,20 +233,6 @@ config PCIE_MEDIATEK Say Y here if you want to enable PCIe controller support on MediaTek SoCs. -config PCIE_TANGO_SMP8759 - bool "Tango SMP8759 PCIe controller (DANGEROUS)" - depends on ARCH_TANGO && PCI_MSI && OF - depends on BROKEN - select PCI_HOST_COMMON - help - Say Y here to enable PCIe controller support for Sigma Designs - Tango SMP8759-based systems. - - Note: The SMP8759 controller multiplexes PCI config and MMIO - accesses, and Linux doesn't provide a way to serialize them. - This can lead to data corruption if drivers perform concurrent - config and MMIO accesses. - config VMD depends on PCI_MSI && X86_64 && SRCU tristate "Intel Volume Management Device Driver" @@ -273,7 +250,7 @@ config VMD config PCIE_BRCMSTB tristate "Broadcom Brcmstb PCIe host controller" - depends on ARCH_BRCMSTB || ARCH_BCM2835 || COMPILE_TEST + depends on ARCH_BRCMSTB || ARCH_BCM2835 || ARCH_BCM4908 || COMPILE_TEST depends on OF depends on PCI_MSI_IRQ_DOMAIN default ARCH_BRCMSTB @@ -298,6 +275,16 @@ config PCI_LOONGSON Say Y here if you want to enable PCI controller support on Loongson systems. +config PCIE_MICROCHIP_HOST + bool "Microchip AXI PCIe host bridge support" + depends on PCI_MSI && OF + select PCI_MSI_IRQ_DOMAIN + select GENERIC_MSI_IRQ_DOMAIN + select PCI_HOST_COMMON + help + Say Y here if you want kernel to support the Microchip AXI PCIe + Host Bridge driver. + config PCIE_HISI_ERR depends on ACPI_APEI_GHES && (ARM64 || COMPILE_TEST) bool "HiSilicon HIP PCIe controller error handling driver" diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 04c6edc285c5..e4559f2182f2 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -27,7 +27,7 @@ obj-$(CONFIG_PCIE_ROCKCHIP) += pcie-rockchip.o obj-$(CONFIG_PCIE_ROCKCHIP_EP) += pcie-rockchip-ep.o obj-$(CONFIG_PCIE_ROCKCHIP_HOST) += pcie-rockchip-host.o obj-$(CONFIG_PCIE_MEDIATEK) += pcie-mediatek.o -obj-$(CONFIG_PCIE_TANGO_SMP8759) += pcie-tango.o +obj-$(CONFIG_PCIE_MICROCHIP_HOST) += pcie-microchip-host.o obj-$(CONFIG_VMD) += vmd.o obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o obj-$(CONFIG_PCI_LOONGSON) += pci-loongson.o diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index dac1ac8a7615..849f1e416ea5 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -64,6 +64,7 @@ enum j721e_pcie_mode { struct j721e_pcie_data { enum j721e_pcie_mode mode; + bool quirk_retrain_flag; }; static inline u32 j721e_pcie_user_readl(struct j721e_pcie *pcie, u32 offset) @@ -280,6 +281,7 @@ static struct pci_ops cdns_ti_pcie_host_ops = { static const struct j721e_pcie_data j721e_pcie_rc_data = { .mode = PCI_MODE_RC, + .quirk_retrain_flag = true, }; static const struct j721e_pcie_data j721e_pcie_ep_data = { @@ -388,6 +390,7 @@ static int j721e_pcie_probe(struct platform_device *pdev) bridge->ops = &cdns_ti_pcie_host_ops; rc = pci_host_bridge_priv(bridge); + rc->quirk_retrain_flag = data->quirk_retrain_flag; cdns_pcie = &rc->pcie; cdns_pcie->dev = dev; diff --git a/drivers/pci/controller/cadence/pcie-cadence-ep.c b/drivers/pci/controller/cadence/pcie-cadence-ep.c index 9e2b024d32f2..897cdde02bd8 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-ep.c +++ b/drivers/pci/controller/cadence/pcie-cadence-ep.c @@ -382,6 +382,57 @@ static int cdns_pcie_ep_send_msi_irq(struct cdns_pcie_ep *ep, u8 fn, return 0; } +static int cdns_pcie_ep_map_msi_irq(struct pci_epc *epc, u8 fn, + phys_addr_t addr, u8 interrupt_num, + u32 entry_size, u32 *msi_data, + u32 *msi_addr_offset) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + u32 cap = CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET; + struct cdns_pcie *pcie = &ep->pcie; + u64 pci_addr, pci_addr_mask = 0xff; + u16 flags, mme, data, data_mask; + u8 msi_count; + int ret; + int i; + + /* Check whether the MSI feature has been enabled by the PCI host. */ + flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS); + if (!(flags & PCI_MSI_FLAGS_ENABLE)) + return -EINVAL; + + /* Get the number of enabled MSIs */ + mme = (flags & PCI_MSI_FLAGS_QSIZE) >> 4; + msi_count = 1 << mme; + if (!interrupt_num || interrupt_num > msi_count) + return -EINVAL; + + /* Compute the data value to be written. */ + data_mask = msi_count - 1; + data = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_DATA_64); + data = data & ~data_mask; + + /* Get the PCI address where to write the data into. */ + pci_addr = cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_HI); + pci_addr <<= 32; + pci_addr |= cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_LO); + pci_addr &= GENMASK_ULL(63, 2); + + for (i = 0; i < interrupt_num; i++) { + ret = cdns_pcie_ep_map_addr(epc, fn, addr, + pci_addr & ~pci_addr_mask, + entry_size); + if (ret) + return ret; + addr = addr + entry_size; + } + + *msi_data = data; + *msi_addr_offset = pci_addr & pci_addr_mask; + + return 0; +} + static int cdns_pcie_ep_send_msix_irq(struct cdns_pcie_ep *ep, u8 fn, u16 interrupt_num) { @@ -455,18 +506,13 @@ static int cdns_pcie_ep_start(struct pci_epc *epc) struct cdns_pcie_ep *ep = epc_get_drvdata(epc); struct cdns_pcie *pcie = &ep->pcie; struct device *dev = pcie->dev; - struct pci_epf *epf; - u32 cfg; int ret; /* * BIT(0) is hardwired to 1, hence function 0 is always enabled * and can't be disabled anyway. */ - cfg = BIT(0); - list_for_each_entry(epf, &epc->pci_epf, list) - cfg |= BIT(epf->func_no); - cdns_pcie_writel(pcie, CDNS_PCIE_LM_EP_FUNC_CFG, cfg); + cdns_pcie_writel(pcie, CDNS_PCIE_LM_EP_FUNC_CFG, epc->function_num_map); ret = cdns_pcie_start_link(pcie); if (ret) { @@ -481,6 +527,7 @@ static const struct pci_epc_features cdns_pcie_epc_features = { .linkup_notifier = false, .msi_capable = true, .msix_capable = true, + .align = 256, }; static const struct pci_epc_features* @@ -500,6 +547,7 @@ static const struct pci_epc_ops cdns_pcie_epc_ops = { .set_msix = cdns_pcie_ep_set_msix, .get_msix = cdns_pcie_ep_get_msix, .raise_irq = cdns_pcie_ep_raise_irq, + .map_msi_irq = cdns_pcie_ep_map_msi_irq, .start = cdns_pcie_ep_start, .get_features = cdns_pcie_ep_get_features, }; diff --git a/drivers/pci/controller/cadence/pcie-cadence-host.c b/drivers/pci/controller/cadence/pcie-cadence-host.c index 811c1cb2e8de..73dcf8cf98fb 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-host.c +++ b/drivers/pci/controller/cadence/pcie-cadence-host.c @@ -77,6 +77,68 @@ static struct pci_ops cdns_pcie_host_ops = { .write = pci_generic_config_write, }; +static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie) +{ + struct device *dev = pcie->dev; + int retries; + + /* Check if the link is up or not */ + for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) { + if (cdns_pcie_link_up(pcie)) { + dev_info(dev, "Link up\n"); + return 0; + } + usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX); + } + + return -ETIMEDOUT; +} + +static int cdns_pcie_retrain(struct cdns_pcie *pcie) +{ + u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET; + u16 lnk_stat, lnk_ctl; + int ret = 0; + + /* + * Set retrain bit if current speed is 2.5 GB/s, + * but the PCIe root port support is > 2.5 GB/s. + */ + + lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off + + PCI_EXP_LNKCAP)); + if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB) + return ret; + + lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA); + if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) { + lnk_ctl = cdns_pcie_rp_readw(pcie, + pcie_cap_off + PCI_EXP_LNKCTL); + lnk_ctl |= PCI_EXP_LNKCTL_RL; + cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL, + lnk_ctl); + + ret = cdns_pcie_host_wait_for_link(pcie); + } + return ret; +} + +static int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + int ret; + + ret = cdns_pcie_host_wait_for_link(pcie); + + /* + * Retrain link for Gen2 training defect + * if quirk flag is set. + */ + if (!ret && rc->quirk_retrain_flag) + ret = cdns_pcie_retrain(pcie); + + return ret; +} static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc) { @@ -321,9 +383,10 @@ static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc) resource_list_for_each_entry(entry, &bridge->dma_ranges) { err = cdns_pcie_host_bar_config(rc, entry); - if (err) + if (err) { dev_err(dev, "Fail to configure IB using dma-ranges\n"); - return err; + return err; + } } return 0; @@ -398,23 +461,6 @@ static int cdns_pcie_host_init(struct device *dev, return cdns_pcie_host_init_address_translation(rc); } -static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie) -{ - struct device *dev = pcie->dev; - int retries; - - /* Check if the link is up or not */ - for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) { - if (cdns_pcie_link_up(pcie)) { - dev_info(dev, "Link up\n"); - return 0; - } - usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX); - } - - return -ETIMEDOUT; -} - int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) { struct device *dev = rc->pcie.dev; @@ -457,7 +503,7 @@ int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) return ret; } - ret = cdns_pcie_host_wait_for_link(pcie); + ret = cdns_pcie_host_start_link(rc); if (ret) dev_dbg(dev, "PCIe link never came up\n"); diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h index 30eba6cafe2c..254d2570f8c9 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.h +++ b/drivers/pci/controller/cadence/pcie-cadence.h @@ -119,7 +119,7 @@ * Root Port Registers (PCI configuration space for the root port function) */ #define CDNS_PCIE_RP_BASE 0x00200000 - +#define CDNS_PCIE_RP_CAP_OFFSET 0xc0 /* * Address Translation Registers @@ -291,6 +291,7 @@ struct cdns_pcie { * @device_id: PCI device ID * @avail_ib_bar: Satus of RP_BAR0, RP_BAR1 and RP_NO_BAR if it's free or * available + * @quirk_retrain_flag: Retrain link as quirk for PCIe Gen2 */ struct cdns_pcie_rc { struct cdns_pcie pcie; @@ -299,6 +300,7 @@ struct cdns_pcie_rc { u32 vendor_id; u32 device_id; bool avail_ib_bar[CDNS_PCIE_RP_MAX_IB]; + bool quirk_retrain_flag; }; /** @@ -414,6 +416,13 @@ static inline void cdns_pcie_rp_writew(struct cdns_pcie *pcie, cdns_pcie_write_sz(addr, 0x2, value); } +static inline u16 cdns_pcie_rp_readw(struct cdns_pcie *pcie, u32 reg) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_RP_BASE + reg; + + return cdns_pcie_read_sz(addr, 0x2); +} + /* Endpoint Function register access */ static inline void cdns_pcie_ep_fn_writeb(struct cdns_pcie *pcie, u8 fn, u32 reg, u8 value) diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c index 4d12efdacd2f..39fe2ed5a6a2 100644 --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c @@ -115,10 +115,17 @@ static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; +static const struct ls_pcie_ep_drvdata lx2_ep_drvdata = { + .func_offset = 0x8000, + .ops = &ls_pcie_ep_ops, + .dw_pcie_ops = &dw_ls_pcie_ep_ops, +}; + static const struct of_device_id ls_pcie_ep_of_match[] = { { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, + { .compatible = "fsl,lx2160ar2-pcie-ep", .data = &lx2_ep_drvdata }, { }, }; diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c index 44ad34cdc3bc..5b9c625df7b8 100644 --- a/drivers/pci/controller/dwc/pci-layerscape.c +++ b/drivers/pci/controller/dwc/pci-layerscape.c @@ -232,7 +232,7 @@ static const struct of_device_id ls_pcie_of_match[] = { { }, }; -static int __init ls_pcie_probe(struct platform_device *pdev) +static int ls_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct dw_pcie *pci; @@ -271,10 +271,11 @@ static int __init ls_pcie_probe(struct platform_device *pdev) } static struct platform_driver ls_pcie_driver = { + .probe = ls_pcie_probe, .driver = { .name = "layerscape-pcie", .of_match_table = ls_pcie_of_match, .suppress_bind_attrs = true, }, }; -builtin_platform_driver_probe(ls_pcie_driver, ls_pcie_probe); +builtin_platform_driver(ls_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-al.c b/drivers/pci/controller/dwc/pcie-al.c index abf37aa68e51..e8afa50129a8 100644 --- a/drivers/pci/controller/dwc/pcie-al.c +++ b/drivers/pci/controller/dwc/pcie-al.c @@ -314,9 +314,6 @@ static const struct dw_pcie_host_ops al_pcie_host_ops = { .host_init = al_pcie_host_init, }; -static const struct dw_pcie_ops dw_pcie_ops = { -}; - static int al_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -334,7 +331,6 @@ static int al_pcie_probe(struct platform_device *pdev) return -ENOMEM; pci->dev = dev; - pci->ops = &dw_pcie_ops; pci->pp.ops = &al_pcie_host_ops; al_pcie->pci = pci; diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index bcd1cd9ba8c8..1c25d8337151 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -434,10 +434,8 @@ static void dw_pcie_ep_stop(struct pci_epc *epc) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); - if (!pci->ops->stop_link) - return; - - pci->ops->stop_link(pci); + if (pci->ops && pci->ops->stop_link) + pci->ops->stop_link(pci); } static int dw_pcie_ep_start(struct pci_epc *epc) @@ -445,7 +443,7 @@ static int dw_pcie_ep_start(struct pci_epc *epc) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); - if (!pci->ops->start_link) + if (!pci->ops || !pci->ops->start_link) return -EINVAL; return pci->ops->start_link(pci); diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index 8a84c005f32b..7e55b2b66182 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -258,10 +258,8 @@ int dw_pcie_allocate_domains(struct pcie_port *pp) static void dw_pcie_free_msi(struct pcie_port *pp) { - if (pp->msi_irq) { - irq_set_chained_handler(pp->msi_irq, NULL); - irq_set_handler_data(pp->msi_irq, NULL); - } + if (pp->msi_irq) + irq_set_chained_handler_and_data(pp->msi_irq, NULL, NULL); irq_domain_remove(pp->msi_domain); irq_domain_remove(pp->irq_domain); @@ -305,8 +303,13 @@ int dw_pcie_host_init(struct pcie_port *pp) if (cfg_res) { pp->cfg0_size = resource_size(cfg_res); pp->cfg0_base = cfg_res->start; - } else if (!pp->va_cfg0_base) { + + pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, cfg_res); + if (IS_ERR(pp->va_cfg0_base)) + return PTR_ERR(pp->va_cfg0_base); + } else { dev_err(dev, "Missing *config* reg space\n"); + return -ENODEV; } if (!pci->dbi_base) { @@ -322,38 +325,12 @@ int dw_pcie_host_init(struct pcie_port *pp) pp->bridge = bridge; - /* Get the I/O and memory ranges from DT */ - resource_list_for_each_entry(win, &bridge->windows) { - switch (resource_type(win->res)) { - case IORESOURCE_IO: - pp->io_size = resource_size(win->res); - pp->io_bus_addr = win->res->start - win->offset; - pp->io_base = pci_pio_to_address(win->res->start); - break; - case 0: - dev_err(dev, "Missing *config* reg space\n"); - pp->cfg0_size = resource_size(win->res); - pp->cfg0_base = win->res->start; - if (!pci->dbi_base) { - pci->dbi_base = devm_pci_remap_cfgspace(dev, - pp->cfg0_base, - pp->cfg0_size); - if (!pci->dbi_base) { - dev_err(dev, "Error with ioremap\n"); - return -ENOMEM; - } - } - break; - } - } - - if (!pp->va_cfg0_base) { - pp->va_cfg0_base = devm_pci_remap_cfgspace(dev, - pp->cfg0_base, pp->cfg0_size); - if (!pp->va_cfg0_base) { - dev_err(dev, "Error with ioremap in function\n"); - return -ENOMEM; - } + /* Get the I/O range from DT */ + win = resource_list_first_type(&bridge->windows, IORESOURCE_IO); + if (win) { + pp->io_size = resource_size(win->res); + pp->io_bus_addr = win->res->start - win->offset; + pp->io_base = pci_pio_to_address(win->res->start); } if (pci->link_gen < 1) @@ -425,7 +402,7 @@ int dw_pcie_host_init(struct pcie_port *pp) dw_pcie_setup_rc(pp); dw_pcie_msi_init(pp); - if (!dw_pcie_link_up(pci) && pci->ops->start_link) { + if (!dw_pcie_link_up(pci) && pci->ops && pci->ops->start_link) { ret = pci->ops->start_link(pci); if (ret) goto err_free_msi; diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 645fa1892375..004cb860e266 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -141,7 +141,7 @@ u32 dw_pcie_read_dbi(struct dw_pcie *pci, u32 reg, size_t size) int ret; u32 val; - if (pci->ops->read_dbi) + if (pci->ops && pci->ops->read_dbi) return pci->ops->read_dbi(pci, pci->dbi_base, reg, size); ret = dw_pcie_read(pci->dbi_base + reg, size, &val); @@ -156,7 +156,7 @@ void dw_pcie_write_dbi(struct dw_pcie *pci, u32 reg, size_t size, u32 val) { int ret; - if (pci->ops->write_dbi) { + if (pci->ops && pci->ops->write_dbi) { pci->ops->write_dbi(pci, pci->dbi_base, reg, size, val); return; } @@ -171,7 +171,7 @@ void dw_pcie_write_dbi2(struct dw_pcie *pci, u32 reg, size_t size, u32 val) { int ret; - if (pci->ops->write_dbi2) { + if (pci->ops && pci->ops->write_dbi2) { pci->ops->write_dbi2(pci, pci->dbi_base2, reg, size, val); return; } @@ -186,7 +186,7 @@ static u32 dw_pcie_readl_atu(struct dw_pcie *pci, u32 reg) int ret; u32 val; - if (pci->ops->read_dbi) + if (pci->ops && pci->ops->read_dbi) return pci->ops->read_dbi(pci, pci->atu_base, reg, 4); ret = dw_pcie_read(pci->atu_base + reg, 4, &val); @@ -200,7 +200,7 @@ static void dw_pcie_writel_atu(struct dw_pcie *pci, u32 reg, u32 val) { int ret; - if (pci->ops->write_dbi) { + if (pci->ops && pci->ops->write_dbi) { pci->ops->write_dbi(pci, pci->atu_base, reg, 4, val); return; } @@ -225,6 +225,47 @@ static void dw_pcie_writel_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg, dw_pcie_writel_atu(pci, offset + reg, val); } +static inline u32 dw_pcie_enable_ecrc(u32 val) +{ + /* + * DesignWare core version 4.90A has a design issue where the 'TD' + * bit in the Control register-1 of the ATU outbound region acts + * like an override for the ECRC setting, i.e., the presence of TLP + * Digest (ECRC) in the outgoing TLPs is solely determined by this + * bit. This is contrary to the PCIe spec which says that the + * enablement of the ECRC is solely determined by the AER + * registers. + * + * Because of this, even when the ECRC is enabled through AER + * registers, the transactions going through ATU won't have TLP + * Digest as there is no way the PCI core AER code could program + * the TD bit which is specific to the DesignWare core. + * + * The best way to handle this scenario is to program the TD bit + * always. It affects only the traffic from root port to downstream + * devices. + * + * At this point, + * When ECRC is enabled in AER registers, everything works normally + * When ECRC is NOT enabled in AER registers, then, + * on Root Port:- TLP Digest (DWord size) gets appended to each packet + * even through it is not required. Since downstream + * TLPs are mostly for configuration accesses and BAR + * accesses, they are not in critical path and won't + * have much negative effect on the performance. + * on End Point:- TLP Digest is received for some/all the packets coming + * from the root port. TLP Digest is ignored because, + * as per the PCIe Spec r5.0 v1.0 section 2.2.3 + * "TLP Digest Rules", when an endpoint receives TLP + * Digest when its ECRC check functionality is disabled + * in AER registers, received TLP Digest is just ignored. + * Since there is no issue or error reported either side, best way to + * handle the scenario is to program TD bit by default. + */ + + return val | PCIE_ATU_TD; +} + static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, u8 func_no, int index, int type, u64 cpu_addr, u64 pci_addr, @@ -248,6 +289,8 @@ static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, u8 func_no, val = type | PCIE_ATU_FUNC_NUM(func_no); val = upper_32_bits(size - 1) ? val | PCIE_ATU_INCREASE_REGION_SIZE : val; + if (pci->version == 0x490A) + val = dw_pcie_enable_ecrc(val); dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, val); dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2, PCIE_ATU_ENABLE); @@ -273,7 +316,7 @@ static void __dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 func_no, { u32 retries, val; - if (pci->ops->cpu_addr_fixup) + if (pci->ops && pci->ops->cpu_addr_fixup) cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr); if (pci->iatu_unroll_enabled) { @@ -290,12 +333,19 @@ static void __dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 func_no, upper_32_bits(cpu_addr)); dw_pcie_writel_dbi(pci, PCIE_ATU_LIMIT, lower_32_bits(cpu_addr + size - 1)); + if (pci->version >= 0x460A) + dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_LIMIT, + upper_32_bits(cpu_addr + size - 1)); dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET, lower_32_bits(pci_addr)); dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET, upper_32_bits(pci_addr)); - dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type | - PCIE_ATU_FUNC_NUM(func_no)); + val = type | PCIE_ATU_FUNC_NUM(func_no); + val = ((upper_32_bits(size - 1)) && (pci->version >= 0x460A)) ? + val | PCIE_ATU_INCREASE_REGION_SIZE : val; + if (pci->version == 0x490A) + val = dw_pcie_enable_ecrc(val); + dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, val); dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE); /* @@ -321,7 +371,7 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index, int type, u64 cpu_addr, u64 pci_addr, - u32 size) + u64 size) { __dw_pcie_prog_outbound_atu(pci, func_no, index, type, cpu_addr, pci_addr, size); @@ -481,7 +531,7 @@ int dw_pcie_link_up(struct dw_pcie *pci) { u32 val; - if (pci->ops->link_up) + if (pci->ops && pci->ops->link_up) return pci->ops->link_up(pci); val = readl(pci->dbi_base + PCIE_PORT_DEBUG1); diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 0207840756c4..7247c8b01f04 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -86,6 +86,7 @@ #define PCIE_ATU_TYPE_IO 0x2 #define PCIE_ATU_TYPE_CFG0 0x4 #define PCIE_ATU_TYPE_CFG1 0x5 +#define PCIE_ATU_TD BIT(8) #define PCIE_ATU_FUNC_NUM(pf) ((pf) << 20) #define PCIE_ATU_CR2 0x908 #define PCIE_ATU_ENABLE BIT(31) @@ -99,6 +100,7 @@ #define PCIE_ATU_DEV(x) FIELD_PREP(GENMASK(23, 19), x) #define PCIE_ATU_FUNC(x) FIELD_PREP(GENMASK(18, 16), x) #define PCIE_ATU_UPPER_TARGET 0x91C +#define PCIE_ATU_UPPER_LIMIT 0x924 #define PCIE_MISC_CONTROL_1_OFF 0x8BC #define PCIE_DBI_RO_WR_EN BIT(0) @@ -297,7 +299,7 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, u64 size); void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index, int type, u64 cpu_addr, u64 pci_addr, - u32 size); + u64 size); int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, int bar, u64 cpu_addr, enum dw_pcie_as_type as_type); diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index affa2713bf80..8a7a300163e5 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -159,8 +159,10 @@ struct qcom_pcie_resources_2_3_3 { struct reset_control *rst[7]; }; +/* 6 clocks typically, 7 for sm8250 */ struct qcom_pcie_resources_2_7_0 { - struct clk_bulk_data clks[6]; + struct clk_bulk_data clks[7]; + int num_clks; struct regulator_bulk_data supplies[2]; struct reset_control *pci_reset; struct clk *pipe_clk; @@ -398,7 +400,9 @@ static int qcom_pcie_init_2_1_0(struct qcom_pcie *pcie) /* enable external reference clock */ val = readl(pcie->parf + PCIE20_PARF_PHY_REFCLK); - val &= ~PHY_REFCLK_USE_PAD; + /* USE_PAD is required only for ipq806x */ + if (!of_device_is_compatible(node, "qcom,pcie-apq8064")) + val &= ~PHY_REFCLK_USE_PAD; val |= PHY_REFCLK_SSP_EN; writel(val, pcie->parf + PCIE20_PARF_PHY_REFCLK); @@ -1152,8 +1156,14 @@ static int qcom_pcie_get_resources_2_7_0(struct qcom_pcie *pcie) res->clks[3].id = "bus_slave"; res->clks[4].id = "slave_q2a"; res->clks[5].id = "tbu"; + if (of_device_is_compatible(dev->of_node, "qcom,pcie-sm8250")) { + res->clks[6].id = "ddrss_sf_tbu"; + res->num_clks = 7; + } else { + res->num_clks = 6; + } - ret = devm_clk_bulk_get(dev, ARRAY_SIZE(res->clks), res->clks); + ret = devm_clk_bulk_get(dev, res->num_clks, res->clks); if (ret < 0) return ret; @@ -1175,7 +1185,7 @@ static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie) return ret; } - ret = clk_bulk_prepare_enable(ARRAY_SIZE(res->clks), res->clks); + ret = clk_bulk_prepare_enable(res->num_clks, res->clks); if (ret < 0) goto err_disable_regulators; @@ -1227,7 +1237,7 @@ static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie) return 0; err_disable_clocks: - clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks); + clk_bulk_disable_unprepare(res->num_clks, res->clks); err_disable_regulators: regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); @@ -1238,7 +1248,7 @@ static void qcom_pcie_deinit_2_7_0(struct qcom_pcie *pcie) { struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; - clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks); + clk_bulk_disable_unprepare(res->num_clks, res->clks); regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); } diff --git a/drivers/pci/controller/pci-host-common.c b/drivers/pci/controller/pci-host-common.c index 6ce34a1deecb..6ab694f8d283 100644 --- a/drivers/pci/controller/pci-host-common.c +++ b/drivers/pci/controller/pci-host-common.c @@ -64,6 +64,8 @@ int pci_host_common_probe(struct platform_device *pdev) if (!bridge) return -ENOMEM; + platform_set_drvdata(pdev, bridge); + of_pci_check_probe_only(); /* Parse and map our Configuration Space windows */ @@ -78,8 +80,6 @@ int pci_host_common_probe(struct platform_device *pdev) bridge->sysdata = cfg; bridge->ops = (struct pci_ops *)&ops->pci_ops; - platform_set_drvdata(pdev, bridge); - return pci_host_probe(bridge); } EXPORT_SYMBOL_GPL(pci_host_common_probe); diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index 87aa62ee0368..27a17a1e4a7c 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -1714,7 +1714,7 @@ static void prepopulate_bars(struct hv_pcibus_device *hbus) * resumed and suspended again: see hibernation_snapshot() and * hibernation_platform_enter(). * - * If the memory enable bit is already set, Hyper-V sliently ignores + * If the memory enable bit is already set, Hyper-V silently ignores * the below BAR updates, and the related PCI device driver can not * work, because reading from the device register(s) always returns * 0xFFFFFFFF. diff --git a/drivers/pci/controller/pci-xgene-msi.c b/drivers/pci/controller/pci-xgene-msi.c index 2470782cb01a..1c34c897a7e2 100644 --- a/drivers/pci/controller/pci-xgene-msi.c +++ b/drivers/pci/controller/pci-xgene-msi.c @@ -384,13 +384,9 @@ static int xgene_msi_hwirq_alloc(unsigned int cpu) if (!msi_group->gic_irq) continue; - irq_set_chained_handler(msi_group->gic_irq, - xgene_msi_isr); - err = irq_set_handler_data(msi_group->gic_irq, msi_group); - if (err) { - pr_err("failed to register GIC IRQ handler\n"); - return -EINVAL; - } + irq_set_chained_handler_and_data(msi_group->gic_irq, + xgene_msi_isr, msi_group); + /* * Statically allocate MSI GIC IRQs to each CPU core. * With 8-core X-Gene v1, 2 MSI GIC IRQs are allocated diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c index 85e7c98265e8..2afdc865253e 100644 --- a/drivers/pci/controller/pci-xgene.c +++ b/drivers/pci/controller/pci-xgene.c @@ -173,12 +173,13 @@ static int xgene_pcie_config_read32(struct pci_bus *bus, unsigned int devfn, /* * The v1 controller has a bug in its Configuration Request - * Retry Status (CRS) logic: when CRS is enabled and we read the - * Vendor and Device ID of a non-existent device, the controller - * fabricates return data of 0xFFFF0001 ("device exists but is not - * ready") instead of 0xFFFFFFFF ("device does not exist"). This - * causes the PCI core to retry the read until it times out. - * Avoid this by not claiming to support CRS. + * Retry Status (CRS) logic: when CRS Software Visibility is + * enabled and we read the Vendor and Device ID of a non-existent + * device, the controller fabricates return data of 0xFFFF0001 + * ("device exists but is not ready") instead of 0xFFFFFFFF + * ("device does not exist"). This causes the PCI core to retry + * the read until it times out. Avoid this by not claiming to + * support CRS SV. */ if (pci_is_root_bus(bus) && (port->version == XGENE_PCIE_IP_VER_1) && ((where & ~0x3) == XGENE_V1_PCI_EXP_CAP + PCI_EXP_RTCTL)) diff --git a/drivers/pci/controller/pcie-altera-msi.c b/drivers/pci/controller/pcie-altera-msi.c index e1636f7714ca..42691dd8ebef 100644 --- a/drivers/pci/controller/pcie-altera-msi.c +++ b/drivers/pci/controller/pcie-altera-msi.c @@ -204,8 +204,7 @@ static int altera_msi_remove(struct platform_device *pdev) struct altera_msi *msi = platform_get_drvdata(pdev); msi_writel(msi, 0, MSI_INTMASK); - irq_set_chained_handler(msi->irq, NULL); - irq_set_handler_data(msi->irq, NULL); + irq_set_chained_handler_and_data(msi->irq, NULL, NULL); altera_free_domains(msi); diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index d41257f43a8f..e330e6811f0b 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -97,6 +97,7 @@ #define PCIE_MISC_REVISION 0x406c #define BRCM_PCIE_HW_REV_33 0x0303 +#define BRCM_PCIE_HW_REV_3_20 0x0320 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT 0x4070 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK 0xfff00000 @@ -187,6 +188,7 @@ struct brcm_pcie; static inline void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val); static inline void brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val); +static inline void brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val); static inline void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val); static inline void brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val); @@ -203,6 +205,7 @@ enum { enum pcie_type { GENERIC, + BCM4908, BCM7278, BCM2711, }; @@ -227,6 +230,13 @@ static const struct pcie_cfg_data generic_cfg = { .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, }; +static const struct pcie_cfg_data bcm4908_cfg = { + .offsets = pcie_offsets, + .type = BCM4908, + .perst_set = brcm_pcie_perst_set_4908, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, +}; + static const int pcie_offset_bcm7278[] = { [RGR1_SW_INIT_1] = 0xc010, [EXT_CFG_INDEX] = 0x9000, @@ -279,6 +289,7 @@ struct brcm_pcie { const int *reg_offsets; enum pcie_type type; struct reset_control *rescal; + struct reset_control *perst_reset; int num_memc; u64 memc_size[PCIE_BRCM_MAX_MEMC]; u32 hw_rev; @@ -603,8 +614,7 @@ static void brcm_msi_remove(struct brcm_pcie *pcie) if (!msi) return; - irq_set_chained_handler(msi->irq, NULL); - irq_set_handler_data(msi->irq, NULL); + irq_set_chained_handler_and_data(msi->irq, NULL, NULL); brcm_free_domains(msi); } @@ -735,6 +745,17 @@ static inline void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); } +static inline void brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val) +{ + if (WARN_ONCE(!pcie->perst_reset, "missing PERST# reset controller\n")) + return; + + if (val) + reset_control_assert(pcie->perst_reset); + else + reset_control_deassert(pcie->perst_reset); +} + static inline void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val) { u32 tmp; @@ -1194,6 +1215,7 @@ static int brcm_pcie_remove(struct platform_device *pdev) static const struct of_device_id brcm_pcie_match[] = { { .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg }, + { .compatible = "brcm,bcm4908-pcie", .data = &bcm4908_cfg }, { .compatible = "brcm,bcm7211-pcie", .data = &generic_cfg }, { .compatible = "brcm,bcm7278-pcie", .data = &bcm7278_cfg }, { .compatible = "brcm,bcm7216-pcie", .data = &bcm7278_cfg }, @@ -1250,6 +1272,11 @@ static int brcm_pcie_probe(struct platform_device *pdev) clk_disable_unprepare(pcie->clk); return PTR_ERR(pcie->rescal); } + pcie->perst_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "perst"); + if (IS_ERR(pcie->perst_reset)) { + clk_disable_unprepare(pcie->clk); + return PTR_ERR(pcie->perst_reset); + } ret = reset_control_deassert(pcie->rescal); if (ret) @@ -1267,6 +1294,10 @@ static int brcm_pcie_probe(struct platform_device *pdev) goto fail; pcie->hw_rev = readl(pcie->base + PCIE_MISC_REVISION); + if (pcie->type == BCM4908 && pcie->hw_rev >= BRCM_PCIE_HW_REV_3_20) { + dev_err(pcie->dev, "hardware revision with unsupported PERST# setup\n"); + goto fail; + } msi_np = of_parse_phandle(pcie->np, "msi-parent", 0); if (pci_msi_enabled() && msi_np == pcie->np) { diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index cf4c18f0c25a..23548b517e4b 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -1035,14 +1035,14 @@ static int mtk_pcie_setup(struct mtk_pcie *pcie) err = of_pci_get_devfn(child); if (err < 0) { dev_err(dev, "failed to parse devfn: %d\n", err); - return err; + goto error_put_node; } slot = PCI_SLOT(err); err = mtk_pcie_parse_port(pcie, child, slot); if (err) - return err; + goto error_put_node; } err = mtk_pcie_subsys_powerup(pcie); @@ -1058,6 +1058,9 @@ static int mtk_pcie_setup(struct mtk_pcie *pcie) mtk_pcie_subsys_powerdown(pcie); return 0; +error_put_node: + of_node_put(child); + return err; } static int mtk_pcie_probe(struct platform_device *pdev) diff --git a/drivers/pci/controller/pcie-microchip-host.c b/drivers/pci/controller/pcie-microchip-host.c new file mode 100644 index 000000000000..04c19ff81aff --- /dev/null +++ b/drivers/pci/controller/pcie-microchip-host.c @@ -0,0 +1,1138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip AXI PCIe Bridge host controller driver + * + * Copyright (c) 2018 - 2020 Microchip Corporation. All rights reserved. + * + * Author: Daire McNamara <daire.mcnamara@microchip.com> + */ + +#include <linux/clk.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/pci-ecam.h> +#include <linux/platform_device.h> + +#include "../pci.h" + +/* Number of MSI IRQs */ +#define MC_NUM_MSI_IRQS 32 +#define MC_NUM_MSI_IRQS_CODED 5 + +/* PCIe Bridge Phy and Controller Phy offsets */ +#define MC_PCIE1_BRIDGE_ADDR 0x00008000u +#define MC_PCIE1_CTRL_ADDR 0x0000a000u + +#define MC_PCIE_BRIDGE_ADDR (MC_PCIE1_BRIDGE_ADDR) +#define MC_PCIE_CTRL_ADDR (MC_PCIE1_CTRL_ADDR) + +/* PCIe Controller Phy Regs */ +#define SEC_ERROR_CNT 0x20 +#define DED_ERROR_CNT 0x24 +#define SEC_ERROR_INT 0x28 +#define SEC_ERROR_INT_TX_RAM_SEC_ERR_INT GENMASK(3, 0) +#define SEC_ERROR_INT_RX_RAM_SEC_ERR_INT GENMASK(7, 4) +#define SEC_ERROR_INT_PCIE2AXI_RAM_SEC_ERR_INT GENMASK(11, 8) +#define SEC_ERROR_INT_AXI2PCIE_RAM_SEC_ERR_INT GENMASK(15, 12) +#define NUM_SEC_ERROR_INTS (4) +#define SEC_ERROR_INT_MASK 0x2c +#define DED_ERROR_INT 0x30 +#define DED_ERROR_INT_TX_RAM_DED_ERR_INT GENMASK(3, 0) +#define DED_ERROR_INT_RX_RAM_DED_ERR_INT GENMASK(7, 4) +#define DED_ERROR_INT_PCIE2AXI_RAM_DED_ERR_INT GENMASK(11, 8) +#define DED_ERROR_INT_AXI2PCIE_RAM_DED_ERR_INT GENMASK(15, 12) +#define NUM_DED_ERROR_INTS (4) +#define DED_ERROR_INT_MASK 0x34 +#define ECC_CONTROL 0x38 +#define ECC_CONTROL_TX_RAM_INJ_ERROR_0 BIT(0) +#define ECC_CONTROL_TX_RAM_INJ_ERROR_1 BIT(1) +#define ECC_CONTROL_TX_RAM_INJ_ERROR_2 BIT(2) +#define ECC_CONTROL_TX_RAM_INJ_ERROR_3 BIT(3) +#define ECC_CONTROL_RX_RAM_INJ_ERROR_0 BIT(4) +#define ECC_CONTROL_RX_RAM_INJ_ERROR_1 BIT(5) +#define ECC_CONTROL_RX_RAM_INJ_ERROR_2 BIT(6) +#define ECC_CONTROL_RX_RAM_INJ_ERROR_3 BIT(7) +#define ECC_CONTROL_PCIE2AXI_RAM_INJ_ERROR_0 BIT(8) +#define ECC_CONTROL_PCIE2AXI_RAM_INJ_ERROR_1 BIT(9) +#define ECC_CONTROL_PCIE2AXI_RAM_INJ_ERROR_2 BIT(10) +#define ECC_CONTROL_PCIE2AXI_RAM_INJ_ERROR_3 BIT(11) +#define ECC_CONTROL_AXI2PCIE_RAM_INJ_ERROR_0 BIT(12) +#define ECC_CONTROL_AXI2PCIE_RAM_INJ_ERROR_1 BIT(13) +#define ECC_CONTROL_AXI2PCIE_RAM_INJ_ERROR_2 BIT(14) +#define ECC_CONTROL_AXI2PCIE_RAM_INJ_ERROR_3 BIT(15) +#define ECC_CONTROL_TX_RAM_ECC_BYPASS BIT(24) +#define ECC_CONTROL_RX_RAM_ECC_BYPASS BIT(25) +#define ECC_CONTROL_PCIE2AXI_RAM_ECC_BYPASS BIT(26) +#define ECC_CONTROL_AXI2PCIE_RAM_ECC_BYPASS BIT(27) +#define LTSSM_STATE 0x5c +#define LTSSM_L0_STATE 0x10 +#define PCIE_EVENT_INT 0x14c +#define PCIE_EVENT_INT_L2_EXIT_INT BIT(0) +#define PCIE_EVENT_INT_HOTRST_EXIT_INT BIT(1) +#define PCIE_EVENT_INT_DLUP_EXIT_INT BIT(2) +#define PCIE_EVENT_INT_MASK GENMASK(2, 0) +#define PCIE_EVENT_INT_L2_EXIT_INT_MASK BIT(16) +#define PCIE_EVENT_INT_HOTRST_EXIT_INT_MASK BIT(17) +#define PCIE_EVENT_INT_DLUP_EXIT_INT_MASK BIT(18) +#define PCIE_EVENT_INT_ENB_MASK GENMASK(18, 16) +#define PCIE_EVENT_INT_ENB_SHIFT 16 +#define NUM_PCIE_EVENTS (3) + +/* PCIe Bridge Phy Regs */ +#define PCIE_PCI_IDS_DW1 0x9c + +/* PCIe Config space MSI capability structure */ +#define MC_MSI_CAP_CTRL_OFFSET 0xe0u +#define MC_MSI_MAX_Q_AVAIL (MC_NUM_MSI_IRQS_CODED << 1) +#define MC_MSI_Q_SIZE (MC_NUM_MSI_IRQS_CODED << 4) + +#define IMASK_LOCAL 0x180 +#define DMA_END_ENGINE_0_MASK 0x00000000u +#define DMA_END_ENGINE_0_SHIFT 0 +#define DMA_END_ENGINE_1_MASK 0x00000000u +#define DMA_END_ENGINE_1_SHIFT 1 +#define DMA_ERROR_ENGINE_0_MASK 0x00000100u +#define DMA_ERROR_ENGINE_0_SHIFT 8 +#define DMA_ERROR_ENGINE_1_MASK 0x00000200u +#define DMA_ERROR_ENGINE_1_SHIFT 9 +#define A_ATR_EVT_POST_ERR_MASK 0x00010000u +#define A_ATR_EVT_POST_ERR_SHIFT 16 +#define A_ATR_EVT_FETCH_ERR_MASK 0x00020000u +#define A_ATR_EVT_FETCH_ERR_SHIFT 17 +#define A_ATR_EVT_DISCARD_ERR_MASK 0x00040000u +#define A_ATR_EVT_DISCARD_ERR_SHIFT 18 +#define A_ATR_EVT_DOORBELL_MASK 0x00000000u +#define A_ATR_EVT_DOORBELL_SHIFT 19 +#define P_ATR_EVT_POST_ERR_MASK 0x00100000u +#define P_ATR_EVT_POST_ERR_SHIFT 20 +#define P_ATR_EVT_FETCH_ERR_MASK 0x00200000u +#define P_ATR_EVT_FETCH_ERR_SHIFT 21 +#define P_ATR_EVT_DISCARD_ERR_MASK 0x00400000u +#define P_ATR_EVT_DISCARD_ERR_SHIFT 22 +#define P_ATR_EVT_DOORBELL_MASK 0x00000000u +#define P_ATR_EVT_DOORBELL_SHIFT 23 +#define PM_MSI_INT_INTA_MASK 0x01000000u +#define PM_MSI_INT_INTA_SHIFT 24 +#define PM_MSI_INT_INTB_MASK 0x02000000u +#define PM_MSI_INT_INTB_SHIFT 25 +#define PM_MSI_INT_INTC_MASK 0x04000000u +#define PM_MSI_INT_INTC_SHIFT 26 +#define PM_MSI_INT_INTD_MASK 0x08000000u +#define PM_MSI_INT_INTD_SHIFT 27 +#define PM_MSI_INT_INTX_MASK 0x0f000000u +#define PM_MSI_INT_INTX_SHIFT 24 +#define PM_MSI_INT_MSI_MASK 0x10000000u +#define PM_MSI_INT_MSI_SHIFT 28 +#define PM_MSI_INT_AER_EVT_MASK 0x20000000u +#define PM_MSI_INT_AER_EVT_SHIFT 29 +#define PM_MSI_INT_EVENTS_MASK 0x40000000u +#define PM_MSI_INT_EVENTS_SHIFT 30 +#define PM_MSI_INT_SYS_ERR_MASK 0x80000000u +#define PM_MSI_INT_SYS_ERR_SHIFT 31 +#define NUM_LOCAL_EVENTS 15 +#define ISTATUS_LOCAL 0x184 +#define IMASK_HOST 0x188 +#define ISTATUS_HOST 0x18c +#define MSI_ADDR 0x190 +#define ISTATUS_MSI 0x194 + +/* PCIe Master table init defines */ +#define ATR0_PCIE_WIN0_SRCADDR_PARAM 0x600u +#define ATR0_PCIE_ATR_SIZE 0x25 +#define ATR0_PCIE_ATR_SIZE_SHIFT 1 +#define ATR0_PCIE_WIN0_SRC_ADDR 0x604u +#define ATR0_PCIE_WIN0_TRSL_ADDR_LSB 0x608u +#define ATR0_PCIE_WIN0_TRSL_ADDR_UDW 0x60cu +#define ATR0_PCIE_WIN0_TRSL_PARAM 0x610u + +/* PCIe AXI slave table init defines */ +#define ATR0_AXI4_SLV0_SRCADDR_PARAM 0x800u +#define ATR_SIZE_SHIFT 1 +#define ATR_IMPL_ENABLE 1 +#define ATR0_AXI4_SLV0_SRC_ADDR 0x804u +#define ATR0_AXI4_SLV0_TRSL_ADDR_LSB 0x808u +#define ATR0_AXI4_SLV0_TRSL_ADDR_UDW 0x80cu +#define ATR0_AXI4_SLV0_TRSL_PARAM 0x810u +#define PCIE_TX_RX_INTERFACE 0x00000000u +#define PCIE_CONFIG_INTERFACE 0x00000001u + +#define ATR_ENTRY_SIZE 32 + +#define EVENT_PCIE_L2_EXIT 0 +#define EVENT_PCIE_HOTRST_EXIT 1 +#define EVENT_PCIE_DLUP_EXIT 2 +#define EVENT_SEC_TX_RAM_SEC_ERR 3 +#define EVENT_SEC_RX_RAM_SEC_ERR 4 +#define EVENT_SEC_AXI2PCIE_RAM_SEC_ERR 5 +#define EVENT_SEC_PCIE2AXI_RAM_SEC_ERR 6 +#define EVENT_DED_TX_RAM_DED_ERR 7 +#define EVENT_DED_RX_RAM_DED_ERR 8 +#define EVENT_DED_AXI2PCIE_RAM_DED_ERR 9 +#define EVENT_DED_PCIE2AXI_RAM_DED_ERR 10 +#define EVENT_LOCAL_DMA_END_ENGINE_0 11 +#define EVENT_LOCAL_DMA_END_ENGINE_1 12 +#define EVENT_LOCAL_DMA_ERROR_ENGINE_0 13 +#define EVENT_LOCAL_DMA_ERROR_ENGINE_1 14 +#define EVENT_LOCAL_A_ATR_EVT_POST_ERR 15 +#define EVENT_LOCAL_A_ATR_EVT_FETCH_ERR 16 +#define EVENT_LOCAL_A_ATR_EVT_DISCARD_ERR 17 +#define EVENT_LOCAL_A_ATR_EVT_DOORBELL 18 +#define EVENT_LOCAL_P_ATR_EVT_POST_ERR 19 +#define EVENT_LOCAL_P_ATR_EVT_FETCH_ERR 20 +#define EVENT_LOCAL_P_ATR_EVT_DISCARD_ERR 21 +#define EVENT_LOCAL_P_ATR_EVT_DOORBELL 22 +#define EVENT_LOCAL_PM_MSI_INT_INTX 23 +#define EVENT_LOCAL_PM_MSI_INT_MSI 24 +#define EVENT_LOCAL_PM_MSI_INT_AER_EVT 25 +#define EVENT_LOCAL_PM_MSI_INT_EVENTS 26 +#define EVENT_LOCAL_PM_MSI_INT_SYS_ERR 27 +#define NUM_EVENTS 28 + +#define PCIE_EVENT_CAUSE(x, s) \ + [EVENT_PCIE_ ## x] = { __stringify(x), s } + +#define SEC_ERROR_CAUSE(x, s) \ + [EVENT_SEC_ ## x] = { __stringify(x), s } + +#define DED_ERROR_CAUSE(x, s) \ + [EVENT_DED_ ## x] = { __stringify(x), s } + +#define LOCAL_EVENT_CAUSE(x, s) \ + [EVENT_LOCAL_ ## x] = { __stringify(x), s } + +#define PCIE_EVENT(x) \ + .base = MC_PCIE_CTRL_ADDR, \ + .offset = PCIE_EVENT_INT, \ + .mask_offset = PCIE_EVENT_INT, \ + .mask_high = 1, \ + .mask = PCIE_EVENT_INT_ ## x ## _INT, \ + .enb_mask = PCIE_EVENT_INT_ENB_MASK + +#define SEC_EVENT(x) \ + .base = MC_PCIE_CTRL_ADDR, \ + .offset = SEC_ERROR_INT, \ + .mask_offset = SEC_ERROR_INT_MASK, \ + .mask = SEC_ERROR_INT_ ## x ## _INT, \ + .mask_high = 1, \ + .enb_mask = 0 + +#define DED_EVENT(x) \ + .base = MC_PCIE_CTRL_ADDR, \ + .offset = DED_ERROR_INT, \ + .mask_offset = DED_ERROR_INT_MASK, \ + .mask_high = 1, \ + .mask = DED_ERROR_INT_ ## x ## _INT, \ + .enb_mask = 0 + +#define LOCAL_EVENT(x) \ + .base = MC_PCIE_BRIDGE_ADDR, \ + .offset = ISTATUS_LOCAL, \ + .mask_offset = IMASK_LOCAL, \ + .mask_high = 0, \ + .mask = x ## _MASK, \ + .enb_mask = 0 + +#define PCIE_EVENT_TO_EVENT_MAP(x) \ + { PCIE_EVENT_INT_ ## x ## _INT, EVENT_PCIE_ ## x } + +#define SEC_ERROR_TO_EVENT_MAP(x) \ + { SEC_ERROR_INT_ ## x ## _INT, EVENT_SEC_ ## x } + +#define DED_ERROR_TO_EVENT_MAP(x) \ + { DED_ERROR_INT_ ## x ## _INT, EVENT_DED_ ## x } + +#define LOCAL_STATUS_TO_EVENT_MAP(x) \ + { x ## _MASK, EVENT_LOCAL_ ## x } + +struct event_map { + u32 reg_mask; + u32 event_bit; +}; + +struct mc_msi { + struct mutex lock; /* Protect used bitmap */ + struct irq_domain *msi_domain; + struct irq_domain *dev_domain; + u32 num_vectors; + u64 vector_phy; + DECLARE_BITMAP(used, MC_NUM_MSI_IRQS); +}; + +struct mc_port { + void __iomem *axi_base_addr; + struct device *dev; + struct irq_domain *intx_domain; + struct irq_domain *event_domain; + raw_spinlock_t lock; + struct mc_msi msi; +}; + +struct cause { + const char *sym; + const char *str; +}; + +static const struct cause event_cause[NUM_EVENTS] = { + PCIE_EVENT_CAUSE(L2_EXIT, "L2 exit event"), + PCIE_EVENT_CAUSE(HOTRST_EXIT, "Hot reset exit event"), + PCIE_EVENT_CAUSE(DLUP_EXIT, "DLUP exit event"), + SEC_ERROR_CAUSE(TX_RAM_SEC_ERR, "sec error in tx buffer"), + SEC_ERROR_CAUSE(RX_RAM_SEC_ERR, "sec error in rx buffer"), + SEC_ERROR_CAUSE(PCIE2AXI_RAM_SEC_ERR, "sec error in pcie2axi buffer"), + SEC_ERROR_CAUSE(AXI2PCIE_RAM_SEC_ERR, "sec error in axi2pcie buffer"), + DED_ERROR_CAUSE(TX_RAM_DED_ERR, "ded error in tx buffer"), + DED_ERROR_CAUSE(RX_RAM_DED_ERR, "ded error in rx buffer"), + DED_ERROR_CAUSE(PCIE2AXI_RAM_DED_ERR, "ded error in pcie2axi buffer"), + DED_ERROR_CAUSE(AXI2PCIE_RAM_DED_ERR, "ded error in axi2pcie buffer"), + LOCAL_EVENT_CAUSE(DMA_ERROR_ENGINE_0, "dma engine 0 error"), + LOCAL_EVENT_CAUSE(DMA_ERROR_ENGINE_1, "dma engine 1 error"), + LOCAL_EVENT_CAUSE(A_ATR_EVT_POST_ERR, "axi write request error"), + LOCAL_EVENT_CAUSE(A_ATR_EVT_FETCH_ERR, "axi read request error"), + LOCAL_EVENT_CAUSE(A_ATR_EVT_DISCARD_ERR, "axi read timeout"), + LOCAL_EVENT_CAUSE(P_ATR_EVT_POST_ERR, "pcie write request error"), + LOCAL_EVENT_CAUSE(P_ATR_EVT_FETCH_ERR, "pcie read request error"), + LOCAL_EVENT_CAUSE(P_ATR_EVT_DISCARD_ERR, "pcie read timeout"), + LOCAL_EVENT_CAUSE(PM_MSI_INT_AER_EVT, "aer event"), + LOCAL_EVENT_CAUSE(PM_MSI_INT_EVENTS, "pm/ltr/hotplug event"), + LOCAL_EVENT_CAUSE(PM_MSI_INT_SYS_ERR, "system error"), +}; + +struct event_map pcie_event_to_event[] = { + PCIE_EVENT_TO_EVENT_MAP(L2_EXIT), + PCIE_EVENT_TO_EVENT_MAP(HOTRST_EXIT), + PCIE_EVENT_TO_EVENT_MAP(DLUP_EXIT), +}; + +struct event_map sec_error_to_event[] = { + SEC_ERROR_TO_EVENT_MAP(TX_RAM_SEC_ERR), + SEC_ERROR_TO_EVENT_MAP(RX_RAM_SEC_ERR), + SEC_ERROR_TO_EVENT_MAP(PCIE2AXI_RAM_SEC_ERR), + SEC_ERROR_TO_EVENT_MAP(AXI2PCIE_RAM_SEC_ERR), +}; + +struct event_map ded_error_to_event[] = { + DED_ERROR_TO_EVENT_MAP(TX_RAM_DED_ERR), + DED_ERROR_TO_EVENT_MAP(RX_RAM_DED_ERR), + DED_ERROR_TO_EVENT_MAP(PCIE2AXI_RAM_DED_ERR), + DED_ERROR_TO_EVENT_MAP(AXI2PCIE_RAM_DED_ERR), +}; + +struct event_map local_status_to_event[] = { + LOCAL_STATUS_TO_EVENT_MAP(DMA_END_ENGINE_0), + LOCAL_STATUS_TO_EVENT_MAP(DMA_END_ENGINE_1), + LOCAL_STATUS_TO_EVENT_MAP(DMA_ERROR_ENGINE_0), + LOCAL_STATUS_TO_EVENT_MAP(DMA_ERROR_ENGINE_1), + LOCAL_STATUS_TO_EVENT_MAP(A_ATR_EVT_POST_ERR), + LOCAL_STATUS_TO_EVENT_MAP(A_ATR_EVT_FETCH_ERR), + LOCAL_STATUS_TO_EVENT_MAP(A_ATR_EVT_DISCARD_ERR), + LOCAL_STATUS_TO_EVENT_MAP(A_ATR_EVT_DOORBELL), + LOCAL_STATUS_TO_EVENT_MAP(P_ATR_EVT_POST_ERR), + LOCAL_STATUS_TO_EVENT_MAP(P_ATR_EVT_FETCH_ERR), + LOCAL_STATUS_TO_EVENT_MAP(P_ATR_EVT_DISCARD_ERR), + LOCAL_STATUS_TO_EVENT_MAP(P_ATR_EVT_DOORBELL), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_INTX), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_MSI), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_AER_EVT), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_EVENTS), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_SYS_ERR), +}; + +struct { + u32 base; + u32 offset; + u32 mask; + u32 shift; + u32 enb_mask; + u32 mask_high; + u32 mask_offset; +} event_descs[] = { + { PCIE_EVENT(L2_EXIT) }, + { PCIE_EVENT(HOTRST_EXIT) }, + { PCIE_EVENT(DLUP_EXIT) }, + { SEC_EVENT(TX_RAM_SEC_ERR) }, + { SEC_EVENT(RX_RAM_SEC_ERR) }, + { SEC_EVENT(PCIE2AXI_RAM_SEC_ERR) }, + { SEC_EVENT(AXI2PCIE_RAM_SEC_ERR) }, + { DED_EVENT(TX_RAM_DED_ERR) }, + { DED_EVENT(RX_RAM_DED_ERR) }, + { DED_EVENT(PCIE2AXI_RAM_DED_ERR) }, + { DED_EVENT(AXI2PCIE_RAM_DED_ERR) }, + { LOCAL_EVENT(DMA_END_ENGINE_0) }, + { LOCAL_EVENT(DMA_END_ENGINE_1) }, + { LOCAL_EVENT(DMA_ERROR_ENGINE_0) }, + { LOCAL_EVENT(DMA_ERROR_ENGINE_1) }, + { LOCAL_EVENT(A_ATR_EVT_POST_ERR) }, + { LOCAL_EVENT(A_ATR_EVT_FETCH_ERR) }, + { LOCAL_EVENT(A_ATR_EVT_DISCARD_ERR) }, + { LOCAL_EVENT(A_ATR_EVT_DOORBELL) }, + { LOCAL_EVENT(P_ATR_EVT_POST_ERR) }, + { LOCAL_EVENT(P_ATR_EVT_FETCH_ERR) }, + { LOCAL_EVENT(P_ATR_EVT_DISCARD_ERR) }, + { LOCAL_EVENT(P_ATR_EVT_DOORBELL) }, + { LOCAL_EVENT(PM_MSI_INT_INTX) }, + { LOCAL_EVENT(PM_MSI_INT_MSI) }, + { LOCAL_EVENT(PM_MSI_INT_AER_EVT) }, + { LOCAL_EVENT(PM_MSI_INT_EVENTS) }, + { LOCAL_EVENT(PM_MSI_INT_SYS_ERR) }, +}; + +static char poss_clks[][5] = { "fic0", "fic1", "fic2", "fic3" }; + +static void mc_pcie_enable_msi(struct mc_port *port, void __iomem *base) +{ + struct mc_msi *msi = &port->msi; + u32 cap_offset = MC_MSI_CAP_CTRL_OFFSET; + u16 msg_ctrl = readw_relaxed(base + cap_offset + PCI_MSI_FLAGS); + + msg_ctrl |= PCI_MSI_FLAGS_ENABLE; + msg_ctrl &= ~PCI_MSI_FLAGS_QMASK; + msg_ctrl |= MC_MSI_MAX_Q_AVAIL; + msg_ctrl &= ~PCI_MSI_FLAGS_QSIZE; + msg_ctrl |= MC_MSI_Q_SIZE; + msg_ctrl |= PCI_MSI_FLAGS_64BIT; + + writew_relaxed(msg_ctrl, base + cap_offset + PCI_MSI_FLAGS); + + writel_relaxed(lower_32_bits(msi->vector_phy), + base + cap_offset + PCI_MSI_ADDRESS_LO); + writel_relaxed(upper_32_bits(msi->vector_phy), + base + cap_offset + PCI_MSI_ADDRESS_HI); +} + +static void mc_handle_msi(struct irq_desc *desc) +{ + struct mc_port *port = irq_desc_get_handler_data(desc); + struct device *dev = port->dev; + struct mc_msi *msi = &port->msi; + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + unsigned long status; + u32 bit; + u32 virq; + + status = readl_relaxed(bridge_base_addr + ISTATUS_LOCAL); + if (status & PM_MSI_INT_MSI_MASK) { + status = readl_relaxed(bridge_base_addr + ISTATUS_MSI); + for_each_set_bit(bit, &status, msi->num_vectors) { + virq = irq_find_mapping(msi->dev_domain, bit); + if (virq) + generic_handle_irq(virq); + else + dev_err_ratelimited(dev, "bad MSI IRQ %d\n", + bit); + } + } +} + +static void mc_msi_bottom_irq_ack(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + u32 bitpos = data->hwirq; + unsigned long status; + + writel_relaxed(BIT(bitpos), bridge_base_addr + ISTATUS_MSI); + status = readl_relaxed(bridge_base_addr + ISTATUS_MSI); + if (!status) + writel_relaxed(BIT(PM_MSI_INT_MSI_SHIFT), + bridge_base_addr + ISTATUS_LOCAL); +} + +static void mc_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + phys_addr_t addr = port->msi.vector_phy; + + msg->address_lo = lower_32_bits(addr); + msg->address_hi = upper_32_bits(addr); + msg->data = data->hwirq; + + dev_dbg(port->dev, "msi#%x address_hi %#x address_lo %#x\n", + (int)data->hwirq, msg->address_hi, msg->address_lo); +} + +static int mc_msi_set_affinity(struct irq_data *irq_data, + const struct cpumask *mask, bool force) +{ + return -EINVAL; +} + +static struct irq_chip mc_msi_bottom_irq_chip = { + .name = "Microchip MSI", + .irq_ack = mc_msi_bottom_irq_ack, + .irq_compose_msi_msg = mc_compose_msi_msg, + .irq_set_affinity = mc_msi_set_affinity, +}; + +static int mc_irq_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct mc_port *port = domain->host_data; + struct mc_msi *msi = &port->msi; + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + unsigned long bit; + u32 val; + + mutex_lock(&msi->lock); + bit = find_first_zero_bit(msi->used, msi->num_vectors); + if (bit >= msi->num_vectors) { + mutex_unlock(&msi->lock); + return -ENOSPC; + } + + set_bit(bit, msi->used); + + irq_domain_set_info(domain, virq, bit, &mc_msi_bottom_irq_chip, + domain->host_data, handle_edge_irq, NULL, NULL); + + /* Enable MSI interrupts */ + val = readl_relaxed(bridge_base_addr + IMASK_LOCAL); + val |= PM_MSI_INT_MSI_MASK; + writel_relaxed(val, bridge_base_addr + IMASK_LOCAL); + + mutex_unlock(&msi->lock); + + return 0; +} + +static void mc_irq_msi_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct mc_port *port = irq_data_get_irq_chip_data(d); + struct mc_msi *msi = &port->msi; + + mutex_lock(&msi->lock); + + if (test_bit(d->hwirq, msi->used)) + __clear_bit(d->hwirq, msi->used); + else + dev_err(port->dev, "trying to free unused MSI%lu\n", d->hwirq); + + mutex_unlock(&msi->lock); +} + +static const struct irq_domain_ops msi_domain_ops = { + .alloc = mc_irq_msi_domain_alloc, + .free = mc_irq_msi_domain_free, +}; + +static struct irq_chip mc_msi_irq_chip = { + .name = "Microchip PCIe MSI", + .irq_ack = irq_chip_ack_parent, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, +}; + +static struct msi_domain_info mc_msi_domain_info = { + .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_PCI_MSIX), + .chip = &mc_msi_irq_chip, +}; + +static int mc_allocate_msi_domains(struct mc_port *port) +{ + struct device *dev = port->dev; + struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node); + struct mc_msi *msi = &port->msi; + + mutex_init(&port->msi.lock); + + msi->dev_domain = irq_domain_add_linear(NULL, msi->num_vectors, + &msi_domain_ops, port); + if (!msi->dev_domain) { + dev_err(dev, "failed to create IRQ domain\n"); + return -ENOMEM; + } + + msi->msi_domain = pci_msi_create_irq_domain(fwnode, &mc_msi_domain_info, + msi->dev_domain); + if (!msi->msi_domain) { + dev_err(dev, "failed to create MSI domain\n"); + irq_domain_remove(msi->dev_domain); + return -ENOMEM; + } + + return 0; +} + +static void mc_handle_intx(struct irq_desc *desc) +{ + struct mc_port *port = irq_desc_get_handler_data(desc); + struct device *dev = port->dev; + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + unsigned long status; + u32 bit; + u32 virq; + + status = readl_relaxed(bridge_base_addr + ISTATUS_LOCAL); + if (status & PM_MSI_INT_INTX_MASK) { + status &= PM_MSI_INT_INTX_MASK; + status >>= PM_MSI_INT_INTX_SHIFT; + for_each_set_bit(bit, &status, PCI_NUM_INTX) { + virq = irq_find_mapping(port->intx_domain, bit); + if (virq) + generic_handle_irq(virq); + else + dev_err_ratelimited(dev, "bad INTx IRQ %d\n", + bit); + } + } +} + +static void mc_ack_intx_irq(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + u32 mask = BIT(data->hwirq + PM_MSI_INT_INTX_SHIFT); + + writel_relaxed(mask, bridge_base_addr + ISTATUS_LOCAL); +} + +static void mc_mask_intx_irq(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + unsigned long flags; + u32 mask = BIT(data->hwirq + PM_MSI_INT_INTX_SHIFT); + u32 val; + + raw_spin_lock_irqsave(&port->lock, flags); + val = readl_relaxed(bridge_base_addr + IMASK_LOCAL); + val &= ~mask; + writel_relaxed(val, bridge_base_addr + IMASK_LOCAL); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static void mc_unmask_intx_irq(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + unsigned long flags; + u32 mask = BIT(data->hwirq + PM_MSI_INT_INTX_SHIFT); + u32 val; + + raw_spin_lock_irqsave(&port->lock, flags); + val = readl_relaxed(bridge_base_addr + IMASK_LOCAL); + val |= mask; + writel_relaxed(val, bridge_base_addr + IMASK_LOCAL); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static struct irq_chip mc_intx_irq_chip = { + .name = "Microchip PCIe INTx", + .irq_ack = mc_ack_intx_irq, + .irq_mask = mc_mask_intx_irq, + .irq_unmask = mc_unmask_intx_irq, +}; + +static int mc_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &mc_intx_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops intx_domain_ops = { + .map = mc_pcie_intx_map, +}; + +static inline u32 reg_to_event(u32 reg, struct event_map field) +{ + return (reg & field.reg_mask) ? BIT(field.event_bit) : 0; +} + +static u32 pcie_events(void __iomem *addr) +{ + u32 reg = readl_relaxed(addr); + u32 val = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(pcie_event_to_event); i++) + val |= reg_to_event(reg, pcie_event_to_event[i]); + + return val; +} + +static u32 sec_errors(void __iomem *addr) +{ + u32 reg = readl_relaxed(addr); + u32 val = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(sec_error_to_event); i++) + val |= reg_to_event(reg, sec_error_to_event[i]); + + return val; +} + +static u32 ded_errors(void __iomem *addr) +{ + u32 reg = readl_relaxed(addr); + u32 val = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(ded_error_to_event); i++) + val |= reg_to_event(reg, ded_error_to_event[i]); + + return val; +} + +static u32 local_events(void __iomem *addr) +{ + u32 reg = readl_relaxed(addr); + u32 val = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(local_status_to_event); i++) + val |= reg_to_event(reg, local_status_to_event[i]); + + return val; +} + +static u32 get_events(struct mc_port *port) +{ + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + void __iomem *ctrl_base_addr = port->axi_base_addr + MC_PCIE_CTRL_ADDR; + u32 events = 0; + + events |= pcie_events(ctrl_base_addr + PCIE_EVENT_INT); + events |= sec_errors(ctrl_base_addr + SEC_ERROR_INT); + events |= ded_errors(ctrl_base_addr + DED_ERROR_INT); + events |= local_events(bridge_base_addr + ISTATUS_LOCAL); + + return events; +} + +static irqreturn_t mc_event_handler(int irq, void *dev_id) +{ + struct mc_port *port = dev_id; + struct device *dev = port->dev; + struct irq_data *data; + + data = irq_domain_get_irq_data(port->event_domain, irq); + + if (event_cause[data->hwirq].str) + dev_err_ratelimited(dev, "%s\n", event_cause[data->hwirq].str); + else + dev_err_ratelimited(dev, "bad event IRQ %ld\n", data->hwirq); + + return IRQ_HANDLED; +} + +static void mc_handle_event(struct irq_desc *desc) +{ + struct mc_port *port = irq_desc_get_handler_data(desc); + unsigned long events; + u32 bit; + struct irq_chip *chip = irq_desc_get_chip(desc); + + chained_irq_enter(chip, desc); + + events = get_events(port); + + for_each_set_bit(bit, &events, NUM_EVENTS) + generic_handle_irq(irq_find_mapping(port->event_domain, bit)); + + chained_irq_exit(chip, desc); +} + +static void mc_ack_event_irq(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + u32 event = data->hwirq; + void __iomem *addr; + u32 mask; + + addr = port->axi_base_addr + event_descs[event].base + + event_descs[event].offset; + mask = event_descs[event].mask; + mask |= event_descs[event].enb_mask; + + writel_relaxed(mask, addr); +} + +static void mc_mask_event_irq(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + u32 event = data->hwirq; + void __iomem *addr; + u32 mask; + u32 val; + + addr = port->axi_base_addr + event_descs[event].base + + event_descs[event].mask_offset; + mask = event_descs[event].mask; + if (event_descs[event].enb_mask) { + mask <<= PCIE_EVENT_INT_ENB_SHIFT; + mask &= PCIE_EVENT_INT_ENB_MASK; + } + + if (!event_descs[event].mask_high) + mask = ~mask; + + raw_spin_lock(&port->lock); + val = readl_relaxed(addr); + if (event_descs[event].mask_high) + val |= mask; + else + val &= mask; + + writel_relaxed(val, addr); + raw_spin_unlock(&port->lock); +} + +static void mc_unmask_event_irq(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + u32 event = data->hwirq; + void __iomem *addr; + u32 mask; + u32 val; + + addr = port->axi_base_addr + event_descs[event].base + + event_descs[event].mask_offset; + mask = event_descs[event].mask; + + if (event_descs[event].enb_mask) + mask <<= PCIE_EVENT_INT_ENB_SHIFT; + + if (event_descs[event].mask_high) + mask = ~mask; + + if (event_descs[event].enb_mask) + mask &= PCIE_EVENT_INT_ENB_MASK; + + raw_spin_lock(&port->lock); + val = readl_relaxed(addr); + if (event_descs[event].mask_high) + val &= mask; + else + val |= mask; + writel_relaxed(val, addr); + raw_spin_unlock(&port->lock); +} + +static struct irq_chip mc_event_irq_chip = { + .name = "Microchip PCIe EVENT", + .irq_ack = mc_ack_event_irq, + .irq_mask = mc_mask_event_irq, + .irq_unmask = mc_unmask_event_irq, +}; + +static int mc_pcie_event_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &mc_event_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops event_domain_ops = { + .map = mc_pcie_event_map, +}; + +static inline struct clk *mc_pcie_init_clk(struct device *dev, const char *id) +{ + struct clk *clk; + int ret; + + clk = devm_clk_get_optional(dev, id); + if (IS_ERR(clk)) + return clk; + if (!clk) + return clk; + + ret = clk_prepare_enable(clk); + if (ret) + return ERR_PTR(ret); + + devm_add_action_or_reset(dev, (void (*) (void *))clk_disable_unprepare, + clk); + + return clk; +} + +static int mc_pcie_init_clks(struct device *dev) +{ + int i; + struct clk *fic; + + /* + * PCIe may be clocked via Fabric Interface using between 1 and 4 + * clocks. Scan DT for clocks and enable them if present + */ + for (i = 0; i < ARRAY_SIZE(poss_clks); i++) { + fic = mc_pcie_init_clk(dev, poss_clks[i]); + if (IS_ERR(fic)) + return PTR_ERR(fic); + } + + return 0; +} + +static int mc_pcie_init_irq_domains(struct mc_port *port) +{ + struct device *dev = port->dev; + struct device_node *node = dev->of_node; + struct device_node *pcie_intc_node; + + /* Setup INTx */ + pcie_intc_node = of_get_next_child(node, NULL); + if (!pcie_intc_node) { + dev_err(dev, "failed to find PCIe Intc node\n"); + return -EINVAL; + } + + port->event_domain = irq_domain_add_linear(pcie_intc_node, NUM_EVENTS, + &event_domain_ops, port); + if (!port->event_domain) { + dev_err(dev, "failed to get event domain\n"); + return -ENOMEM; + } + + irq_domain_update_bus_token(port->event_domain, DOMAIN_BUS_NEXUS); + + port->intx_domain = irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX, + &intx_domain_ops, port); + if (!port->intx_domain) { + dev_err(dev, "failed to get an INTx IRQ domain\n"); + return -ENOMEM; + } + + irq_domain_update_bus_token(port->intx_domain, DOMAIN_BUS_WIRED); + + of_node_put(pcie_intc_node); + raw_spin_lock_init(&port->lock); + + return mc_allocate_msi_domains(port); +} + +static void mc_pcie_setup_window(void __iomem *bridge_base_addr, u32 index, + phys_addr_t axi_addr, phys_addr_t pci_addr, + size_t size) +{ + u32 atr_sz = ilog2(size) - 1; + u32 val; + + if (index == 0) + val = PCIE_CONFIG_INTERFACE; + else + val = PCIE_TX_RX_INTERFACE; + + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_TRSL_PARAM); + + val = lower_32_bits(axi_addr) | (atr_sz << ATR_SIZE_SHIFT) | + ATR_IMPL_ENABLE; + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_SRCADDR_PARAM); + + val = upper_32_bits(axi_addr); + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_SRC_ADDR); + + val = lower_32_bits(pci_addr); + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_TRSL_ADDR_LSB); + + val = upper_32_bits(pci_addr); + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_TRSL_ADDR_UDW); + + val = readl(bridge_base_addr + ATR0_PCIE_WIN0_SRCADDR_PARAM); + val |= (ATR0_PCIE_ATR_SIZE << ATR0_PCIE_ATR_SIZE_SHIFT); + writel(val, bridge_base_addr + ATR0_PCIE_WIN0_SRCADDR_PARAM); + writel(0, bridge_base_addr + ATR0_PCIE_WIN0_SRC_ADDR); +} + +static int mc_pcie_setup_windows(struct platform_device *pdev, + struct mc_port *port) +{ + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + struct pci_host_bridge *bridge = platform_get_drvdata(pdev); + struct resource_entry *entry; + u64 pci_addr; + u32 index = 1; + + resource_list_for_each_entry(entry, &bridge->windows) { + if (resource_type(entry->res) == IORESOURCE_MEM) { + pci_addr = entry->res->start - entry->offset; + mc_pcie_setup_window(bridge_base_addr, index, + entry->res->start, pci_addr, + resource_size(entry->res)); + index++; + } + } + + return 0; +} + +static int mc_platform_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct platform_device *pdev = to_platform_device(dev); + struct mc_port *port; + void __iomem *bridge_base_addr; + void __iomem *ctrl_base_addr; + int ret; + int irq; + int i, intx_irq, msi_irq, event_irq; + u32 val; + int err; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + port->dev = dev; + + ret = mc_pcie_init_clks(dev); + if (ret) { + dev_err(dev, "failed to get clock resources, error %d\n", ret); + return -ENODEV; + } + + port->axi_base_addr = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(port->axi_base_addr)) + return PTR_ERR(port->axi_base_addr); + + bridge_base_addr = port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + ctrl_base_addr = port->axi_base_addr + MC_PCIE_CTRL_ADDR; + + port->msi.vector_phy = MSI_ADDR; + port->msi.num_vectors = MC_NUM_MSI_IRQS; + ret = mc_pcie_init_irq_domains(port); + if (ret) { + dev_err(dev, "failed creating IRQ domains\n"); + return ret; + } + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(dev, "unable to request IRQ%d\n", irq); + return -ENODEV; + } + + for (i = 0; i < NUM_EVENTS; i++) { + event_irq = irq_create_mapping(port->event_domain, i); + if (!event_irq) { + dev_err(dev, "failed to map hwirq %d\n", i); + return -ENXIO; + } + + err = devm_request_irq(dev, event_irq, mc_event_handler, + 0, event_cause[i].sym, port); + if (err) { + dev_err(dev, "failed to request IRQ %d\n", event_irq); + return err; + } + } + + intx_irq = irq_create_mapping(port->event_domain, + EVENT_LOCAL_PM_MSI_INT_INTX); + if (!intx_irq) { + dev_err(dev, "failed to map INTx interrupt\n"); + return -ENXIO; + } + + /* Plug the INTx chained handler */ + irq_set_chained_handler_and_data(intx_irq, mc_handle_intx, port); + + msi_irq = irq_create_mapping(port->event_domain, + EVENT_LOCAL_PM_MSI_INT_MSI); + if (!msi_irq) + return -ENXIO; + + /* Plug the MSI chained handler */ + irq_set_chained_handler_and_data(msi_irq, mc_handle_msi, port); + + /* Plug the main event chained handler */ + irq_set_chained_handler_and_data(irq, mc_handle_event, port); + + /* Hardware doesn't setup MSI by default */ + mc_pcie_enable_msi(port, cfg->win); + + val = readl_relaxed(bridge_base_addr + IMASK_LOCAL); + val |= PM_MSI_INT_INTX_MASK; + writel_relaxed(val, bridge_base_addr + IMASK_LOCAL); + + writel_relaxed(val, ctrl_base_addr + ECC_CONTROL); + + val = PCIE_EVENT_INT_L2_EXIT_INT | + PCIE_EVENT_INT_HOTRST_EXIT_INT | + PCIE_EVENT_INT_DLUP_EXIT_INT; + writel_relaxed(val, ctrl_base_addr + PCIE_EVENT_INT); + + val = SEC_ERROR_INT_TX_RAM_SEC_ERR_INT | + SEC_ERROR_INT_RX_RAM_SEC_ERR_INT | + SEC_ERROR_INT_PCIE2AXI_RAM_SEC_ERR_INT | + SEC_ERROR_INT_AXI2PCIE_RAM_SEC_ERR_INT; + writel_relaxed(val, ctrl_base_addr + SEC_ERROR_INT); + writel_relaxed(0, ctrl_base_addr + SEC_ERROR_INT_MASK); + writel_relaxed(0, ctrl_base_addr + SEC_ERROR_CNT); + + val = DED_ERROR_INT_TX_RAM_DED_ERR_INT | + DED_ERROR_INT_RX_RAM_DED_ERR_INT | + DED_ERROR_INT_PCIE2AXI_RAM_DED_ERR_INT | + DED_ERROR_INT_AXI2PCIE_RAM_DED_ERR_INT; + writel_relaxed(val, ctrl_base_addr + DED_ERROR_INT); + writel_relaxed(0, ctrl_base_addr + DED_ERROR_INT_MASK); + writel_relaxed(0, ctrl_base_addr + DED_ERROR_CNT); + + writel_relaxed(0, bridge_base_addr + IMASK_HOST); + writel_relaxed(GENMASK(31, 0), bridge_base_addr + ISTATUS_HOST); + + /* Configure Address Translation Table 0 for PCIe config space */ + mc_pcie_setup_window(bridge_base_addr, 0, cfg->res.start & 0xffffffff, + cfg->res.start, resource_size(&cfg->res)); + + return mc_pcie_setup_windows(pdev, port); +} + +static const struct pci_ecam_ops mc_ecam_ops = { + .init = mc_platform_init, + .pci_ops = { + .map_bus = pci_ecam_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +static const struct of_device_id mc_pcie_of_match[] = { + { + .compatible = "microchip,pcie-host-1.0", + .data = &mc_ecam_ops, + }, + {}, +}; + +MODULE_DEVICE_TABLE(of, mc_pcie_of_match) + +static struct platform_driver mc_pcie_driver = { + .probe = pci_host_common_probe, + .driver = { + .name = "microchip-pcie", + .of_match_table = mc_pcie_of_match, + .suppress_bind_attrs = true, + }, +}; + +builtin_platform_driver(mc_pcie_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Microchip PCIe host controller driver"); +MODULE_AUTHOR("Daire McNamara <daire.mcnamara@microchip.com>"); diff --git a/drivers/pci/controller/pcie-rcar-host.c b/drivers/pci/controller/pcie-rcar-host.c index 4d1c4b24e537..a728e8f9ad3c 100644 --- a/drivers/pci/controller/pcie-rcar-host.c +++ b/drivers/pci/controller/pcie-rcar-host.c @@ -735,7 +735,7 @@ static int rcar_pcie_enable_msi(struct rcar_pcie_host *host) } /* setup MSI data target */ - msi->pages = __get_free_pages(GFP_KERNEL, 0); + msi->pages = __get_free_pages(GFP_KERNEL | GFP_DMA32, 0); rcar_pcie_hw_enable_msi(host); return 0; diff --git a/drivers/pci/controller/pcie-rockchip.c b/drivers/pci/controller/pcie-rockchip.c index 904dec0d3a88..990a00e08bc5 100644 --- a/drivers/pci/controller/pcie-rockchip.c +++ b/drivers/pci/controller/pcie-rockchip.c @@ -82,7 +82,7 @@ int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip) } rockchip->mgmt_sticky_rst = devm_reset_control_get_exclusive(dev, - "mgmt-sticky"); + "mgmt-sticky"); if (IS_ERR(rockchip->mgmt_sticky_rst)) { if (PTR_ERR(rockchip->mgmt_sticky_rst) != -EPROBE_DEFER) dev_err(dev, "missing mgmt-sticky reset property in node\n"); @@ -118,11 +118,11 @@ int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip) } if (rockchip->is_rc) { - rockchip->ep_gpio = devm_gpiod_get(dev, "ep", GPIOD_OUT_HIGH); - if (IS_ERR(rockchip->ep_gpio)) { - dev_err(dev, "missing ep-gpios property in node\n"); - return PTR_ERR(rockchip->ep_gpio); - } + rockchip->ep_gpio = devm_gpiod_get_optional(dev, "ep", + GPIOD_OUT_HIGH); + if (IS_ERR(rockchip->ep_gpio)) + return dev_err_probe(dev, PTR_ERR(rockchip->ep_gpio), + "failed to get ep GPIO\n"); } rockchip->aclk_pcie = devm_clk_get(dev, "aclk"); diff --git a/drivers/pci/controller/pcie-tango.c b/drivers/pci/controller/pcie-tango.c deleted file mode 100644 index 62a061f1d62e..000000000000 --- a/drivers/pci/controller/pcie-tango.c +++ /dev/null @@ -1,341 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include <linux/irqchip/chained_irq.h> -#include <linux/irqdomain.h> -#include <linux/pci-ecam.h> -#include <linux/delay.h> -#include <linux/msi.h> -#include <linux/of_address.h> - -#define MSI_MAX 256 - -#define SMP8759_MUX 0x48 -#define SMP8759_TEST_OUT 0x74 -#define SMP8759_DOORBELL 0x7c -#define SMP8759_STATUS 0x80 -#define SMP8759_ENABLE 0xa0 - -struct tango_pcie { - DECLARE_BITMAP(used_msi, MSI_MAX); - u64 msi_doorbell; - spinlock_t used_msi_lock; - void __iomem *base; - struct irq_domain *dom; -}; - -static void tango_msi_isr(struct irq_desc *desc) -{ - struct irq_chip *chip = irq_desc_get_chip(desc); - struct tango_pcie *pcie = irq_desc_get_handler_data(desc); - unsigned long status, base, virq, idx, pos = 0; - - chained_irq_enter(chip, desc); - spin_lock(&pcie->used_msi_lock); - - while ((pos = find_next_bit(pcie->used_msi, MSI_MAX, pos)) < MSI_MAX) { - base = round_down(pos, 32); - status = readl_relaxed(pcie->base + SMP8759_STATUS + base / 8); - for_each_set_bit(idx, &status, 32) { - virq = irq_find_mapping(pcie->dom, base + idx); - generic_handle_irq(virq); - } - pos = base + 32; - } - - spin_unlock(&pcie->used_msi_lock); - chained_irq_exit(chip, desc); -} - -static void tango_ack(struct irq_data *d) -{ - struct tango_pcie *pcie = d->chip_data; - u32 offset = (d->hwirq / 32) * 4; - u32 bit = BIT(d->hwirq % 32); - - writel_relaxed(bit, pcie->base + SMP8759_STATUS + offset); -} - -static void update_msi_enable(struct irq_data *d, bool unmask) -{ - unsigned long flags; - struct tango_pcie *pcie = d->chip_data; - u32 offset = (d->hwirq / 32) * 4; - u32 bit = BIT(d->hwirq % 32); - u32 val; - - spin_lock_irqsave(&pcie->used_msi_lock, flags); - val = readl_relaxed(pcie->base + SMP8759_ENABLE + offset); - val = unmask ? val | bit : val & ~bit; - writel_relaxed(val, pcie->base + SMP8759_ENABLE + offset); - spin_unlock_irqrestore(&pcie->used_msi_lock, flags); -} - -static void tango_mask(struct irq_data *d) -{ - update_msi_enable(d, false); -} - -static void tango_unmask(struct irq_data *d) -{ - update_msi_enable(d, true); -} - -static int tango_set_affinity(struct irq_data *d, const struct cpumask *mask, - bool force) -{ - return -EINVAL; -} - -static void tango_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) -{ - struct tango_pcie *pcie = d->chip_data; - msg->address_lo = lower_32_bits(pcie->msi_doorbell); - msg->address_hi = upper_32_bits(pcie->msi_doorbell); - msg->data = d->hwirq; -} - -static struct irq_chip tango_chip = { - .irq_ack = tango_ack, - .irq_mask = tango_mask, - .irq_unmask = tango_unmask, - .irq_set_affinity = tango_set_affinity, - .irq_compose_msi_msg = tango_compose_msi_msg, -}; - -static void msi_ack(struct irq_data *d) -{ - irq_chip_ack_parent(d); -} - -static void msi_mask(struct irq_data *d) -{ - pci_msi_mask_irq(d); - irq_chip_mask_parent(d); -} - -static void msi_unmask(struct irq_data *d) -{ - pci_msi_unmask_irq(d); - irq_chip_unmask_parent(d); -} - -static struct irq_chip msi_chip = { - .name = "MSI", - .irq_ack = msi_ack, - .irq_mask = msi_mask, - .irq_unmask = msi_unmask, -}; - -static struct msi_domain_info msi_dom_info = { - .flags = MSI_FLAG_PCI_MSIX - | MSI_FLAG_USE_DEF_DOM_OPS - | MSI_FLAG_USE_DEF_CHIP_OPS, - .chip = &msi_chip, -}; - -static int tango_irq_domain_alloc(struct irq_domain *dom, unsigned int virq, - unsigned int nr_irqs, void *args) -{ - struct tango_pcie *pcie = dom->host_data; - unsigned long flags; - int pos; - - spin_lock_irqsave(&pcie->used_msi_lock, flags); - pos = find_first_zero_bit(pcie->used_msi, MSI_MAX); - if (pos >= MSI_MAX) { - spin_unlock_irqrestore(&pcie->used_msi_lock, flags); - return -ENOSPC; - } - __set_bit(pos, pcie->used_msi); - spin_unlock_irqrestore(&pcie->used_msi_lock, flags); - irq_domain_set_info(dom, virq, pos, &tango_chip, - pcie, handle_edge_irq, NULL, NULL); - - return 0; -} - -static void tango_irq_domain_free(struct irq_domain *dom, unsigned int virq, - unsigned int nr_irqs) -{ - unsigned long flags; - struct irq_data *d = irq_domain_get_irq_data(dom, virq); - struct tango_pcie *pcie = d->chip_data; - - spin_lock_irqsave(&pcie->used_msi_lock, flags); - __clear_bit(d->hwirq, pcie->used_msi); - spin_unlock_irqrestore(&pcie->used_msi_lock, flags); -} - -static const struct irq_domain_ops dom_ops = { - .alloc = tango_irq_domain_alloc, - .free = tango_irq_domain_free, -}; - -static int smp8759_config_read(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 *val) -{ - struct pci_config_window *cfg = bus->sysdata; - struct tango_pcie *pcie = dev_get_drvdata(cfg->parent); - int ret; - - /* Reads in configuration space outside devfn 0 return garbage */ - if (devfn != 0) - return PCIBIOS_FUNC_NOT_SUPPORTED; - - /* - * PCI config and MMIO accesses are muxed. Linux doesn't have a - * mutual exclusion mechanism for config vs. MMIO accesses, so - * concurrent accesses may cause corruption. - */ - writel_relaxed(1, pcie->base + SMP8759_MUX); - ret = pci_generic_config_read(bus, devfn, where, size, val); - writel_relaxed(0, pcie->base + SMP8759_MUX); - - return ret; -} - -static int smp8759_config_write(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 val) -{ - struct pci_config_window *cfg = bus->sysdata; - struct tango_pcie *pcie = dev_get_drvdata(cfg->parent); - int ret; - - writel_relaxed(1, pcie->base + SMP8759_MUX); - ret = pci_generic_config_write(bus, devfn, where, size, val); - writel_relaxed(0, pcie->base + SMP8759_MUX); - - return ret; -} - -static const struct pci_ecam_ops smp8759_ecam_ops = { - .pci_ops = { - .map_bus = pci_ecam_map_bus, - .read = smp8759_config_read, - .write = smp8759_config_write, - } -}; - -static int tango_pcie_link_up(struct tango_pcie *pcie) -{ - void __iomem *test_out = pcie->base + SMP8759_TEST_OUT; - int i; - - writel_relaxed(16, test_out); - for (i = 0; i < 10; ++i) { - u32 ltssm_state = readl_relaxed(test_out) >> 8; - if ((ltssm_state & 0x1f) == 0xf) /* L0 */ - return 1; - usleep_range(3000, 4000); - } - - return 0; -} - -static int tango_pcie_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct tango_pcie *pcie; - struct resource *res; - struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node); - struct irq_domain *msi_dom, *irq_dom; - struct of_pci_range_parser parser; - struct of_pci_range range; - int virq, offset; - - dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n"); - add_taint(TAINT_CRAP, LOCKDEP_STILL_OK); - - pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); - if (!pcie) - return -ENOMEM; - - res = platform_get_resource(pdev, IORESOURCE_MEM, 1); - pcie->base = devm_ioremap_resource(dev, res); - if (IS_ERR(pcie->base)) - return PTR_ERR(pcie->base); - - platform_set_drvdata(pdev, pcie); - - if (!tango_pcie_link_up(pcie)) - return -ENODEV; - - if (of_pci_dma_range_parser_init(&parser, dev->of_node) < 0) - return -ENOENT; - - if (of_pci_range_parser_one(&parser, &range) == NULL) - return -ENOENT; - - range.pci_addr += range.size; - pcie->msi_doorbell = range.pci_addr + res->start + SMP8759_DOORBELL; - - for (offset = 0; offset < MSI_MAX / 8; offset += 4) - writel_relaxed(0, pcie->base + SMP8759_ENABLE + offset); - - virq = platform_get_irq(pdev, 1); - if (virq < 0) - return virq; - - irq_dom = irq_domain_create_linear(fwnode, MSI_MAX, &dom_ops, pcie); - if (!irq_dom) { - dev_err(dev, "Failed to create IRQ domain\n"); - return -ENOMEM; - } - - msi_dom = pci_msi_create_irq_domain(fwnode, &msi_dom_info, irq_dom); - if (!msi_dom) { - dev_err(dev, "Failed to create MSI domain\n"); - irq_domain_remove(irq_dom); - return -ENOMEM; - } - - pcie->dom = irq_dom; - spin_lock_init(&pcie->used_msi_lock); - irq_set_chained_handler_and_data(virq, tango_msi_isr, pcie); - - return pci_host_common_probe(pdev); -} - -static const struct of_device_id tango_pcie_ids[] = { - { - .compatible = "sigma,smp8759-pcie", - .data = &smp8759_ecam_ops, - }, - { }, -}; - -static struct platform_driver tango_pcie_driver = { - .probe = tango_pcie_probe, - .driver = { - .name = KBUILD_MODNAME, - .of_match_table = tango_pcie_ids, - .suppress_bind_attrs = true, - }, -}; -builtin_platform_driver(tango_pcie_driver); - -/* - * The root complex advertises the wrong device class. - * Header Type 1 is for PCI-to-PCI bridges. - */ -static void tango_fixup_class(struct pci_dev *dev) -{ - dev->class = PCI_CLASS_BRIDGE_PCI << 8; -} -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIGMA, 0x0024, tango_fixup_class); -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIGMA, 0x0028, tango_fixup_class); - -/* - * The root complex exposes a "fake" BAR, which is used to filter - * bus-to-system accesses. Only accesses within the range defined by this - * BAR are forwarded to the host, others are ignored. - * - * By default, the DMA framework expects an identity mapping, and DRAM0 is - * mapped at 0x80000000. - */ -static void tango_fixup_bar(struct pci_dev *dev) -{ - dev->non_compliant_bars = true; - pci_write_config_dword(dev, PCI_BASE_ADDRESS_0, 0x80000000); -} -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIGMA, 0x0024, tango_fixup_bar); -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIGMA, 0x0028, tango_fixup_bar); diff --git a/drivers/pci/controller/pcie-xilinx-cpm.c b/drivers/pci/controller/pcie-xilinx-cpm.c index f92e0152e65e..67937facd90c 100644 --- a/drivers/pci/controller/pcie-xilinx-cpm.c +++ b/drivers/pci/controller/pcie-xilinx-cpm.c @@ -404,6 +404,7 @@ static int xilinx_cpm_pcie_init_irq_domain(struct xilinx_cpm_pcie_port *port) return 0; out: xilinx_cpm_free_irq_domains(port); + of_node_put(pcie_intc_node); dev_err(dev, "Failed to allocate IRQ domains\n"); return -ENOMEM; diff --git a/drivers/pci/endpoint/functions/Kconfig b/drivers/pci/endpoint/functions/Kconfig index 8820d0f7ec77..5f1242ca2f4e 100644 --- a/drivers/pci/endpoint/functions/Kconfig +++ b/drivers/pci/endpoint/functions/Kconfig @@ -12,3 +12,16 @@ config PCI_EPF_TEST for PCI Endpoint. If in doubt, say "N" to disable Endpoint test driver. + +config PCI_EPF_NTB + tristate "PCI Endpoint NTB driver" + depends on PCI_ENDPOINT + select CONFIGFS_FS + help + Select this configuration option to enable the Non-Transparent + Bridge (NTB) driver for PCI Endpoint. NTB driver implements NTB + controller functionality using multiple PCIe endpoint instances. + It can support NTB endpoint function devices created using + device tree. + + If in doubt, say "N" to disable Endpoint NTB driver. diff --git a/drivers/pci/endpoint/functions/Makefile b/drivers/pci/endpoint/functions/Makefile index d6fafff080e2..96ab932a537a 100644 --- a/drivers/pci/endpoint/functions/Makefile +++ b/drivers/pci/endpoint/functions/Makefile @@ -4,3 +4,4 @@ # obj-$(CONFIG_PCI_EPF_TEST) += pci-epf-test.o +obj-$(CONFIG_PCI_EPF_NTB) += pci-epf-ntb.o diff --git a/drivers/pci/endpoint/functions/pci-epf-ntb.c b/drivers/pci/endpoint/functions/pci-epf-ntb.c new file mode 100644 index 000000000000..338148cf56f5 --- /dev/null +++ b/drivers/pci/endpoint/functions/pci-epf-ntb.c @@ -0,0 +1,2128 @@ +// SPDX-License-Identifier: GPL-2.0 +/** + * Endpoint Function Driver to implement Non-Transparent Bridge functionality + * + * Copyright (C) 2020 Texas Instruments + * Author: Kishon Vijay Abraham I <kishon@ti.com> + */ + +/* + * The PCI NTB function driver configures the SoC with multiple PCIe Endpoint + * (EP) controller instances (see diagram below) in such a way that + * transactions from one EP controller are routed to the other EP controller. + * Once PCI NTB function driver configures the SoC with multiple EP instances, + * HOST1 and HOST2 can communicate with each other using SoC as a bridge. + * + * +-------------+ +-------------+ + * | | | | + * | HOST1 | | HOST2 | + * | | | | + * +------^------+ +------^------+ + * | | + * | | + * +---------|-------------------------------------------------|---------+ + * | +------v------+ +------v------+ | + * | | | | | | + * | | EP | | EP | | + * | | CONTROLLER1 | | CONTROLLER2 | | + * | | <-----------------------------------> | | + * | | | | | | + * | | | | | | + * | | | SoC With Multiple EP Instances | | | + * | | | (Configured using NTB Function) | | | + * | +-------------+ +-------------+ | + * +---------------------------------------------------------------------+ + */ + +#include <linux/delay.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/slab.h> + +#include <linux/pci-epc.h> +#include <linux/pci-epf.h> + +static struct workqueue_struct *kpcintb_workqueue; + +#define COMMAND_CONFIGURE_DOORBELL 1 +#define COMMAND_TEARDOWN_DOORBELL 2 +#define COMMAND_CONFIGURE_MW 3 +#define COMMAND_TEARDOWN_MW 4 +#define COMMAND_LINK_UP 5 +#define COMMAND_LINK_DOWN 6 + +#define COMMAND_STATUS_OK 1 +#define COMMAND_STATUS_ERROR 2 + +#define LINK_STATUS_UP BIT(0) + +#define SPAD_COUNT 64 +#define DB_COUNT 4 +#define NTB_MW_OFFSET 2 +#define DB_COUNT_MASK GENMASK(15, 0) +#define MSIX_ENABLE BIT(16) +#define MAX_DB_COUNT 32 +#define MAX_MW 4 + +enum epf_ntb_bar { + BAR_CONFIG, + BAR_PEER_SPAD, + BAR_DB_MW1, + BAR_MW2, + BAR_MW3, + BAR_MW4, +}; + +struct epf_ntb { + u32 num_mws; + u32 db_count; + u32 spad_count; + struct pci_epf *epf; + u64 mws_size[MAX_MW]; + struct config_group group; + struct epf_ntb_epc *epc[2]; +}; + +#define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group) + +struct epf_ntb_epc { + u8 func_no; + bool linkup; + bool is_msix; + int msix_bar; + u32 spad_size; + struct pci_epc *epc; + struct epf_ntb *epf_ntb; + void __iomem *mw_addr[6]; + size_t msix_table_offset; + struct epf_ntb_ctrl *reg; + struct pci_epf_bar *epf_bar; + enum pci_barno epf_ntb_bar[6]; + struct delayed_work cmd_handler; + enum pci_epc_interface_type type; + const struct pci_epc_features *epc_features; +}; + +struct epf_ntb_ctrl { + u32 command; + u32 argument; + u16 command_status; + u16 link_status; + u32 topology; + u64 addr; + u64 size; + u32 num_mws; + u32 mw1_offset; + u32 spad_offset; + u32 spad_count; + u32 db_entry_size; + u32 db_data[MAX_DB_COUNT]; + u32 db_offset[MAX_DB_COUNT]; +} __packed; + +static struct pci_epf_header epf_ntb_header = { + .vendorid = PCI_ANY_ID, + .deviceid = PCI_ANY_ID, + .baseclass_code = PCI_BASE_CLASS_MEMORY, + .interrupt_pin = PCI_INTERRUPT_INTA, +}; + +/** + * epf_ntb_link_up() - Raise link_up interrupt to both the hosts + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @link_up: true or false indicating Link is UP or Down + * + * Once NTB function in HOST1 and the NTB function in HOST2 invoke + * ntb_link_enable(), this NTB function driver will trigger a link event to + * the NTB client in both the hosts. + */ +static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up) +{ + enum pci_epc_interface_type type; + enum pci_epc_irq_type irq_type; + struct epf_ntb_epc *ntb_epc; + struct epf_ntb_ctrl *ctrl; + struct pci_epc *epc; + bool is_msix; + u8 func_no; + int ret; + + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + func_no = ntb_epc->func_no; + is_msix = ntb_epc->is_msix; + ctrl = ntb_epc->reg; + if (link_up) + ctrl->link_status |= LINK_STATUS_UP; + else + ctrl->link_status &= ~LINK_STATUS_UP; + irq_type = is_msix ? PCI_EPC_IRQ_MSIX : PCI_EPC_IRQ_MSI; + ret = pci_epc_raise_irq(epc, func_no, irq_type, 1); + if (ret) { + dev_err(&epc->dev, + "%s intf: Failed to raise Link Up IRQ\n", + pci_epc_interface_string(type)); + return ret; + } + } + + return 0; +} + +/** + * epf_ntb_configure_mw() - Configure the Outbound Address Space for one host + * to access the memory window of other host + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * @mw: Index of the memory window (either 0, 1, 2 or 3) + * + * +-----------------+ +---->+----------------+-----------+-----------------+ + * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | + * +-----------------+ | +----------------+ +-----------------+ + * | BAR1 | | | Doorbell 2 +---------+ | | + * +-----------------+----+ +----------------+ | | | + * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ + * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | + * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + * +-----------------+ | |----------------+ | | | | + * | BAR4 | | | | | | +-----------------+ + * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| + * | BAR5 | | | | | | +-----------------+ + * +-----------------+ +---->-----------------+ | | | | + * EP CONTROLLER 1 | | | | +-----------------+ + * | | | +---->+ MSI|X ADDRESS 4 | + * +----------------+ | +-----------------+ + * (A) EP CONTROLLER 2 | | | + * (OB SPACE) | | | + * +-------> MW1 | + * | | + * | | + * (B) +-----------------+ + * | | + * | | + * | | + * | | + * | | + * +-----------------+ + * PCI Address Space + * (Managed by HOST2) + * + * This function performs stage (B) in the above diagram (see MW1) i.e., map OB + * address space of memory window to PCI address space. + * + * This operation requires 3 parameters + * 1) Address in the outbound address space + * 2) Address in the PCI Address space + * 3) Size of the address region to be mapped + * + * The address in the outbound address space (for MW1, MW2, MW3 and MW4) is + * stored in epf_bar corresponding to BAR_DB_MW1 for MW1 and BAR_MW2, BAR_MW3 + * BAR_MW4 for rest of the BARs of epf_ntb_epc that is connected to HOST1. This + * is populated in epf_ntb_alloc_peer_mem() in this driver. + * + * The address and size of the PCI address region that has to be mapped would + * be provided by HOST2 in ctrl->addr and ctrl->size of epf_ntb_epc that is + * connected to HOST2. + * + * Please note Memory window1 (MW1) and Doorbell registers together will be + * mapped to a single BAR (BAR2) above for 32-bit BARs. The exact BAR that's + * used for Memory window (MW) can be obtained from epf_ntb_bar[BAR_DB_MW1], + * epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2]. + */ +static int epf_ntb_configure_mw(struct epf_ntb *ntb, + enum pci_epc_interface_type type, u32 mw) +{ + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + struct pci_epf_bar *peer_epf_bar; + enum pci_barno peer_barno; + struct epf_ntb_ctrl *ctrl; + phys_addr_t phys_addr; + struct pci_epc *epc; + u64 addr, size; + int ret = 0; + u8 func_no; + + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + + peer_ntb_epc = ntb->epc[!type]; + peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET]; + peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; + + phys_addr = peer_epf_bar->phys_addr; + ctrl = ntb_epc->reg; + addr = ctrl->addr; + size = ctrl->size; + if (mw + NTB_MW_OFFSET == BAR_DB_MW1) + phys_addr += ctrl->mw1_offset; + + if (size > ntb->mws_size[mw]) { + dev_err(&epc->dev, + "%s intf: MW: %d Req Sz:%llxx > Supported Sz:%llx\n", + pci_epc_interface_string(type), mw, size, + ntb->mws_size[mw]); + ret = -EINVAL; + goto err_invalid_size; + } + + func_no = ntb_epc->func_no; + + ret = pci_epc_map_addr(epc, func_no, phys_addr, addr, size); + if (ret) + dev_err(&epc->dev, + "%s intf: Failed to map memory window %d address\n", + pci_epc_interface_string(type), mw); + +err_invalid_size: + + return ret; +} + +/** + * epf_ntb_teardown_mw() - Teardown the configured OB ATU + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * @mw: Index of the memory window (either 0, 1, 2 or 3) + * + * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using + * pci_epc_unmap_addr() + */ +static void epf_ntb_teardown_mw(struct epf_ntb *ntb, + enum pci_epc_interface_type type, u32 mw) +{ + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + struct pci_epf_bar *peer_epf_bar; + enum pci_barno peer_barno; + struct epf_ntb_ctrl *ctrl; + phys_addr_t phys_addr; + struct pci_epc *epc; + u8 func_no; + + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + + peer_ntb_epc = ntb->epc[!type]; + peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET]; + peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; + + phys_addr = peer_epf_bar->phys_addr; + ctrl = ntb_epc->reg; + if (mw + NTB_MW_OFFSET == BAR_DB_MW1) + phys_addr += ctrl->mw1_offset; + func_no = ntb_epc->func_no; + + pci_epc_unmap_addr(epc, func_no, phys_addr); +} + +/** + * epf_ntb_configure_msi() - Map OB address space to MSI address + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * @db_count: Number of doorbell interrupts to map + * + *+-----------------+ +----->+----------------+-----------+-----------------+ + *| BAR0 | | | Doorbell 1 +---+-------> MSI ADDRESS | + *+-----------------+ | +----------------+ | +-----------------+ + *| BAR1 | | | Doorbell 2 +---+ | | + *+-----------------+----+ +----------------+ | | | + *| BAR2 | | Doorbell 3 +---+ | | + *+-----------------+----+ +----------------+ | | | + *| BAR3 | | | Doorbell 4 +---+ | | + *+-----------------+ | |----------------+ | | + *| BAR4 | | | | | | + *+-----------------+ | | MW1 | | | + *| BAR5 | | | | | | + *+-----------------+ +----->-----------------+ | | + * EP CONTROLLER 1 | | | | + * | | | | + * +----------------+ +-----------------+ + * (A) EP CONTROLLER 2 | | + * (OB SPACE) | | + * | MW1 | + * | | + * | | + * (B) +-----------------+ + * | | + * | | + * | | + * | | + * | | + * +-----------------+ + * PCI Address Space + * (Managed by HOST2) + * + * + * This function performs stage (B) in the above diagram (see Doorbell 1, + * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to + * doorbell to MSI address in PCI address space. + * + * This operation requires 3 parameters + * 1) Address reserved for doorbell in the outbound address space + * 2) MSI-X address in the PCIe Address space + * 3) Number of MSI-X interrupts that has to be configured + * + * The address in the outbound address space (for the Doorbell) is stored in + * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to + * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along + * with address for MW1. + * + * pci_epc_map_msi_irq() takes the MSI address from MSI capability register + * and maps the OB address (obtained in epf_ntb_alloc_peer_mem()) to the MSI + * address. + * + * epf_ntb_configure_msi() also stores the MSI data to raise each interrupt + * in db_data of the peer's control region. This helps the peer to raise + * doorbell of the other host by writing db_data to the BAR corresponding to + * BAR_DB_MW1. + */ +static int epf_ntb_configure_msi(struct epf_ntb *ntb, + enum pci_epc_interface_type type, u16 db_count) +{ + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + u32 db_entry_size, db_data, db_offset; + struct pci_epf_bar *peer_epf_bar; + struct epf_ntb_ctrl *peer_ctrl; + enum pci_barno peer_barno; + phys_addr_t phys_addr; + struct pci_epc *epc; + u8 func_no; + int ret, i; + + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + + peer_ntb_epc = ntb->epc[!type]; + peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; + peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; + peer_ctrl = peer_ntb_epc->reg; + db_entry_size = peer_ctrl->db_entry_size; + + phys_addr = peer_epf_bar->phys_addr; + func_no = ntb_epc->func_no; + + ret = pci_epc_map_msi_irq(epc, func_no, phys_addr, db_count, + db_entry_size, &db_data, &db_offset); + if (ret) { + dev_err(&epc->dev, "%s intf: Failed to map MSI IRQ\n", + pci_epc_interface_string(type)); + return ret; + } + + for (i = 0; i < db_count; i++) { + peer_ctrl->db_data[i] = db_data | i; + peer_ctrl->db_offset[i] = db_offset; + } + + return 0; +} + +/** + * epf_ntb_configure_msix() - Map OB address space to MSI-X address + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * @db_count: Number of doorbell interrupts to map + * + *+-----------------+ +----->+----------------+-----------+-----------------+ + *| BAR0 | | | Doorbell 1 +-----------> MSI-X ADDRESS 1 | + *+-----------------+ | +----------------+ +-----------------+ + *| BAR1 | | | Doorbell 2 +---------+ | | + *+-----------------+----+ +----------------+ | | | + *| BAR2 | | Doorbell 3 +-------+ | +-----------------+ + *+-----------------+----+ +----------------+ | +-> MSI-X ADDRESS 2 | + *| BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + *+-----------------+ | |----------------+ | | | | + *| BAR4 | | | | | | +-----------------+ + *+-----------------+ | | MW1 + | +-->+ MSI-X ADDRESS 3|| + *| BAR5 | | | | | +-----------------+ + *+-----------------+ +----->-----------------+ | | | + * EP CONTROLLER 1 | | | +-----------------+ + * | | +---->+ MSI-X ADDRESS 4 | + * +----------------+ +-----------------+ + * (A) EP CONTROLLER 2 | | + * (OB SPACE) | | + * | MW1 | + * | | + * | | + * (B) +-----------------+ + * | | + * | | + * | | + * | | + * | | + * +-----------------+ + * PCI Address Space + * (Managed by HOST2) + * + * This function performs stage (B) in the above diagram (see Doorbell 1, + * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to + * doorbell to MSI-X address in PCI address space. + * + * This operation requires 3 parameters + * 1) Address reserved for doorbell in the outbound address space + * 2) MSI-X address in the PCIe Address space + * 3) Number of MSI-X interrupts that has to be configured + * + * The address in the outbound address space (for the Doorbell) is stored in + * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to + * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along + * with address for MW1. + * + * The MSI-X address is in the MSI-X table of EP CONTROLLER 2 and + * the count of doorbell is in ctrl->argument of epf_ntb_epc that is connected + * to HOST2. MSI-X table is stored memory mapped to ntb_epc->msix_bar and the + * offset is in ntb_epc->msix_table_offset. From this epf_ntb_configure_msix() + * gets the MSI-X address and data. + * + * epf_ntb_configure_msix() also stores the MSI-X data to raise each interrupt + * in db_data of the peer's control region. This helps the peer to raise + * doorbell of the other host by writing db_data to the BAR corresponding to + * BAR_DB_MW1. + */ +static int epf_ntb_configure_msix(struct epf_ntb *ntb, + enum pci_epc_interface_type type, + u16 db_count) +{ + const struct pci_epc_features *epc_features; + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + struct pci_epf_bar *peer_epf_bar, *epf_bar; + struct pci_epf_msix_tbl *msix_tbl; + struct epf_ntb_ctrl *peer_ctrl; + u32 db_entry_size, msg_data; + enum pci_barno peer_barno; + phys_addr_t phys_addr; + struct pci_epc *epc; + size_t align; + u64 msg_addr; + u8 func_no; + int ret, i; + + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + + epf_bar = &ntb_epc->epf_bar[ntb_epc->msix_bar]; + msix_tbl = epf_bar->addr + ntb_epc->msix_table_offset; + + peer_ntb_epc = ntb->epc[!type]; + peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; + peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; + phys_addr = peer_epf_bar->phys_addr; + peer_ctrl = peer_ntb_epc->reg; + epc_features = ntb_epc->epc_features; + align = epc_features->align; + + func_no = ntb_epc->func_no; + db_entry_size = peer_ctrl->db_entry_size; + + for (i = 0; i < db_count; i++) { + msg_addr = ALIGN_DOWN(msix_tbl[i].msg_addr, align); + msg_data = msix_tbl[i].msg_data; + ret = pci_epc_map_addr(epc, func_no, phys_addr, msg_addr, + db_entry_size); + if (ret) { + dev_err(&epc->dev, + "%s intf: Failed to configure MSI-X IRQ\n", + pci_epc_interface_string(type)); + return ret; + } + phys_addr = phys_addr + db_entry_size; + peer_ctrl->db_data[i] = msg_data; + peer_ctrl->db_offset[i] = msix_tbl[i].msg_addr & (align - 1); + } + ntb_epc->is_msix = true; + + return 0; +} + +/** + * epf_ntb_configure_db() - Configure the Outbound Address Space for one host + * to ring the doorbell of other host + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * @db_count: Count of the number of doorbells that has to be configured + * @msix: Indicates whether MSI-X or MSI should be used + * + * Invokes epf_ntb_configure_msix() or epf_ntb_configure_msi() required for + * one HOST to ring the doorbell of other HOST. + */ +static int epf_ntb_configure_db(struct epf_ntb *ntb, + enum pci_epc_interface_type type, + u16 db_count, bool msix) +{ + struct epf_ntb_epc *ntb_epc; + struct pci_epc *epc; + int ret; + + if (db_count > MAX_DB_COUNT) + return -EINVAL; + + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + + if (msix) + ret = epf_ntb_configure_msix(ntb, type, db_count); + else + ret = epf_ntb_configure_msi(ntb, type, db_count); + + if (ret) + dev_err(&epc->dev, "%s intf: Failed to configure DB\n", + pci_epc_interface_string(type)); + + return ret; +} + +/** + * epf_ntb_teardown_db() - Unmap address in OB address space to MSI/MSI-X + * address + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Invoke pci_epc_unmap_addr() to unmap OB address to MSI/MSI-X address. + */ +static void +epf_ntb_teardown_db(struct epf_ntb *ntb, enum pci_epc_interface_type type) +{ + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + struct pci_epf_bar *peer_epf_bar; + enum pci_barno peer_barno; + phys_addr_t phys_addr; + struct pci_epc *epc; + u8 func_no; + + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + + peer_ntb_epc = ntb->epc[!type]; + peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; + peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; + phys_addr = peer_epf_bar->phys_addr; + func_no = ntb_epc->func_no; + + pci_epc_unmap_addr(epc, func_no, phys_addr); +} + +/** + * epf_ntb_cmd_handler() - Handle commands provided by the NTB Host + * @work: work_struct for the two epf_ntb_epc (PRIMARY and SECONDARY) + * + * Workqueue function that gets invoked for the two epf_ntb_epc + * periodically (once every 5ms) to see if it has received any commands + * from NTB host. The host can send commands to configure doorbell or + * configure memory window or to update link status. + */ +static void epf_ntb_cmd_handler(struct work_struct *work) +{ + enum pci_epc_interface_type type; + struct epf_ntb_epc *ntb_epc; + struct epf_ntb_ctrl *ctrl; + u32 command, argument; + struct epf_ntb *ntb; + struct device *dev; + u16 db_count; + bool is_msix; + int ret; + + ntb_epc = container_of(work, struct epf_ntb_epc, cmd_handler.work); + ctrl = ntb_epc->reg; + command = ctrl->command; + if (!command) + goto reset_handler; + argument = ctrl->argument; + + ctrl->command = 0; + ctrl->argument = 0; + + ctrl = ntb_epc->reg; + type = ntb_epc->type; + ntb = ntb_epc->epf_ntb; + dev = &ntb->epf->dev; + + switch (command) { + case COMMAND_CONFIGURE_DOORBELL: + db_count = argument & DB_COUNT_MASK; + is_msix = argument & MSIX_ENABLE; + ret = epf_ntb_configure_db(ntb, type, db_count, is_msix); + if (ret < 0) + ctrl->command_status = COMMAND_STATUS_ERROR; + else + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_TEARDOWN_DOORBELL: + epf_ntb_teardown_db(ntb, type); + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_CONFIGURE_MW: + ret = epf_ntb_configure_mw(ntb, type, argument); + if (ret < 0) + ctrl->command_status = COMMAND_STATUS_ERROR; + else + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_TEARDOWN_MW: + epf_ntb_teardown_mw(ntb, type, argument); + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_LINK_UP: + ntb_epc->linkup = true; + if (ntb->epc[PRIMARY_INTERFACE]->linkup && + ntb->epc[SECONDARY_INTERFACE]->linkup) { + ret = epf_ntb_link_up(ntb, true); + if (ret < 0) + ctrl->command_status = COMMAND_STATUS_ERROR; + else + ctrl->command_status = COMMAND_STATUS_OK; + goto reset_handler; + } + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_LINK_DOWN: + ntb_epc->linkup = false; + ret = epf_ntb_link_up(ntb, false); + if (ret < 0) + ctrl->command_status = COMMAND_STATUS_ERROR; + else + ctrl->command_status = COMMAND_STATUS_OK; + break; + default: + dev_err(dev, "%s intf UNKNOWN command: %d\n", + pci_epc_interface_string(type), command); + break; + } + +reset_handler: + queue_delayed_work(kpcintb_workqueue, &ntb_epc->cmd_handler, + msecs_to_jiffies(5)); +} + +/** + * epf_ntb_peer_spad_bar_clear() - Clear Peer Scratchpad BAR + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + *+-----------------+------->+------------------+ +-----------------+ + *| BAR0 | | CONFIG REGION | | BAR0 | + *+-----------------+----+ +------------------+<-------+-----------------+ + *| BAR1 | | |SCRATCHPAD REGION | | BAR1 | + *+-----------------+ +-->+------------------+<-------+-----------------+ + *| BAR2 | Local Memory | BAR2 | + *+-----------------+ +-----------------+ + *| BAR3 | | BAR3 | + *+-----------------+ +-----------------+ + *| BAR4 | | BAR4 | + *+-----------------+ +-----------------+ + *| BAR5 | | BAR5 | + *+-----------------+ +-----------------+ + * EP CONTROLLER 1 EP CONTROLLER 2 + * + * Clear BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad + * region. While BAR1 is the default peer scratchpad BAR, an NTB could have + * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs). + * This function can get the exact BAR used for peer scratchpad from + * epf_ntb_bar[BAR_PEER_SPAD]. + * + * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function + * gets the address of peer scratchpad from + * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]. + */ +static void epf_ntb_peer_spad_bar_clear(struct epf_ntb_epc *ntb_epc) +{ + struct pci_epf_bar *epf_bar; + enum pci_barno barno; + struct pci_epc *epc; + u8 func_no; + + epc = ntb_epc->epc; + func_no = ntb_epc->func_no; + barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; + epf_bar = &ntb_epc->epf_bar[barno]; + pci_epc_clear_bar(epc, func_no, epf_bar); +} + +/** + * epf_ntb_peer_spad_bar_set() - Set peer scratchpad BAR + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + *+-----------------+------->+------------------+ +-----------------+ + *| BAR0 | | CONFIG REGION | | BAR0 | + *+-----------------+----+ +------------------+<-------+-----------------+ + *| BAR1 | | |SCRATCHPAD REGION | | BAR1 | + *+-----------------+ +-->+------------------+<-------+-----------------+ + *| BAR2 | Local Memory | BAR2 | + *+-----------------+ +-----------------+ + *| BAR3 | | BAR3 | + *+-----------------+ +-----------------+ + *| BAR4 | | BAR4 | + *+-----------------+ +-----------------+ + *| BAR5 | | BAR5 | + *+-----------------+ +-----------------+ + * EP CONTROLLER 1 EP CONTROLLER 2 + * + * Set BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad + * region. While BAR1 is the default peer scratchpad BAR, an NTB could have + * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs). + * This function can get the exact BAR used for peer scratchpad from + * epf_ntb_bar[BAR_PEER_SPAD]. + * + * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function + * gets the address of peer scratchpad from + * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]. + */ +static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + struct pci_epf_bar *peer_epf_bar, *epf_bar; + enum pci_barno peer_barno, barno; + u32 peer_spad_offset; + struct pci_epc *epc; + struct device *dev; + u8 func_no; + int ret; + + dev = &ntb->epf->dev; + + peer_ntb_epc = ntb->epc[!type]; + peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]; + peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; + + ntb_epc = ntb->epc[type]; + barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; + epf_bar = &ntb_epc->epf_bar[barno]; + func_no = ntb_epc->func_no; + epc = ntb_epc->epc; + + peer_spad_offset = peer_ntb_epc->reg->spad_offset; + epf_bar->phys_addr = peer_epf_bar->phys_addr + peer_spad_offset; + epf_bar->size = peer_ntb_epc->spad_size; + epf_bar->barno = barno; + epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; + + ret = pci_epc_set_bar(epc, func_no, epf_bar); + if (ret) { + dev_err(dev, "%s intf: peer SPAD BAR set failed\n", + pci_epc_interface_string(type)); + return ret; + } + + return 0; +} + +/** + * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * +-----------------+------->+------------------+ +-----------------+ + * | BAR0 | | CONFIG REGION | | BAR0 | + * +-----------------+----+ +------------------+<-------+-----------------+ + * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | + * +-----------------+ +-->+------------------+<-------+-----------------+ + * | BAR2 | Local Memory | BAR2 | + * +-----------------+ +-----------------+ + * | BAR3 | | BAR3 | + * +-----------------+ +-----------------+ + * | BAR4 | | BAR4 | + * +-----------------+ +-----------------+ + * | BAR5 | | BAR5 | + * +-----------------+ +-----------------+ + * EP CONTROLLER 1 EP CONTROLLER 2 + * + * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and + * self scratchpad region (removes inbound ATU configuration). While BAR0 is + * the default self scratchpad BAR, an NTB could have other BARs for self + * scratchpad (because of reserved BARs). This function can get the exact BAR + * used for self scratchpad from epf_ntb_bar[BAR_CONFIG]. + * + * Please note the self scratchpad region and config region is combined to + * a single region and mapped using the same BAR. Also note HOST2's peer + * scratchpad is HOST1's self scratchpad. + */ +static void epf_ntb_config_sspad_bar_clear(struct epf_ntb_epc *ntb_epc) +{ + struct pci_epf_bar *epf_bar; + enum pci_barno barno; + struct pci_epc *epc; + u8 func_no; + + epc = ntb_epc->epc; + func_no = ntb_epc->func_no; + barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; + epf_bar = &ntb_epc->epf_bar[barno]; + pci_epc_clear_bar(epc, func_no, epf_bar); +} + +/** + * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * +-----------------+------->+------------------+ +-----------------+ + * | BAR0 | | CONFIG REGION | | BAR0 | + * +-----------------+----+ +------------------+<-------+-----------------+ + * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | + * +-----------------+ +-->+------------------+<-------+-----------------+ + * | BAR2 | Local Memory | BAR2 | + * +-----------------+ +-----------------+ + * | BAR3 | | BAR3 | + * +-----------------+ +-----------------+ + * | BAR4 | | BAR4 | + * +-----------------+ +-----------------+ + * | BAR5 | | BAR5 | + * +-----------------+ +-----------------+ + * EP CONTROLLER 1 EP CONTROLLER 2 + * + * Map BAR0 of EP CONTROLLER 1 which contains the HOST1's config and + * self scratchpad region. While BAR0 is the default self scratchpad BAR, an + * NTB could have other BARs for self scratchpad (because of reserved BARs). + * This function can get the exact BAR used for self scratchpad from + * epf_ntb_bar[BAR_CONFIG]. + * + * Please note the self scratchpad region and config region is combined to + * a single region and mapped using the same BAR. Also note HOST2's peer + * scratchpad is HOST1's self scratchpad. + */ +static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc) +{ + struct pci_epf_bar *epf_bar; + enum pci_barno barno; + struct epf_ntb *ntb; + struct pci_epc *epc; + struct device *dev; + u8 func_no; + int ret; + + ntb = ntb_epc->epf_ntb; + dev = &ntb->epf->dev; + + epc = ntb_epc->epc; + func_no = ntb_epc->func_no; + barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; + epf_bar = &ntb_epc->epf_bar[barno]; + + ret = pci_epc_set_bar(epc, func_no, epf_bar); + if (ret) { + dev_err(dev, "%s inft: Config/Status/SPAD BAR set failed\n", + pci_epc_interface_string(ntb_epc->type)); + return ret; + } + + return 0; +} + +/** + * epf_ntb_config_spad_bar_free() - Free the physical memory associated with + * config + scratchpad region + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * +-----------------+------->+------------------+ +-----------------+ + * | BAR0 | | CONFIG REGION | | BAR0 | + * +-----------------+----+ +------------------+<-------+-----------------+ + * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | + * +-----------------+ +-->+------------------+<-------+-----------------+ + * | BAR2 | Local Memory | BAR2 | + * +-----------------+ +-----------------+ + * | BAR3 | | BAR3 | + * +-----------------+ +-----------------+ + * | BAR4 | | BAR4 | + * +-----------------+ +-----------------+ + * | BAR5 | | BAR5 | + * +-----------------+ +-----------------+ + * EP CONTROLLER 1 EP CONTROLLER 2 + * + * Free the Local Memory mentioned in the above diagram. After invoking this + * function, any of config + self scratchpad region of HOST1 or peer scratchpad + * region of HOST2 should not be accessed. + */ +static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb) +{ + enum pci_epc_interface_type type; + struct epf_ntb_epc *ntb_epc; + enum pci_barno barno; + struct pci_epf *epf; + + epf = ntb->epf; + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { + ntb_epc = ntb->epc[type]; + barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; + if (ntb_epc->reg) + pci_epf_free_space(epf, ntb_epc->reg, barno, type); + } +} + +/** + * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad + * region + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * +-----------------+------->+------------------+ +-----------------+ + * | BAR0 | | CONFIG REGION | | BAR0 | + * +-----------------+----+ +------------------+<-------+-----------------+ + * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | + * +-----------------+ +-->+------------------+<-------+-----------------+ + * | BAR2 | Local Memory | BAR2 | + * +-----------------+ +-----------------+ + * | BAR3 | | BAR3 | + * +-----------------+ +-----------------+ + * | BAR4 | | BAR4 | + * +-----------------+ +-----------------+ + * | BAR5 | | BAR5 | + * +-----------------+ +-----------------+ + * EP CONTROLLER 1 EP CONTROLLER 2 + * + * Allocate the Local Memory mentioned in the above diagram. The size of + * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION + * is obtained from "spad-count" configfs entry. + * + * The size of both config region and scratchpad region has to be aligned, + * since the scratchpad region will also be mapped as PEER SCRATCHPAD of + * other host using a separate BAR. + */ +static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + const struct pci_epc_features *peer_epc_features, *epc_features; + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + size_t msix_table_size, pba_size, align; + enum pci_barno peer_barno, barno; + struct epf_ntb_ctrl *ctrl; + u32 spad_size, ctrl_size; + u64 size, peer_size; + struct pci_epf *epf; + struct device *dev; + bool msix_capable; + u32 spad_count; + void *base; + + epf = ntb->epf; + dev = &epf->dev; + ntb_epc = ntb->epc[type]; + + epc_features = ntb_epc->epc_features; + barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; + size = epc_features->bar_fixed_size[barno]; + align = epc_features->align; + + peer_ntb_epc = ntb->epc[!type]; + peer_epc_features = peer_ntb_epc->epc_features; + peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; + peer_size = peer_epc_features->bar_fixed_size[peer_barno]; + + /* Check if epc_features is populated incorrectly */ + if ((!IS_ALIGNED(size, align))) + return -EINVAL; + + spad_count = ntb->spad_count; + + ctrl_size = sizeof(struct epf_ntb_ctrl); + spad_size = spad_count * 4; + + msix_capable = epc_features->msix_capable; + if (msix_capable) { + msix_table_size = PCI_MSIX_ENTRY_SIZE * ntb->db_count; + ctrl_size = ALIGN(ctrl_size, 8); + ntb_epc->msix_table_offset = ctrl_size; + ntb_epc->msix_bar = barno; + /* Align to QWORD or 8 Bytes */ + pba_size = ALIGN(DIV_ROUND_UP(ntb->db_count, 8), 8); + ctrl_size = ctrl_size + msix_table_size + pba_size; + } + + if (!align) { + ctrl_size = roundup_pow_of_two(ctrl_size); + spad_size = roundup_pow_of_two(spad_size); + } else { + ctrl_size = ALIGN(ctrl_size, align); + spad_size = ALIGN(spad_size, align); + } + + if (peer_size) { + if (peer_size < spad_size) + spad_count = peer_size / 4; + spad_size = peer_size; + } + + /* + * In order to make sure SPAD offset is aligned to its size, + * expand control region size to the size of SPAD if SPAD size + * is greater than control region size. + */ + if (spad_size > ctrl_size) + ctrl_size = spad_size; + + if (!size) + size = ctrl_size + spad_size; + else if (size < ctrl_size + spad_size) + return -EINVAL; + + base = pci_epf_alloc_space(epf, size, barno, align, type); + if (!base) { + dev_err(dev, "%s intf: Config/Status/SPAD alloc region fail\n", + pci_epc_interface_string(type)); + return -ENOMEM; + } + + ntb_epc->reg = base; + + ctrl = ntb_epc->reg; + ctrl->spad_offset = ctrl_size; + ctrl->spad_count = spad_count; + ctrl->num_mws = ntb->num_mws; + ctrl->db_entry_size = align ? align : 4; + ntb_epc->spad_size = spad_size; + + return 0; +} + +/** + * epf_ntb_config_spad_bar_alloc_interface() - Allocate memory for config + + * scratchpad region for each of PRIMARY and SECONDARY interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * Wrapper for epf_ntb_config_spad_bar_alloc() which allocates memory for + * config + scratchpad region for a specific interface + */ +static int epf_ntb_config_spad_bar_alloc_interface(struct epf_ntb *ntb) +{ + enum pci_epc_interface_type type; + struct device *dev; + int ret; + + dev = &ntb->epf->dev; + + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { + ret = epf_ntb_config_spad_bar_alloc(ntb, type); + if (ret) { + dev_err(dev, "%s intf: Config/SPAD BAR alloc failed\n", + pci_epc_interface_string(type)); + return ret; + } + } + + return 0; +} + +/** + * epf_ntb_free_peer_mem() - Free memory allocated in peers outbound address + * space + * @ntb_epc: EPC associated with one of the HOST which holds peers outbound + * address regions + * + * +-----------------+ +---->+----------------+-----------+-----------------+ + * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | + * +-----------------+ | +----------------+ +-----------------+ + * | BAR1 | | | Doorbell 2 +---------+ | | + * +-----------------+----+ +----------------+ | | | + * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ + * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | + * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + * +-----------------+ | |----------------+ | | | | + * | BAR4 | | | | | | +-----------------+ + * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| + * | BAR5 | | | | | | +-----------------+ + * +-----------------+ +---->-----------------+ | | | | + * EP CONTROLLER 1 | | | | +-----------------+ + * | | | +---->+ MSI|X ADDRESS 4 | + * +----------------+ | +-----------------+ + * (A) EP CONTROLLER 2 | | | + * (OB SPACE) | | | + * +-------> MW1 | + * | | + * | | + * (B) +-----------------+ + * | | + * | | + * | | + * | | + * | | + * +-----------------+ + * PCI Address Space + * (Managed by HOST2) + * + * Free memory allocated in EP CONTROLLER 2 (OB SPACE) in the above diagram. + * It'll free Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, + * MW4). + */ +static void epf_ntb_free_peer_mem(struct epf_ntb_epc *ntb_epc) +{ + struct pci_epf_bar *epf_bar; + void __iomem *mw_addr; + phys_addr_t phys_addr; + enum epf_ntb_bar bar; + enum pci_barno barno; + struct pci_epc *epc; + size_t size; + + epc = ntb_epc->epc; + + for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) { + barno = ntb_epc->epf_ntb_bar[bar]; + mw_addr = ntb_epc->mw_addr[barno]; + epf_bar = &ntb_epc->epf_bar[barno]; + phys_addr = epf_bar->phys_addr; + size = epf_bar->size; + if (mw_addr) { + pci_epc_mem_free_addr(epc, phys_addr, mw_addr, size); + ntb_epc->mw_addr[barno] = NULL; + } + } +} + +/** + * epf_ntb_db_mw_bar_clear() - Clear doorbell and memory BAR + * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound + * address + * + * +-----------------+ +---->+----------------+-----------+-----------------+ + * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | + * +-----------------+ | +----------------+ +-----------------+ + * | BAR1 | | | Doorbell 2 +---------+ | | + * +-----------------+----+ +----------------+ | | | + * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ + * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | + * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + * +-----------------+ | |----------------+ | | | | + * | BAR4 | | | | | | +-----------------+ + * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| + * | BAR5 | | | | | | +-----------------+ + * +-----------------+ +---->-----------------+ | | | | + * EP CONTROLLER 1 | | | | +-----------------+ + * | | | +---->+ MSI|X ADDRESS 4 | + * +----------------+ | +-----------------+ + * (A) EP CONTROLLER 2 | | | + * (OB SPACE) | | | + * +-------> MW1 | + * | | + * | | + * (B) +-----------------+ + * | | + * | | + * | | + * | | + * | | + * +-----------------+ + * PCI Address Space + * (Managed by HOST2) + * + * Clear doorbell and memory BARs (remove inbound ATU configuration). In the above + * diagram it clears BAR2 TO BAR5 of EP CONTROLLER 1 (Doorbell BAR, MW1 BAR, MW2 + * BAR, MW3 BAR and MW4 BAR). + */ +static void epf_ntb_db_mw_bar_clear(struct epf_ntb_epc *ntb_epc) +{ + struct pci_epf_bar *epf_bar; + enum epf_ntb_bar bar; + enum pci_barno barno; + struct pci_epc *epc; + u8 func_no; + + epc = ntb_epc->epc; + + func_no = ntb_epc->func_no; + + for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) { + barno = ntb_epc->epf_ntb_bar[bar]; + epf_bar = &ntb_epc->epf_bar[barno]; + pci_epc_clear_bar(epc, func_no, epf_bar); + } +} + +/** + * epf_ntb_db_mw_bar_cleanup() - Clear doorbell/memory BAR and free memory + * allocated in peers outbound address space + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Wrapper for epf_ntb_db_mw_bar_clear() to clear HOST1's BAR and + * epf_ntb_free_peer_mem() which frees up HOST2 outbound memory. + */ +static void epf_ntb_db_mw_bar_cleanup(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + + ntb_epc = ntb->epc[type]; + peer_ntb_epc = ntb->epc[!type]; + + epf_ntb_db_mw_bar_clear(ntb_epc); + epf_ntb_free_peer_mem(peer_ntb_epc); +} + +/** + * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capaiblity + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Configure MSI/MSI-X capability for each interface with number of + * interrupts equal to "db_count" configfs entry. + */ +static int epf_ntb_configure_interrupt(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + const struct pci_epc_features *epc_features; + bool msix_capable, msi_capable; + struct epf_ntb_epc *ntb_epc; + struct pci_epc *epc; + struct device *dev; + u32 db_count; + u8 func_no; + int ret; + + ntb_epc = ntb->epc[type]; + dev = &ntb->epf->dev; + + epc_features = ntb_epc->epc_features; + msix_capable = epc_features->msix_capable; + msi_capable = epc_features->msi_capable; + + if (!(msix_capable || msi_capable)) { + dev_err(dev, "MSI or MSI-X is required for doorbell\n"); + return -EINVAL; + } + + func_no = ntb_epc->func_no; + + db_count = ntb->db_count; + if (db_count > MAX_DB_COUNT) { + dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT); + return -EINVAL; + } + + ntb->db_count = db_count; + epc = ntb_epc->epc; + + if (msi_capable) { + ret = pci_epc_set_msi(epc, func_no, db_count); + if (ret) { + dev_err(dev, "%s intf: MSI configuration failed\n", + pci_epc_interface_string(type)); + return ret; + } + } + + if (msix_capable) { + ret = pci_epc_set_msix(epc, func_no, db_count, + ntb_epc->msix_bar, + ntb_epc->msix_table_offset); + if (ret) { + dev_err(dev, "MSI configuration failed\n"); + return ret; + } + } + + return 0; +} + +/** + * epf_ntb_alloc_peer_mem() - Allocate memory in peer's outbound address space + * @ntb_epc: EPC associated with one of the HOST whose BAR holds peer's outbound + * address + * @bar: BAR of @ntb_epc in for which memory has to be allocated (could be + * BAR_DB_MW1, BAR_MW2, BAR_MW3, BAR_MW4) + * @peer_ntb_epc: EPC associated with HOST whose outbound address space is + * used by @ntb_epc + * @size: Size of the address region that has to be allocated in peers OB SPACE + * + * + * +-----------------+ +---->+----------------+-----------+-----------------+ + * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | + * +-----------------+ | +----------------+ +-----------------+ + * | BAR1 | | | Doorbell 2 +---------+ | | + * +-----------------+----+ +----------------+ | | | + * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ + * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | + * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + * +-----------------+ | |----------------+ | | | | + * | BAR4 | | | | | | +-----------------+ + * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| + * | BAR5 | | | | | | +-----------------+ + * +-----------------+ +---->-----------------+ | | | | + * EP CONTROLLER 1 | | | | +-----------------+ + * | | | +---->+ MSI|X ADDRESS 4 | + * +----------------+ | +-----------------+ + * (A) EP CONTROLLER 2 | | | + * (OB SPACE) | | | + * +-------> MW1 | + * | | + * | | + * (B) +-----------------+ + * | | + * | | + * | | + * | | + * | | + * +-----------------+ + * PCI Address Space + * (Managed by HOST2) + * + * Allocate memory in OB space of EP CONTROLLER 2 in the above diagram. Allocate + * for Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, MW4). + */ +static int epf_ntb_alloc_peer_mem(struct device *dev, + struct epf_ntb_epc *ntb_epc, + enum epf_ntb_bar bar, + struct epf_ntb_epc *peer_ntb_epc, + size_t size) +{ + const struct pci_epc_features *epc_features; + struct pci_epf_bar *epf_bar; + struct pci_epc *peer_epc; + phys_addr_t phys_addr; + void __iomem *mw_addr; + enum pci_barno barno; + size_t align; + + epc_features = ntb_epc->epc_features; + align = epc_features->align; + + if (size < 128) + size = 128; + + if (align) + size = ALIGN(size, align); + else + size = roundup_pow_of_two(size); + + peer_epc = peer_ntb_epc->epc; + mw_addr = pci_epc_mem_alloc_addr(peer_epc, &phys_addr, size); + if (!mw_addr) { + dev_err(dev, "%s intf: Failed to allocate OB address\n", + pci_epc_interface_string(peer_ntb_epc->type)); + return -ENOMEM; + } + + barno = ntb_epc->epf_ntb_bar[bar]; + epf_bar = &ntb_epc->epf_bar[barno]; + ntb_epc->mw_addr[barno] = mw_addr; + + epf_bar->phys_addr = phys_addr; + epf_bar->size = size; + epf_bar->barno = barno; + epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; + + return 0; +} + +/** + * epf_ntb_db_mw_bar_init() - Configure Doorbell and Memory window BARs + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Wrapper for epf_ntb_alloc_peer_mem() and pci_epc_set_bar() that allocates + * memory in OB address space of HOST2 and configures BAR of HOST1 + */ +static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + const struct pci_epc_features *epc_features; + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + struct pci_epf_bar *epf_bar; + struct epf_ntb_ctrl *ctrl; + u32 num_mws, db_count; + enum epf_ntb_bar bar; + enum pci_barno barno; + struct pci_epc *epc; + struct device *dev; + size_t align; + int ret, i; + u8 func_no; + u64 size; + + ntb_epc = ntb->epc[type]; + peer_ntb_epc = ntb->epc[!type]; + + dev = &ntb->epf->dev; + epc_features = ntb_epc->epc_features; + align = epc_features->align; + func_no = ntb_epc->func_no; + epc = ntb_epc->epc; + num_mws = ntb->num_mws; + db_count = ntb->db_count; + + for (bar = BAR_DB_MW1, i = 0; i < num_mws; bar++, i++) { + if (bar == BAR_DB_MW1) { + align = align ? align : 4; + size = db_count * align; + size = ALIGN(size, ntb->mws_size[i]); + ctrl = ntb_epc->reg; + ctrl->mw1_offset = size; + size += ntb->mws_size[i]; + } else { + size = ntb->mws_size[i]; + } + + ret = epf_ntb_alloc_peer_mem(dev, ntb_epc, bar, + peer_ntb_epc, size); + if (ret) { + dev_err(dev, "%s intf: DoorBell mem alloc failed\n", + pci_epc_interface_string(type)); + goto err_alloc_peer_mem; + } + + barno = ntb_epc->epf_ntb_bar[bar]; + epf_bar = &ntb_epc->epf_bar[barno]; + + ret = pci_epc_set_bar(epc, func_no, epf_bar); + if (ret) { + dev_err(dev, "%s intf: DoorBell BAR set failed\n", + pci_epc_interface_string(type)); + goto err_alloc_peer_mem; + } + } + + return 0; + +err_alloc_peer_mem: + epf_ntb_db_mw_bar_cleanup(ntb, type); + + return ret; +} + +/** + * epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Unbind NTB function device from EPC and relinquish reference to pci_epc + * for each of the interface. + */ +static void epf_ntb_epc_destroy_interface(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + struct epf_ntb_epc *ntb_epc; + struct pci_epc *epc; + struct pci_epf *epf; + + if (type < 0) + return; + + epf = ntb->epf; + ntb_epc = ntb->epc[type]; + if (!ntb_epc) + return; + epc = ntb_epc->epc; + pci_epc_remove_epf(epc, epf, type); + pci_epc_put(epc); +} + +/** + * epf_ntb_epc_destroy() - Cleanup NTB EPC interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces + */ +static void epf_ntb_epc_destroy(struct epf_ntb *ntb) +{ + enum pci_epc_interface_type type; + + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) + epf_ntb_epc_destroy_interface(ntb, type); +} + +/** + * epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @epc: struct pci_epc to which a particular NTB interface should be associated + * @type: PRIMARY interface or SECONDARY interface + * + * Allocate memory for NTB EPC interface and initialize it. + */ +static int epf_ntb_epc_create_interface(struct epf_ntb *ntb, + struct pci_epc *epc, + enum pci_epc_interface_type type) +{ + const struct pci_epc_features *epc_features; + struct pci_epf_bar *epf_bar; + struct epf_ntb_epc *ntb_epc; + struct pci_epf *epf; + struct device *dev; + u8 func_no; + + dev = &ntb->epf->dev; + + ntb_epc = devm_kzalloc(dev, sizeof(*ntb_epc), GFP_KERNEL); + if (!ntb_epc) + return -ENOMEM; + + epf = ntb->epf; + if (type == PRIMARY_INTERFACE) { + func_no = epf->func_no; + epf_bar = epf->bar; + } else { + func_no = epf->sec_epc_func_no; + epf_bar = epf->sec_epc_bar; + } + + ntb_epc->linkup = false; + ntb_epc->epc = epc; + ntb_epc->func_no = func_no; + ntb_epc->type = type; + ntb_epc->epf_bar = epf_bar; + ntb_epc->epf_ntb = ntb; + + epc_features = pci_epc_get_features(epc, func_no); + if (!epc_features) + return -EINVAL; + ntb_epc->epc_features = epc_features; + + ntb->epc[type] = ntb_epc; + + return 0; +} + +/** + * epf_ntb_epc_create() - Create and initialize NTB EPC interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * Get a reference to EPC device and bind NTB function device to that EPC + * for each of the interface. It is also a wrapper to + * epf_ntb_epc_create_interface() to allocate memory for NTB EPC interface + * and initialize it + */ +static int epf_ntb_epc_create(struct epf_ntb *ntb) +{ + struct pci_epf *epf; + struct device *dev; + int ret; + + epf = ntb->epf; + dev = &epf->dev; + + ret = epf_ntb_epc_create_interface(ntb, epf->epc, PRIMARY_INTERFACE); + if (ret) { + dev_err(dev, "PRIMARY intf: Fail to create NTB EPC\n"); + return ret; + } + + ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc, + SECONDARY_INTERFACE); + if (ret) { + dev_err(dev, "SECONDARY intf: Fail to create NTB EPC\n"); + goto err_epc_create; + } + + return 0; + +err_epc_create: + epf_ntb_epc_destroy_interface(ntb, PRIMARY_INTERFACE); + + return ret; +} + +/** + * epf_ntb_init_epc_bar_interface() - Identify BARs to be used for each of + * the NTB constructs (scratchpad region, doorbell, memorywindow) + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Identify the free BARs to be used for each of BAR_CONFIG, BAR_PEER_SPAD, + * BAR_DB_MW1, BAR_MW2, BAR_MW3 and BAR_MW4. + */ +static int epf_ntb_init_epc_bar_interface(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + const struct pci_epc_features *epc_features; + struct epf_ntb_epc *ntb_epc; + enum pci_barno barno; + enum epf_ntb_bar bar; + struct device *dev; + u32 num_mws; + int i; + + barno = BAR_0; + ntb_epc = ntb->epc[type]; + num_mws = ntb->num_mws; + dev = &ntb->epf->dev; + epc_features = ntb_epc->epc_features; + + /* These are required BARs which are mandatory for NTB functionality */ + for (bar = BAR_CONFIG; bar <= BAR_DB_MW1; bar++, barno++) { + barno = pci_epc_get_next_free_bar(epc_features, barno); + if (barno < 0) { + dev_err(dev, "%s intf: Fail to get NTB function BAR\n", + pci_epc_interface_string(type)); + return barno; + } + ntb_epc->epf_ntb_bar[bar] = barno; + } + + /* These are optional BARs which don't impact NTB functionality */ + for (bar = BAR_MW2, i = 1; i < num_mws; bar++, barno++, i++) { + barno = pci_epc_get_next_free_bar(epc_features, barno); + if (barno < 0) { + ntb->num_mws = i; + dev_dbg(dev, "BAR not available for > MW%d\n", i + 1); + } + ntb_epc->epf_ntb_bar[bar] = barno; + } + + return 0; +} + +/** + * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB + * constructs (scratchpad region, doorbell, memorywindow) + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Wrapper to epf_ntb_init_epc_bar_interface() to identify the free BARs + * to be used for each of BAR_CONFIG, BAR_PEER_SPAD, BAR_DB_MW1, BAR_MW2, + * BAR_MW3 and BAR_MW4 for all the interfaces. + */ +static int epf_ntb_init_epc_bar(struct epf_ntb *ntb) +{ + enum pci_epc_interface_type type; + struct device *dev; + int ret; + + dev = &ntb->epf->dev; + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { + ret = epf_ntb_init_epc_bar_interface(ntb, type); + if (ret) { + dev_err(dev, "Fail to init EPC bar for %s interface\n", + pci_epc_interface_string(type)); + return ret; + } + } + + return 0; +} + +/** + * epf_ntb_epc_init_interface() - Initialize NTB interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Wrapper to initialize a particular EPC interface and start the workqueue + * to check for commands from host. This function will write to the + * EP controller HW for configuring it. + */ +static int epf_ntb_epc_init_interface(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + struct epf_ntb_epc *ntb_epc; + struct pci_epc *epc; + struct pci_epf *epf; + struct device *dev; + u8 func_no; + int ret; + + ntb_epc = ntb->epc[type]; + epf = ntb->epf; + dev = &epf->dev; + epc = ntb_epc->epc; + func_no = ntb_epc->func_no; + + ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]); + if (ret) { + dev_err(dev, "%s intf: Config/self SPAD BAR init failed\n", + pci_epc_interface_string(type)); + return ret; + } + + ret = epf_ntb_peer_spad_bar_set(ntb, type); + if (ret) { + dev_err(dev, "%s intf: Peer SPAD BAR init failed\n", + pci_epc_interface_string(type)); + goto err_peer_spad_bar_init; + } + + ret = epf_ntb_configure_interrupt(ntb, type); + if (ret) { + dev_err(dev, "%s intf: Interrupt configuration failed\n", + pci_epc_interface_string(type)); + goto err_peer_spad_bar_init; + } + + ret = epf_ntb_db_mw_bar_init(ntb, type); + if (ret) { + dev_err(dev, "%s intf: DB/MW BAR init failed\n", + pci_epc_interface_string(type)); + goto err_db_mw_bar_init; + } + + ret = pci_epc_write_header(epc, func_no, epf->header); + if (ret) { + dev_err(dev, "%s intf: Configuration header write failed\n", + pci_epc_interface_string(type)); + goto err_write_header; + } + + INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler); + queue_work(kpcintb_workqueue, &ntb->epc[type]->cmd_handler.work); + + return 0; + +err_write_header: + epf_ntb_db_mw_bar_cleanup(ntb, type); + +err_db_mw_bar_init: + epf_ntb_peer_spad_bar_clear(ntb->epc[type]); + +err_peer_spad_bar_init: + epf_ntb_config_sspad_bar_clear(ntb->epc[type]); + + return ret; +} + +/** + * epf_ntb_epc_cleanup_interface() - Cleanup NTB interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Wrapper to cleanup a particular NTB interface. + */ +static void epf_ntb_epc_cleanup_interface(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + struct epf_ntb_epc *ntb_epc; + + if (type < 0) + return; + + ntb_epc = ntb->epc[type]; + cancel_delayed_work(&ntb_epc->cmd_handler); + epf_ntb_db_mw_bar_cleanup(ntb, type); + epf_ntb_peer_spad_bar_clear(ntb_epc); + epf_ntb_config_sspad_bar_clear(ntb_epc); +} + +/** + * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * Wrapper to cleanup all NTB interfaces. + */ +static void epf_ntb_epc_cleanup(struct epf_ntb *ntb) +{ + enum pci_epc_interface_type type; + + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) + epf_ntb_epc_cleanup_interface(ntb, type); +} + +/** + * epf_ntb_epc_init() - Initialize all NTB interfaces + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * Wrapper to initialize all NTB interface and start the workqueue + * to check for commands from host. + */ +static int epf_ntb_epc_init(struct epf_ntb *ntb) +{ + enum pci_epc_interface_type type; + struct device *dev; + int ret; + + dev = &ntb->epf->dev; + + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { + ret = epf_ntb_epc_init_interface(ntb, type); + if (ret) { + dev_err(dev, "%s intf: Failed to initialize\n", + pci_epc_interface_string(type)); + goto err_init_type; + } + } + + return 0; + +err_init_type: + epf_ntb_epc_cleanup_interface(ntb, type - 1); + + return ret; +} + +/** + * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality + * @epf: NTB endpoint function device + * + * Initialize both the endpoint controllers associated with NTB function device. + * Invoked when a primary interface or secondary interface is bound to EPC + * device. This function will succeed only when EPC is bound to both the + * interfaces. + */ +static int epf_ntb_bind(struct pci_epf *epf) +{ + struct epf_ntb *ntb = epf_get_drvdata(epf); + struct device *dev = &epf->dev; + int ret; + + if (!epf->epc) { + dev_dbg(dev, "PRIMARY EPC interface not yet bound\n"); + return 0; + } + + if (!epf->sec_epc) { + dev_dbg(dev, "SECONDARY EPC interface not yet bound\n"); + return 0; + } + + ret = epf_ntb_epc_create(ntb); + if (ret) { + dev_err(dev, "Failed to create NTB EPC\n"); + return ret; + } + + ret = epf_ntb_init_epc_bar(ntb); + if (ret) { + dev_err(dev, "Failed to create NTB EPC\n"); + goto err_bar_init; + } + + ret = epf_ntb_config_spad_bar_alloc_interface(ntb); + if (ret) { + dev_err(dev, "Failed to allocate BAR memory\n"); + goto err_bar_alloc; + } + + ret = epf_ntb_epc_init(ntb); + if (ret) { + dev_err(dev, "Failed to initialize EPC\n"); + goto err_bar_alloc; + } + + epf_set_drvdata(epf, ntb); + + return 0; + +err_bar_alloc: + epf_ntb_config_spad_bar_free(ntb); + +err_bar_init: + epf_ntb_epc_destroy(ntb); + + return ret; +} + +/** + * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind() + * @epf: NTB endpoint function device + * + * Cleanup the initialization from epf_ntb_bind() + */ +static void epf_ntb_unbind(struct pci_epf *epf) +{ + struct epf_ntb *ntb = epf_get_drvdata(epf); + + epf_ntb_epc_cleanup(ntb); + epf_ntb_config_spad_bar_free(ntb); + epf_ntb_epc_destroy(ntb); +} + +#define EPF_NTB_R(_name) \ +static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ + char *page) \ +{ \ + struct config_group *group = to_config_group(item); \ + struct epf_ntb *ntb = to_epf_ntb(group); \ + \ + return sprintf(page, "%d\n", ntb->_name); \ +} + +#define EPF_NTB_W(_name) \ +static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ + const char *page, size_t len) \ +{ \ + struct config_group *group = to_config_group(item); \ + struct epf_ntb *ntb = to_epf_ntb(group); \ + u32 val; \ + int ret; \ + \ + ret = kstrtou32(page, 0, &val); \ + if (ret) \ + return ret; \ + \ + ntb->_name = val; \ + \ + return len; \ +} + +#define EPF_NTB_MW_R(_name) \ +static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ + char *page) \ +{ \ + struct config_group *group = to_config_group(item); \ + struct epf_ntb *ntb = to_epf_ntb(group); \ + int win_no; \ + \ + sscanf(#_name, "mw%d", &win_no); \ + \ + return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]); \ +} + +#define EPF_NTB_MW_W(_name) \ +static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ + const char *page, size_t len) \ +{ \ + struct config_group *group = to_config_group(item); \ + struct epf_ntb *ntb = to_epf_ntb(group); \ + struct device *dev = &ntb->epf->dev; \ + int win_no; \ + u64 val; \ + int ret; \ + \ + ret = kstrtou64(page, 0, &val); \ + if (ret) \ + return ret; \ + \ + if (sscanf(#_name, "mw%d", &win_no) != 1) \ + return -EINVAL; \ + \ + if (ntb->num_mws < win_no) { \ + dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \ + return -EINVAL; \ + } \ + \ + ntb->mws_size[win_no - 1] = val; \ + \ + return len; \ +} + +static ssize_t epf_ntb_num_mws_store(struct config_item *item, + const char *page, size_t len) +{ + struct config_group *group = to_config_group(item); + struct epf_ntb *ntb = to_epf_ntb(group); + u32 val; + int ret; + + ret = kstrtou32(page, 0, &val); + if (ret) + return ret; + + if (val > MAX_MW) + return -EINVAL; + + ntb->num_mws = val; + + return len; +} + +EPF_NTB_R(spad_count) +EPF_NTB_W(spad_count) +EPF_NTB_R(db_count) +EPF_NTB_W(db_count) +EPF_NTB_R(num_mws) +EPF_NTB_MW_R(mw1) +EPF_NTB_MW_W(mw1) +EPF_NTB_MW_R(mw2) +EPF_NTB_MW_W(mw2) +EPF_NTB_MW_R(mw3) +EPF_NTB_MW_W(mw3) +EPF_NTB_MW_R(mw4) +EPF_NTB_MW_W(mw4) + +CONFIGFS_ATTR(epf_ntb_, spad_count); +CONFIGFS_ATTR(epf_ntb_, db_count); +CONFIGFS_ATTR(epf_ntb_, num_mws); +CONFIGFS_ATTR(epf_ntb_, mw1); +CONFIGFS_ATTR(epf_ntb_, mw2); +CONFIGFS_ATTR(epf_ntb_, mw3); +CONFIGFS_ATTR(epf_ntb_, mw4); + +static struct configfs_attribute *epf_ntb_attrs[] = { + &epf_ntb_attr_spad_count, + &epf_ntb_attr_db_count, + &epf_ntb_attr_num_mws, + &epf_ntb_attr_mw1, + &epf_ntb_attr_mw2, + &epf_ntb_attr_mw3, + &epf_ntb_attr_mw4, + NULL, +}; + +static const struct config_item_type ntb_group_type = { + .ct_attrs = epf_ntb_attrs, + .ct_owner = THIS_MODULE, +}; + +/** + * epf_ntb_add_cfs() - Add configfs directory specific to NTB + * @epf: NTB endpoint function device + * + * Add configfs directory specific to NTB. This directory will hold + * NTB specific properties like db_count, spad_count, num_mws etc., + */ +static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf, + struct config_group *group) +{ + struct epf_ntb *ntb = epf_get_drvdata(epf); + struct config_group *ntb_group = &ntb->group; + struct device *dev = &epf->dev; + + config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type); + + return ntb_group; +} + +/** + * epf_ntb_probe() - Probe NTB function driver + * @epf: NTB endpoint function device + * + * Probe NTB function driver when endpoint function bus detects a NTB + * endpoint function. + */ +static int epf_ntb_probe(struct pci_epf *epf) +{ + struct epf_ntb *ntb; + struct device *dev; + + dev = &epf->dev; + + ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL); + if (!ntb) + return -ENOMEM; + + epf->header = &epf_ntb_header; + ntb->epf = epf; + epf_set_drvdata(epf, ntb); + + return 0; +} + +static struct pci_epf_ops epf_ntb_ops = { + .bind = epf_ntb_bind, + .unbind = epf_ntb_unbind, + .add_cfs = epf_ntb_add_cfs, +}; + +static const struct pci_epf_device_id epf_ntb_ids[] = { + { + .name = "pci_epf_ntb", + }, + {}, +}; + +static struct pci_epf_driver epf_ntb_driver = { + .driver.name = "pci_epf_ntb", + .probe = epf_ntb_probe, + .id_table = epf_ntb_ids, + .ops = &epf_ntb_ops, + .owner = THIS_MODULE, +}; + +static int __init epf_ntb_init(void) +{ + int ret; + + kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM | + WQ_HIGHPRI, 0); + ret = pci_epf_register_driver(&epf_ntb_driver); + if (ret) { + destroy_workqueue(kpcintb_workqueue); + pr_err("Failed to register pci epf ntb driver --> %d\n", ret); + return ret; + } + + return 0; +} +module_init(epf_ntb_init); + +static void __exit epf_ntb_exit(void) +{ + pci_epf_unregister_driver(&epf_ntb_driver); + destroy_workqueue(kpcintb_workqueue); +} +module_exit(epf_ntb_exit); + +MODULE_DESCRIPTION("PCI EPF NTB DRIVER"); +MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index e4e51d884553..c0ac4e9cbe72 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -619,7 +619,8 @@ static void pci_epf_test_unbind(struct pci_epf *epf) if (epf_test->reg[bar]) { pci_epc_clear_bar(epc, epf->func_no, epf_bar); - pci_epf_free_space(epf, epf_test->reg[bar], bar); + pci_epf_free_space(epf, epf_test->reg[bar], bar, + PRIMARY_INTERFACE); } } } @@ -651,7 +652,8 @@ static int pci_epf_test_set_bar(struct pci_epf *epf) ret = pci_epc_set_bar(epc, epf->func_no, epf_bar); if (ret) { - pci_epf_free_space(epf, epf_test->reg[bar], bar); + pci_epf_free_space(epf, epf_test->reg[bar], bar, + PRIMARY_INTERFACE); dev_err(dev, "Failed to set BAR%d\n", bar); if (bar == test_reg_bar) return ret; @@ -771,7 +773,7 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf) } base = pci_epf_alloc_space(epf, test_reg_size, test_reg_bar, - epc_features->align); + epc_features->align, PRIMARY_INTERFACE); if (!base) { dev_err(dev, "Failed to allocated register space\n"); return -ENOMEM; @@ -789,7 +791,8 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf) continue; base = pci_epf_alloc_space(epf, bar_size[bar], bar, - epc_features->align); + epc_features->align, + PRIMARY_INTERFACE); if (!base) dev_err(dev, "Failed to allocate space for BAR%d\n", bar); @@ -834,6 +837,8 @@ static int pci_epf_test_bind(struct pci_epf *epf) linkup_notifier = epc_features->linkup_notifier; core_init_notifier = epc_features->core_init_notifier; test_reg_bar = pci_epc_get_first_free_bar(epc_features); + if (test_reg_bar < 0) + return -EINVAL; pci_epf_configure_bar(epf, epc_features); } diff --git a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c index 3710adf51912..f3a8b833b479 100644 --- a/drivers/pci/endpoint/pci-ep-cfs.c +++ b/drivers/pci/endpoint/pci-ep-cfs.c @@ -21,6 +21,9 @@ static struct config_group *controllers_group; struct pci_epf_group { struct config_group group; + struct config_group primary_epc_group; + struct config_group secondary_epc_group; + struct delayed_work cfs_work; struct pci_epf *epf; int index; }; @@ -41,6 +44,127 @@ static inline struct pci_epc_group *to_pci_epc_group(struct config_item *item) return container_of(to_config_group(item), struct pci_epc_group, group); } +static int pci_secondary_epc_epf_link(struct config_item *epf_item, + struct config_item *epc_item) +{ + int ret; + struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); + struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); + struct pci_epc *epc = epc_group->epc; + struct pci_epf *epf = epf_group->epf; + + ret = pci_epc_add_epf(epc, epf, SECONDARY_INTERFACE); + if (ret) + return ret; + + ret = pci_epf_bind(epf); + if (ret) { + pci_epc_remove_epf(epc, epf, SECONDARY_INTERFACE); + return ret; + } + + return 0; +} + +static void pci_secondary_epc_epf_unlink(struct config_item *epc_item, + struct config_item *epf_item) +{ + struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); + struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); + struct pci_epc *epc; + struct pci_epf *epf; + + WARN_ON_ONCE(epc_group->start); + + epc = epc_group->epc; + epf = epf_group->epf; + pci_epf_unbind(epf); + pci_epc_remove_epf(epc, epf, SECONDARY_INTERFACE); +} + +static struct configfs_item_operations pci_secondary_epc_item_ops = { + .allow_link = pci_secondary_epc_epf_link, + .drop_link = pci_secondary_epc_epf_unlink, +}; + +static const struct config_item_type pci_secondary_epc_type = { + .ct_item_ops = &pci_secondary_epc_item_ops, + .ct_owner = THIS_MODULE, +}; + +static struct config_group +*pci_ep_cfs_add_secondary_group(struct pci_epf_group *epf_group) +{ + struct config_group *secondary_epc_group; + + secondary_epc_group = &epf_group->secondary_epc_group; + config_group_init_type_name(secondary_epc_group, "secondary", + &pci_secondary_epc_type); + configfs_register_group(&epf_group->group, secondary_epc_group); + + return secondary_epc_group; +} + +static int pci_primary_epc_epf_link(struct config_item *epf_item, + struct config_item *epc_item) +{ + int ret; + struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); + struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); + struct pci_epc *epc = epc_group->epc; + struct pci_epf *epf = epf_group->epf; + + ret = pci_epc_add_epf(epc, epf, PRIMARY_INTERFACE); + if (ret) + return ret; + + ret = pci_epf_bind(epf); + if (ret) { + pci_epc_remove_epf(epc, epf, PRIMARY_INTERFACE); + return ret; + } + + return 0; +} + +static void pci_primary_epc_epf_unlink(struct config_item *epc_item, + struct config_item *epf_item) +{ + struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); + struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); + struct pci_epc *epc; + struct pci_epf *epf; + + WARN_ON_ONCE(epc_group->start); + + epc = epc_group->epc; + epf = epf_group->epf; + pci_epf_unbind(epf); + pci_epc_remove_epf(epc, epf, PRIMARY_INTERFACE); +} + +static struct configfs_item_operations pci_primary_epc_item_ops = { + .allow_link = pci_primary_epc_epf_link, + .drop_link = pci_primary_epc_epf_unlink, +}; + +static const struct config_item_type pci_primary_epc_type = { + .ct_item_ops = &pci_primary_epc_item_ops, + .ct_owner = THIS_MODULE, +}; + +static struct config_group +*pci_ep_cfs_add_primary_group(struct pci_epf_group *epf_group) +{ + struct config_group *primary_epc_group = &epf_group->primary_epc_group; + + config_group_init_type_name(primary_epc_group, "primary", + &pci_primary_epc_type); + configfs_register_group(&epf_group->group, primary_epc_group); + + return primary_epc_group; +} + static ssize_t pci_epc_start_store(struct config_item *item, const char *page, size_t len) { @@ -94,13 +218,13 @@ static int pci_epc_epf_link(struct config_item *epc_item, struct pci_epc *epc = epc_group->epc; struct pci_epf *epf = epf_group->epf; - ret = pci_epc_add_epf(epc, epf); + ret = pci_epc_add_epf(epc, epf, PRIMARY_INTERFACE); if (ret) return ret; ret = pci_epf_bind(epf); if (ret) { - pci_epc_remove_epf(epc, epf); + pci_epc_remove_epf(epc, epf, PRIMARY_INTERFACE); return ret; } @@ -120,7 +244,7 @@ static void pci_epc_epf_unlink(struct config_item *epc_item, epc = epc_group->epc; epf = epf_group->epf; pci_epf_unbind(epf); - pci_epc_remove_epf(epc, epf); + pci_epc_remove_epf(epc, epf, PRIMARY_INTERFACE); } static struct configfs_item_operations pci_epc_item_ops = { @@ -366,12 +490,53 @@ static struct configfs_item_operations pci_epf_ops = { .release = pci_epf_release, }; +static struct config_group *pci_epf_type_make(struct config_group *group, + const char *name) +{ + struct pci_epf_group *epf_group = to_pci_epf_group(&group->cg_item); + struct config_group *epf_type_group; + + epf_type_group = pci_epf_type_add_cfs(epf_group->epf, group); + return epf_type_group; +} + +static void pci_epf_type_drop(struct config_group *group, + struct config_item *item) +{ + config_item_put(item); +} + +static struct configfs_group_operations pci_epf_type_group_ops = { + .make_group = &pci_epf_type_make, + .drop_item = &pci_epf_type_drop, +}; + static const struct config_item_type pci_epf_type = { + .ct_group_ops = &pci_epf_type_group_ops, .ct_item_ops = &pci_epf_ops, .ct_attrs = pci_epf_attrs, .ct_owner = THIS_MODULE, }; +static void pci_epf_cfs_work(struct work_struct *work) +{ + struct pci_epf_group *epf_group; + struct config_group *group; + + epf_group = container_of(work, struct pci_epf_group, cfs_work.work); + group = pci_ep_cfs_add_primary_group(epf_group); + if (IS_ERR(group)) { + pr_err("failed to create 'primary' EPC interface\n"); + return; + } + + group = pci_ep_cfs_add_secondary_group(epf_group); + if (IS_ERR(group)) { + pr_err("failed to create 'secondary' EPC interface\n"); + return; + } +} + static struct config_group *pci_epf_make(struct config_group *group, const char *name) { @@ -410,10 +575,15 @@ static struct config_group *pci_epf_make(struct config_group *group, goto free_name; } + epf->group = &epf_group->group; epf_group->epf = epf; kfree(epf_name); + INIT_DELAYED_WORK(&epf_group->cfs_work, pci_epf_cfs_work); + queue_delayed_work(system_wq, &epf_group->cfs_work, + msecs_to_jiffies(1)); + return &epf_group->group; free_name: diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index cadd3db0cbb0..cc8f9eb2b177 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -87,24 +87,50 @@ EXPORT_SYMBOL_GPL(pci_epc_get); * pci_epc_get_first_free_bar() - helper to get first unreserved BAR * @epc_features: pci_epc_features structure that holds the reserved bar bitmap * - * Invoke to get the first unreserved BAR that can be used for endpoint + * Invoke to get the first unreserved BAR that can be used by the endpoint * function. For any incorrect value in reserved_bar return '0'. */ -unsigned int pci_epc_get_first_free_bar(const struct pci_epc_features - *epc_features) +enum pci_barno +pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features) { - int free_bar; + return pci_epc_get_next_free_bar(epc_features, BAR_0); +} +EXPORT_SYMBOL_GPL(pci_epc_get_first_free_bar); + +/** + * pci_epc_get_next_free_bar() - helper to get unreserved BAR starting from @bar + * @epc_features: pci_epc_features structure that holds the reserved bar bitmap + * @bar: the starting BAR number from where unreserved BAR should be searched + * + * Invoke to get the next unreserved BAR starting from @bar that can be used + * for endpoint function. For any incorrect value in reserved_bar return '0'. + */ +enum pci_barno pci_epc_get_next_free_bar(const struct pci_epc_features + *epc_features, enum pci_barno bar) +{ + unsigned long free_bar; if (!epc_features) - return 0; + return BAR_0; + + /* If 'bar - 1' is a 64-bit BAR, move to the next BAR */ + if ((epc_features->bar_fixed_64bit << 1) & 1 << bar) + bar++; + + /* Find if the reserved BAR is also a 64-bit BAR */ + free_bar = epc_features->reserved_bar & epc_features->bar_fixed_64bit; - free_bar = ffz(epc_features->reserved_bar); + /* Set the adjacent bit if the reserved BAR is also a 64-bit BAR */ + free_bar <<= 1; + free_bar |= epc_features->reserved_bar; + + free_bar = find_next_zero_bit(&free_bar, 6, bar); if (free_bar > 5) - return 0; + return NO_BAR; return free_bar; } -EXPORT_SYMBOL_GPL(pci_epc_get_first_free_bar); +EXPORT_SYMBOL_GPL(pci_epc_get_next_free_bar); /** * pci_epc_get_features() - get the features supported by EPC @@ -205,6 +231,47 @@ int pci_epc_raise_irq(struct pci_epc *epc, u8 func_no, EXPORT_SYMBOL_GPL(pci_epc_raise_irq); /** + * pci_epc_map_msi_irq() - Map physical address to MSI address and return + * MSI data + * @epc: the EPC device which has the MSI capability + * @func_no: the physical endpoint function number in the EPC device + * @phys_addr: the physical address of the outbound region + * @interrupt_num: the MSI interrupt number + * @entry_size: Size of Outbound address region for each interrupt + * @msi_data: the data that should be written in order to raise MSI interrupt + * with interrupt number as 'interrupt num' + * @msi_addr_offset: Offset of MSI address from the aligned outbound address + * to which the MSI address is mapped + * + * Invoke to map physical address to MSI address and return MSI data. The + * physical address should be an address in the outbound region. This is + * required to implement doorbell functionality of NTB wherein EPC on either + * side of the interface (primary and secondary) can directly write to the + * physical address (in outbound region) of the other interface to ring + * doorbell. + */ +int pci_epc_map_msi_irq(struct pci_epc *epc, u8 func_no, phys_addr_t phys_addr, + u8 interrupt_num, u32 entry_size, u32 *msi_data, + u32 *msi_addr_offset) +{ + int ret; + + if (IS_ERR_OR_NULL(epc)) + return -EINVAL; + + if (!epc->ops->map_msi_irq) + return -EINVAL; + + mutex_lock(&epc->lock); + ret = epc->ops->map_msi_irq(epc, func_no, phys_addr, interrupt_num, + entry_size, msi_data, msi_addr_offset); + mutex_unlock(&epc->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(pci_epc_map_msi_irq); + +/** * pci_epc_get_msi() - get the number of MSI interrupt numbers allocated * @epc: the EPC device to which MSI interrupts was requested * @func_no: the endpoint function number in the EPC device @@ -467,21 +534,28 @@ EXPORT_SYMBOL_GPL(pci_epc_write_header); * pci_epc_add_epf() - bind PCI endpoint function to an endpoint controller * @epc: the EPC device to which the endpoint function should be added * @epf: the endpoint function to be added + * @type: Identifies if the EPC is connected to the primary or secondary + * interface of EPF * * A PCI endpoint device can have one or more functions. In the case of PCIe, * the specification allows up to 8 PCIe endpoint functions. Invoke * pci_epc_add_epf() to add a PCI endpoint function to an endpoint controller. */ -int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf) +int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf, + enum pci_epc_interface_type type) { + struct list_head *list; u32 func_no; int ret = 0; - if (epf->epc) + if (IS_ERR_OR_NULL(epc)) + return -EINVAL; + + if (type == PRIMARY_INTERFACE && epf->epc) return -EBUSY; - if (IS_ERR(epc)) - return -EINVAL; + if (type == SECONDARY_INTERFACE && epf->sec_epc) + return -EBUSY; mutex_lock(&epc->lock); func_no = find_first_zero_bit(&epc->function_num_map, @@ -498,11 +572,17 @@ int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf) } set_bit(func_no, &epc->function_num_map); - epf->func_no = func_no; - epf->epc = epc; - - list_add_tail(&epf->list, &epc->pci_epf); + if (type == PRIMARY_INTERFACE) { + epf->func_no = func_no; + epf->epc = epc; + list = &epf->list; + } else { + epf->sec_epc_func_no = func_no; + epf->sec_epc = epc; + list = &epf->sec_epc_list; + } + list_add_tail(list, &epc->pci_epf); ret: mutex_unlock(&epc->lock); @@ -517,14 +597,26 @@ EXPORT_SYMBOL_GPL(pci_epc_add_epf); * * Invoke to remove PCI endpoint function from the endpoint controller. */ -void pci_epc_remove_epf(struct pci_epc *epc, struct pci_epf *epf) +void pci_epc_remove_epf(struct pci_epc *epc, struct pci_epf *epf, + enum pci_epc_interface_type type) { + struct list_head *list; + u32 func_no = 0; + if (!epc || IS_ERR(epc) || !epf) return; + if (type == PRIMARY_INTERFACE) { + func_no = epf->func_no; + list = &epf->list; + } else { + func_no = epf->sec_epc_func_no; + list = &epf->sec_epc_list; + } + mutex_lock(&epc->lock); - clear_bit(epf->func_no, &epc->function_num_map); - list_del(&epf->list); + clear_bit(func_no, &epc->function_num_map); + list_del(list); epf->epc = NULL; mutex_unlock(&epc->lock); } diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c index c977cf9dce56..7646c8660d42 100644 --- a/drivers/pci/endpoint/pci-epf-core.c +++ b/drivers/pci/endpoint/pci-epf-core.c @@ -21,6 +21,38 @@ static struct bus_type pci_epf_bus_type; static const struct device_type pci_epf_type; /** + * pci_epf_type_add_cfs() - Help function drivers to expose function specific + * attributes in configfs + * @epf: the EPF device that has to be configured using configfs + * @group: the parent configfs group (corresponding to entries in + * pci_epf_device_id) + * + * Invoke to expose function specific attributes in configfs. If the function + * driver does not have anything to expose (attributes configured by user), + * return NULL. + */ +struct config_group *pci_epf_type_add_cfs(struct pci_epf *epf, + struct config_group *group) +{ + struct config_group *epf_type_group; + + if (!epf->driver) { + dev_err(&epf->dev, "epf device not bound to driver\n"); + return NULL; + } + + if (!epf->driver->ops->add_cfs) + return NULL; + + mutex_lock(&epf->lock); + epf_type_group = epf->driver->ops->add_cfs(epf, group); + mutex_unlock(&epf->lock); + + return epf_type_group; +} +EXPORT_SYMBOL_GPL(pci_epf_type_add_cfs); + +/** * pci_epf_unbind() - Notify the function driver that the binding between the * EPF device and EPC device has been lost * @epf: the EPF device which has lost the binding with the EPC device @@ -74,24 +106,37 @@ EXPORT_SYMBOL_GPL(pci_epf_bind); * @epf: the EPF device from whom to free the memory * @addr: the virtual address of the PCI EPF register space * @bar: the BAR number corresponding to the register space + * @type: Identifies if the allocated space is for primary EPC or secondary EPC * * Invoke to free the allocated PCI EPF register space. */ -void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar) +void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar, + enum pci_epc_interface_type type) { struct device *dev = epf->epc->dev.parent; + struct pci_epf_bar *epf_bar; + struct pci_epc *epc; if (!addr) return; - dma_free_coherent(dev, epf->bar[bar].size, addr, - epf->bar[bar].phys_addr); + if (type == PRIMARY_INTERFACE) { + epc = epf->epc; + epf_bar = epf->bar; + } else { + epc = epf->sec_epc; + epf_bar = epf->sec_epc_bar; + } - epf->bar[bar].phys_addr = 0; - epf->bar[bar].addr = NULL; - epf->bar[bar].size = 0; - epf->bar[bar].barno = 0; - epf->bar[bar].flags = 0; + dev = epc->dev.parent; + dma_free_coherent(dev, epf_bar[bar].size, addr, + epf_bar[bar].phys_addr); + + epf_bar[bar].phys_addr = 0; + epf_bar[bar].addr = NULL; + epf_bar[bar].size = 0; + epf_bar[bar].barno = 0; + epf_bar[bar].flags = 0; } EXPORT_SYMBOL_GPL(pci_epf_free_space); @@ -101,15 +146,18 @@ EXPORT_SYMBOL_GPL(pci_epf_free_space); * @size: the size of the memory that has to be allocated * @bar: the BAR number corresponding to the allocated register space * @align: alignment size for the allocation region + * @type: Identifies if the allocation is for primary EPC or secondary EPC * * Invoke to allocate memory for the PCI EPF register space. */ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, - size_t align) + size_t align, enum pci_epc_interface_type type) { - void *space; - struct device *dev = epf->epc->dev.parent; + struct pci_epf_bar *epf_bar; dma_addr_t phys_addr; + struct pci_epc *epc; + struct device *dev; + void *space; if (size < 128) size = 128; @@ -119,17 +167,26 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, else size = roundup_pow_of_two(size); + if (type == PRIMARY_INTERFACE) { + epc = epf->epc; + epf_bar = epf->bar; + } else { + epc = epf->sec_epc; + epf_bar = epf->sec_epc_bar; + } + + dev = epc->dev.parent; space = dma_alloc_coherent(dev, size, &phys_addr, GFP_KERNEL); if (!space) { dev_err(dev, "failed to allocate mem space\n"); return NULL; } - epf->bar[bar].phys_addr = phys_addr; - epf->bar[bar].addr = space; - epf->bar[bar].size = size; - epf->bar[bar].barno = bar; - epf->bar[bar].flags |= upper_32_bits(size) ? + epf_bar[bar].phys_addr = phys_addr; + epf_bar[bar].addr = space; + epf_bar[bar].size = size; + epf_bar[bar].barno = bar; + epf_bar[bar].flags |= upper_32_bits(size) ? PCI_BASE_ADDRESS_MEM_TYPE_64 : PCI_BASE_ADDRESS_MEM_TYPE_32; @@ -282,22 +339,6 @@ struct pci_epf *pci_epf_create(const char *name) } EXPORT_SYMBOL_GPL(pci_epf_create); -const struct pci_epf_device_id * -pci_epf_match_device(const struct pci_epf_device_id *id, struct pci_epf *epf) -{ - if (!id || !epf) - return NULL; - - while (*id->name) { - if (strcmp(epf->name, id->name) == 0) - return id; - id++; - } - - return NULL; -} -EXPORT_SYMBOL_GPL(pci_epf_match_device); - static void pci_epf_dev_release(struct device *dev) { struct pci_epf *epf = to_pci_epf(dev); diff --git a/drivers/pci/hotplug/acpiphp.h b/drivers/pci/hotplug/acpiphp.h index a2094c07af6a..a74b274a8c45 100644 --- a/drivers/pci/hotplug/acpiphp.h +++ b/drivers/pci/hotplug/acpiphp.h @@ -176,9 +176,6 @@ int acpiphp_unregister_attention(struct acpiphp_attention_info *info); int acpiphp_register_hotplug_slot(struct acpiphp_slot *slot, unsigned int sun); void acpiphp_unregister_hotplug_slot(struct acpiphp_slot *slot); -/* acpiphp_glue.c */ -typedef int (*acpiphp_callback)(struct acpiphp_slot *slot, void *data); - int acpiphp_enable_slot(struct acpiphp_slot *slot); int acpiphp_disable_slot(struct acpiphp_slot *slot); u8 acpiphp_get_power_status(struct acpiphp_slot *slot); diff --git a/drivers/pci/pci-bridge-emul.c b/drivers/pci/pci-bridge-emul.c index 139869d50eb2..fdaf86a888b7 100644 --- a/drivers/pci/pci-bridge-emul.c +++ b/drivers/pci/pci-bridge-emul.c @@ -21,8 +21,9 @@ #include "pci-bridge-emul.h" #define PCI_BRIDGE_CONF_END PCI_STD_HEADER_SIZEOF +#define PCI_CAP_PCIE_SIZEOF (PCI_EXP_SLTSTA2 + 2) #define PCI_CAP_PCIE_START PCI_BRIDGE_CONF_END -#define PCI_CAP_PCIE_END (PCI_CAP_PCIE_START + PCI_EXP_SLTSTA2 + 2) +#define PCI_CAP_PCIE_END (PCI_CAP_PCIE_START + PCI_CAP_PCIE_SIZEOF) /** * struct pci_bridge_reg_behavior - register bits behaviors @@ -46,7 +47,8 @@ struct pci_bridge_reg_behavior { u32 w1c; }; -static const struct pci_bridge_reg_behavior pci_regs_behavior[] = { +static const +struct pci_bridge_reg_behavior pci_regs_behavior[PCI_STD_HEADER_SIZEOF / 4] = { [PCI_VENDOR_ID / 4] = { .ro = ~0 }, [PCI_COMMAND / 4] = { .rw = (PCI_COMMAND_IO | PCI_COMMAND_MEMORY | @@ -164,7 +166,8 @@ static const struct pci_bridge_reg_behavior pci_regs_behavior[] = { }, }; -static const struct pci_bridge_reg_behavior pcie_cap_regs_behavior[] = { +static const +struct pci_bridge_reg_behavior pcie_cap_regs_behavior[PCI_CAP_PCIE_SIZEOF / 4] = { [PCI_CAP_LIST_ID / 4] = { /* * Capability ID, Next Capability Pointer and @@ -260,6 +263,8 @@ static const struct pci_bridge_reg_behavior pcie_cap_regs_behavior[] = { int pci_bridge_emul_init(struct pci_bridge_emul *bridge, unsigned int flags) { + BUILD_BUG_ON(sizeof(bridge->conf) != PCI_BRIDGE_CONF_END); + bridge->conf.class_revision |= cpu_to_le32(PCI_CLASS_BRIDGE_PCI << 16); bridge->conf.header_type = PCI_HEADER_TYPE_BRIDGE; bridge->conf.cache_line_size = 0x10; diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index b67c4327d307..16a17215f633 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -4030,6 +4030,10 @@ int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr, ret = logic_pio_register_range(range); if (ret) kfree(range); + + /* Ignore duplicates due to deferred probing */ + if (ret == -EEXIST) + ret = 0; #endif return ret; diff --git a/drivers/pci/pcie/Kconfig b/drivers/pci/pcie/Kconfig index 3946555a6042..45a2ef702b45 100644 --- a/drivers/pci/pcie/Kconfig +++ b/drivers/pci/pcie/Kconfig @@ -133,14 +133,6 @@ config PCIE_PTM This is only useful if you have devices that support PTM, but it is safe to enable even if you don't. -config PCIE_BW - bool "PCI Express Bandwidth Change Notification" - depends on PCIEPORTBUS - help - This enables PCI Express Bandwidth Change Notification. If - you know link width or rate changes occur only to correct - unreliable links, you may answer Y. - config PCIE_EDR bool "PCI Express Error Disconnect Recover support" depends on PCIE_DPC && ACPI diff --git a/drivers/pci/pcie/Makefile b/drivers/pci/pcie/Makefile index d9697892fa3e..b2980db88cc0 100644 --- a/drivers/pci/pcie/Makefile +++ b/drivers/pci/pcie/Makefile @@ -12,5 +12,4 @@ obj-$(CONFIG_PCIEAER_INJECT) += aer_inject.o obj-$(CONFIG_PCIE_PME) += pme.o obj-$(CONFIG_PCIE_DPC) += dpc.o obj-$(CONFIG_PCIE_PTM) += ptm.o -obj-$(CONFIG_PCIE_BW) += bw_notification.o obj-$(CONFIG_PCIE_EDR) += edr.o diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 77b0f2c45bc0..ba22388342d1 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -1388,7 +1388,7 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) if (type == PCI_EXP_TYPE_RC_END) root = dev->rcec; else - root = dev; + root = pcie_find_root_port(dev); /* * If the platform retained control of AER, an RCiEP may not have @@ -1414,7 +1414,8 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) } } else { rc = pci_bus_error_reset(dev); - pci_info(dev, "Root Port link has been reset (%d)\n", rc); + pci_info(dev, "%s Port link has been reset (%d)\n", + pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc); } if ((host->native_aer || pcie_ports_native) && aer) { diff --git a/drivers/pci/pcie/bw_notification.c b/drivers/pci/pcie/bw_notification.c deleted file mode 100644 index 565d23cccb8b..000000000000 --- a/drivers/pci/pcie/bw_notification.c +++ /dev/null @@ -1,138 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * PCI Express Link Bandwidth Notification services driver - * Author: Alexandru Gagniuc <mr.nuke.me@gmail.com> - * - * Copyright (C) 2019, Dell Inc - * - * The PCIe Link Bandwidth Notification provides a way to notify the - * operating system when the link width or data rate changes. This - * capability is required for all root ports and downstream ports - * supporting links wider than x1 and/or multiple link speeds. - * - * This service port driver hooks into the bandwidth notification interrupt - * and warns when links become degraded in operation. - */ - -#define dev_fmt(fmt) "bw_notification: " fmt - -#include "../pci.h" -#include "portdrv.h" - -static bool pcie_link_bandwidth_notification_supported(struct pci_dev *dev) -{ - int ret; - u32 lnk_cap; - - ret = pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnk_cap); - return (ret == PCIBIOS_SUCCESSFUL) && (lnk_cap & PCI_EXP_LNKCAP_LBNC); -} - -static void pcie_enable_link_bandwidth_notification(struct pci_dev *dev) -{ - u16 lnk_ctl; - - pcie_capability_write_word(dev, PCI_EXP_LNKSTA, PCI_EXP_LNKSTA_LBMS); - - pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &lnk_ctl); - lnk_ctl |= PCI_EXP_LNKCTL_LBMIE; - pcie_capability_write_word(dev, PCI_EXP_LNKCTL, lnk_ctl); -} - -static void pcie_disable_link_bandwidth_notification(struct pci_dev *dev) -{ - u16 lnk_ctl; - - pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &lnk_ctl); - lnk_ctl &= ~PCI_EXP_LNKCTL_LBMIE; - pcie_capability_write_word(dev, PCI_EXP_LNKCTL, lnk_ctl); -} - -static irqreturn_t pcie_bw_notification_irq(int irq, void *context) -{ - struct pcie_device *srv = context; - struct pci_dev *port = srv->port; - u16 link_status, events; - int ret; - - ret = pcie_capability_read_word(port, PCI_EXP_LNKSTA, &link_status); - events = link_status & PCI_EXP_LNKSTA_LBMS; - - if (ret != PCIBIOS_SUCCESSFUL || !events) - return IRQ_NONE; - - pcie_capability_write_word(port, PCI_EXP_LNKSTA, events); - pcie_update_link_speed(port->subordinate, link_status); - return IRQ_WAKE_THREAD; -} - -static irqreturn_t pcie_bw_notification_handler(int irq, void *context) -{ - struct pcie_device *srv = context; - struct pci_dev *port = srv->port; - struct pci_dev *dev; - - /* - * Print status from downstream devices, not this root port or - * downstream switch port. - */ - down_read(&pci_bus_sem); - list_for_each_entry(dev, &port->subordinate->devices, bus_list) - pcie_report_downtraining(dev); - up_read(&pci_bus_sem); - - return IRQ_HANDLED; -} - -static int pcie_bandwidth_notification_probe(struct pcie_device *srv) -{ - int ret; - - /* Single-width or single-speed ports do not have to support this. */ - if (!pcie_link_bandwidth_notification_supported(srv->port)) - return -ENODEV; - - ret = request_threaded_irq(srv->irq, pcie_bw_notification_irq, - pcie_bw_notification_handler, - IRQF_SHARED, "PCIe BW notif", srv); - if (ret) - return ret; - - pcie_enable_link_bandwidth_notification(srv->port); - pci_info(srv->port, "enabled with IRQ %d\n", srv->irq); - - return 0; -} - -static void pcie_bandwidth_notification_remove(struct pcie_device *srv) -{ - pcie_disable_link_bandwidth_notification(srv->port); - free_irq(srv->irq, srv); -} - -static int pcie_bandwidth_notification_suspend(struct pcie_device *srv) -{ - pcie_disable_link_bandwidth_notification(srv->port); - return 0; -} - -static int pcie_bandwidth_notification_resume(struct pcie_device *srv) -{ - pcie_enable_link_bandwidth_notification(srv->port); - return 0; -} - -static struct pcie_port_service_driver pcie_bandwidth_notification_driver = { - .name = "pcie_bw_notification", - .port_type = PCIE_ANY_PORT, - .service = PCIE_PORT_SERVICE_BWNOTIF, - .probe = pcie_bandwidth_notification_probe, - .suspend = pcie_bandwidth_notification_suspend, - .resume = pcie_bandwidth_notification_resume, - .remove = pcie_bandwidth_notification_remove, -}; - -int __init pcie_bandwidth_notification_init(void) -{ - return pcie_port_service_register(&pcie_bandwidth_notification_driver); -} diff --git a/drivers/pci/pcie/err.c b/drivers/pci/pcie/err.c index 510f31f0ef6d..b576aa890c76 100644 --- a/drivers/pci/pcie/err.c +++ b/drivers/pci/pcie/err.c @@ -198,8 +198,7 @@ pci_ers_result_t pcie_do_recovery(struct pci_dev *dev, pci_dbg(bridge, "broadcast error_detected message\n"); if (state == pci_channel_io_frozen) { pci_walk_bridge(bridge, report_frozen_detected, &status); - status = reset_subordinates(bridge); - if (status != PCI_ERS_RESULT_RECOVERED) { + if (reset_subordinates(bridge) != PCI_ERS_RESULT_RECOVERED) { pci_warn(bridge, "subordinate device reset failed\n"); goto failed; } @@ -231,15 +230,14 @@ pci_ers_result_t pcie_do_recovery(struct pci_dev *dev, pci_walk_bridge(bridge, report_resume, &status); /* - * If we have native control of AER, clear error status in the Root - * Port or Downstream Port that signaled the error. If the - * platform retained control of AER, it is responsible for clearing - * this status. In that case, the signaling device may not even be - * visible to the OS. + * If we have native control of AER, clear error status in the device + * that detected the error. If the platform retained control of AER, + * it is responsible for clearing this status. In that case, the + * signaling device may not even be visible to the OS. */ if (host->native_aer || pcie_ports_native) { - pcie_clear_device_status(bridge); - pci_aer_clear_nonfatal_status(bridge); + pcie_clear_device_status(dev); + pci_aer_clear_nonfatal_status(dev); } pci_info(bridge, "device recovery successful\n"); return status; diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h index af7cf237432a..2ff5724b8f13 100644 --- a/drivers/pci/pcie/portdrv.h +++ b/drivers/pci/pcie/portdrv.h @@ -53,12 +53,6 @@ int pcie_dpc_init(void); static inline int pcie_dpc_init(void) { return 0; } #endif -#ifdef CONFIG_PCIE_BW -int pcie_bandwidth_notification_init(void); -#else -static inline int pcie_bandwidth_notification_init(void) { return 0; } -#endif - /* Port Type */ #define PCIE_ANY_PORT (~0) diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c index 0b250bc5f405..c7ff1eea225a 100644 --- a/drivers/pci/pcie/portdrv_pci.c +++ b/drivers/pci/pcie/portdrv_pci.c @@ -153,7 +153,8 @@ static void pcie_portdrv_remove(struct pci_dev *dev) static pci_ers_result_t pcie_portdrv_error_detected(struct pci_dev *dev, pci_channel_state_t error) { - /* Root Port has no impact. Always recovers. */ + if (error == pci_channel_io_frozen) + return PCI_ERS_RESULT_NEED_RESET; return PCI_ERS_RESULT_CAN_RECOVER; } @@ -255,7 +256,6 @@ static void __init pcie_init_services(void) pcie_pme_init(); pcie_dpc_init(); pcie_hp_init(); - pcie_bandwidth_notification_init(); } static int __init pcie_portdrv_init(void) diff --git a/drivers/pci/search.c b/drivers/pci/search.c index 2061672954ee..b4c138a6ec02 100644 --- a/drivers/pci/search.c +++ b/drivers/pci/search.c @@ -168,7 +168,6 @@ struct pci_bus *pci_find_next_bus(const struct pci_bus *from) struct list_head *n; struct pci_bus *b = NULL; - WARN_ON(in_interrupt()); down_read(&pci_bus_sem); n = from ? from->node.next : pci_root_buses.next; if (n != &pci_root_buses) @@ -196,7 +195,6 @@ struct pci_dev *pci_get_slot(struct pci_bus *bus, unsigned int devfn) { struct pci_dev *dev; - WARN_ON(in_interrupt()); down_read(&pci_bus_sem); list_for_each_entry(dev, &bus->devices, bus_list) { @@ -274,7 +272,6 @@ static struct pci_dev *pci_get_dev_by_id(const struct pci_device_id *id, struct device *dev_start = NULL; struct pci_dev *pdev = NULL; - WARN_ON(in_interrupt()); if (from) dev_start = &from->dev; dev = bus_find_device(&pci_bus_type, dev_start, (void *)id, @@ -381,7 +378,6 @@ int pci_dev_present(const struct pci_device_id *ids) { struct pci_dev *found = NULL; - WARN_ON(in_interrupt()); while (ids->vendor || ids->subvendor || ids->class_mask) { found = pci_get_dev_by_id(ids, NULL); if (found) { diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index 43eda101fcf4..7f1acb3918d0 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -410,10 +410,16 @@ EXPORT_SYMBOL(pci_release_resource); int pci_resize_resource(struct pci_dev *dev, int resno, int size) { struct resource *res = dev->resource + resno; + struct pci_host_bridge *host; int old, ret; u32 sizes; u16 cmd; + /* Check if we must preserve the firmware's resource assignment */ + host = pci_find_host_bridge(dev->bus); + if (host->preserve_config) + return -ENOTSUPP; + /* Make sure the resource isn't assigned before resizing it. */ if (!(res->flags & IORESOURCE_UNSET)) return -EBUSY; diff --git a/drivers/pci/syscall.c b/drivers/pci/syscall.c index 31e39558d49d..8b003c890b87 100644 --- a/drivers/pci/syscall.c +++ b/drivers/pci/syscall.c @@ -20,7 +20,7 @@ SYSCALL_DEFINE5(pciconfig_read, unsigned long, bus, unsigned long, dfn, u16 word; u32 dword; long err; - long cfg_ret; + int cfg_ret; if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -46,7 +46,7 @@ SYSCALL_DEFINE5(pciconfig_read, unsigned long, bus, unsigned long, dfn, } err = -EIO; - if (cfg_ret != PCIBIOS_SUCCESSFUL) + if (cfg_ret) goto error; switch (len) { @@ -105,7 +105,7 @@ SYSCALL_DEFINE5(pciconfig_write, unsigned long, bus, unsigned long, dfn, if (err) break; err = pci_user_write_config_byte(dev, off, byte); - if (err != PCIBIOS_SUCCESSFUL) + if (err) err = -EIO; break; @@ -114,7 +114,7 @@ SYSCALL_DEFINE5(pciconfig_write, unsigned long, bus, unsigned long, dfn, if (err) break; err = pci_user_write_config_word(dev, off, word); - if (err != PCIBIOS_SUCCESSFUL) + if (err) err = -EIO; break; @@ -123,7 +123,7 @@ SYSCALL_DEFINE5(pciconfig_write, unsigned long, bus, unsigned long, dfn, if (err) break; err = pci_user_write_config_dword(dev, off, dword); - if (err != PCIBIOS_SUCCESSFUL) + if (err) err = -EIO; break; diff --git a/drivers/pcmcia/cistpl.c b/drivers/pcmcia/cistpl.c index cf109d9a1112..e6939103991b 100644 --- a/drivers/pcmcia/cistpl.c +++ b/drivers/pcmcia/cistpl.c @@ -1554,7 +1554,7 @@ static ssize_t pccard_show_cis(struct file *filp, struct kobject *kobj, if (off + count > size) count = size - off; - s = to_socket(container_of(kobj, struct device, kobj)); + s = to_socket(kobj_to_dev(kobj)); if (!(s->state & SOCKET_PRESENT)) return -ENODEV; @@ -1581,7 +1581,7 @@ static ssize_t pccard_store_cis(struct file *filp, struct kobject *kobj, if (error) return error; - s = to_socket(container_of(kobj, struct device, kobj)); + s = to_socket(kobj_to_dev(kobj)); if (off) return -EINVAL; diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig index 00dabe5fab8a..68d9c2f6a5ca 100644 --- a/drivers/phy/Kconfig +++ b/drivers/phy/Kconfig @@ -52,6 +52,7 @@ config PHY_XGENE config USB_LGM_PHY tristate "INTEL Lightning Mountain USB PHY Driver" depends on USB_SUPPORT + depends on X86 || COMPILE_TEST select USB_PHY select REGULATOR select REGULATOR_FIXED_VOLTAGE diff --git a/drivers/phy/broadcom/Kconfig b/drivers/phy/broadcom/Kconfig index a1f1a9c90d0d..09256339bd04 100644 --- a/drivers/phy/broadcom/Kconfig +++ b/drivers/phy/broadcom/Kconfig @@ -91,10 +91,11 @@ config PHY_BRCM_SATA config PHY_BRCM_USB tristate "Broadcom STB USB PHY driver" - depends on ARCH_BRCMSTB || COMPILE_TEST + depends on ARCH_BCM4908 || ARCH_BRCMSTB || COMPILE_TEST depends on OF select GENERIC_PHY select SOC_BRCMSTB + default ARCH_BCM4908 default ARCH_BRCMSTB help Enable this to support the Broadcom STB USB PHY. diff --git a/drivers/phy/broadcom/phy-brcm-sata.c b/drivers/phy/broadcom/phy-brcm-sata.c index 3ecf41359591..769c707d9b71 100644 --- a/drivers/phy/broadcom/phy-brcm-sata.c +++ b/drivers/phy/broadcom/phy-brcm-sata.c @@ -651,7 +651,7 @@ static int brcm_dsl_sata_init(struct brcm_sata_port *port) break; msleep(20); try--; - }; + } if (!try) { /* PLL did not lock; give up */ diff --git a/drivers/phy/broadcom/phy-brcm-usb.c b/drivers/phy/broadcom/phy-brcm-usb.c index 99fbc7e4138b..116fb23aebd9 100644 --- a/drivers/phy/broadcom/phy-brcm-usb.c +++ b/drivers/phy/broadcom/phy-brcm-usb.c @@ -11,6 +11,7 @@ #include <linux/io.h> #include <linux/module.h> #include <linux/of.h> +#include <linux/of_device.h> #include <linux/phy/phy.h> #include <linux/platform_device.h> #include <linux/interrupt.h> @@ -34,7 +35,7 @@ struct value_to_name_map { }; struct match_chip_info { - void *init_func; + void (*init_func)(struct brcm_usb_init_params *params); u8 required_regs[BRCM_REGS_MAX + 1]; u8 optional_reg; }; @@ -286,6 +287,10 @@ static const struct match_chip_info chip_info_7445 = { static const struct of_device_id brcm_usb_dt_ids[] = { { + .compatible = "brcm,bcm4908-usb-phy", + .data = &chip_info_7445, + }, + { .compatible = "brcm,bcm7216-usb-phy", .data = &chip_info_7216, }, @@ -427,8 +432,6 @@ static int brcm_usb_phy_probe(struct platform_device *pdev) struct device_node *dn = pdev->dev.of_node; int err; const char *mode; - const struct of_device_id *match; - void (*dvr_init)(struct brcm_usb_init_params *params); const struct match_chip_info *info; struct regmap *rmap; int x; @@ -441,10 +444,11 @@ static int brcm_usb_phy_probe(struct platform_device *pdev) priv->ini.family_id = brcmstb_get_family_id(); priv->ini.product_id = brcmstb_get_product_id(); - match = of_match_node(brcm_usb_dt_ids, dev->of_node); - info = match->data; - dvr_init = info->init_func; - (*dvr_init)(&priv->ini); + info = of_device_get_match_data(&pdev->dev); + if (!info) + return -ENOENT; + + info->init_func(&priv->ini); dev_dbg(dev, "Best mapping table is for %s\n", priv->ini.family_name); diff --git a/drivers/phy/cadence/phy-cadence-torrent.c b/drivers/phy/cadence/phy-cadence-torrent.c index f310e15d94cb..591a15834b48 100644 --- a/drivers/phy/cadence/phy-cadence-torrent.c +++ b/drivers/phy/cadence/phy-cadence-torrent.c @@ -2298,6 +2298,7 @@ static int cdns_torrent_phy_probe(struct platform_device *pdev) if (total_num_lanes > MAX_NUM_LANES) { dev_err(dev, "Invalid lane configuration\n"); + ret = -EINVAL; goto put_lnk_rst; } diff --git a/drivers/phy/ingenic/phy-ingenic-usb.c b/drivers/phy/ingenic/phy-ingenic-usb.c index 4d1587d82286..ea127b177f46 100644 --- a/drivers/phy/ingenic/phy-ingenic-usb.c +++ b/drivers/phy/ingenic/phy-ingenic-usb.c @@ -82,18 +82,7 @@ #define USBPCR1_PORT_RST BIT(21) #define USBPCR1_WORD_IF_16BIT BIT(19) -enum ingenic_usb_phy_version { - ID_JZ4770, - ID_JZ4775, - ID_JZ4780, - ID_X1000, - ID_X1830, - ID_X2000, -}; - struct ingenic_soc_info { - enum ingenic_usb_phy_version version; - void (*usb_phy_init)(struct phy *phy); }; @@ -300,38 +289,26 @@ static void x2000_usb_phy_init(struct phy *phy) } static const struct ingenic_soc_info jz4770_soc_info = { - .version = ID_JZ4770, - .usb_phy_init = jz4770_usb_phy_init, }; static const struct ingenic_soc_info jz4775_soc_info = { - .version = ID_JZ4775, - .usb_phy_init = jz4775_usb_phy_init, }; static const struct ingenic_soc_info jz4780_soc_info = { - .version = ID_JZ4780, - .usb_phy_init = jz4780_usb_phy_init, }; static const struct ingenic_soc_info x1000_soc_info = { - .version = ID_X1000, - .usb_phy_init = x1000_usb_phy_init, }; static const struct ingenic_soc_info x1830_soc_info = { - .version = ID_X1830, - .usb_phy_init = x1830_usb_phy_init, }; static const struct ingenic_soc_info x2000_soc_info = { - .version = ID_X2000, - .usb_phy_init = x2000_usb_phy_init, }; diff --git a/drivers/phy/lantiq/phy-lantiq-rcu-usb2.c b/drivers/phy/lantiq/phy-lantiq-rcu-usb2.c index a7d126192cf1..29d246ea24b4 100644 --- a/drivers/phy/lantiq/phy-lantiq-rcu-usb2.c +++ b/drivers/phy/lantiq/phy-lantiq-rcu-usb2.c @@ -124,8 +124,16 @@ static int ltq_rcu_usb2_phy_power_on(struct phy *phy) reset_control_deassert(priv->phy_reset); ret = clk_prepare_enable(priv->phy_gate_clk); - if (ret) + if (ret) { dev_err(dev, "failed to enable PHY gate\n"); + return ret; + } + + /* + * at least the xrx200 usb2 phy requires some extra time to be + * operational after enabling the clock + */ + usleep_range(100, 200); return ret; } diff --git a/drivers/phy/mediatek/phy-mtk-hdmi.c b/drivers/phy/mediatek/phy-mtk-hdmi.c index 45be8aa724f3..8313bd517e4c 100644 --- a/drivers/phy/mediatek/phy-mtk-hdmi.c +++ b/drivers/phy/mediatek/phy-mtk-hdmi.c @@ -201,6 +201,7 @@ static const struct of_device_id mtk_hdmi_phy_match[] = { }, {}, }; +MODULE_DEVICE_TABLE(of, mtk_hdmi_phy_match); static struct platform_driver mtk_hdmi_phy_driver = { .probe = mtk_hdmi_phy_probe, diff --git a/drivers/phy/mediatek/phy-mtk-mipi-dsi.c b/drivers/phy/mediatek/phy-mtk-mipi-dsi.c index 18c481251f04..c51114d8e437 100644 --- a/drivers/phy/mediatek/phy-mtk-mipi-dsi.c +++ b/drivers/phy/mediatek/phy-mtk-mipi-dsi.c @@ -233,8 +233,9 @@ static const struct of_device_id mtk_mipi_tx_match[] = { .data = &mt8183_mipitx_data }, { }, }; +MODULE_DEVICE_TABLE(of, mtk_mipi_tx_match); -struct platform_driver mtk_mipi_tx_driver = { +static struct platform_driver mtk_mipi_tx_driver = { .probe = mtk_mipi_tx_probe, .remove = mtk_mipi_tx_remove, .driver = { diff --git a/drivers/phy/motorola/phy-cpcap-usb.c b/drivers/phy/motorola/phy-cpcap-usb.c index 4728e2bff662..6ee478bc5211 100644 --- a/drivers/phy/motorola/phy-cpcap-usb.c +++ b/drivers/phy/motorola/phy-cpcap-usb.c @@ -143,7 +143,7 @@ static bool cpcap_usb_vbus_valid(struct cpcap_phy_ddata *ddata) error = iio_read_channel_processed(ddata->vbus, &value); if (error >= 0) - return value > 3900 ? true : false; + return value > 3900; dev_err(ddata->dev, "error reading VBUS: %i\n", error); diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c index 0939a9e9d448..9cdebe7f26cb 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp.c @@ -212,6 +212,15 @@ static const unsigned int qmp_v4_usb3_uniphy_regs_layout[QPHY_LAYOUT_SIZE] = { [QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR] = 0x614, }; +static const unsigned int sm8350_usb3_uniphy_regs_layout[QPHY_LAYOUT_SIZE] = { + [QPHY_SW_RESET] = 0x00, + [QPHY_START_CTRL] = 0x44, + [QPHY_PCS_STATUS] = 0x14, + [QPHY_PCS_POWER_DOWN_CONTROL] = 0x40, + [QPHY_PCS_AUTONOMOUS_MODE_CTRL] = 0x1008, + [QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR] = 0x1014, +}; + static const unsigned int sdm845_ufsphy_regs_layout[QPHY_LAYOUT_SIZE] = { [QPHY_START_CTRL] = 0x00, [QPHY_PCS_READY_STATUS] = 0x160, @@ -1974,6 +1983,291 @@ static const struct qmp_phy_init_tbl sm8250_qmp_gen3x2_pcie_pcs_misc_tbl[] = { QMP_PHY_INIT_CFG(QPHY_V4_PCS_PCIE_POWER_STATE_CONFIG4, 0x07), }; +static const struct qmp_phy_init_tbl sdx55_usb3_uniphy_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V4_TX_RCV_DETECT_LVL_2, 0x12), + QMP_PHY_INIT_CFG(QSERDES_V4_TX_LANE_MODE_1, 0xd5), + QMP_PHY_INIT_CFG(QSERDES_V4_TX_LANE_MODE_2, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V4_TX_PI_QEC_CTRL, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V4_TX_RES_CODE_LANE_OFFSET_TX, 0x08), +}; + +static const struct qmp_phy_init_tbl sdx55_usb3_uniphy_rx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_HIGH4, 0x26), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_HIGH3, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_HIGH2, 0xbf), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_HIGH, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_LOW, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_HIGH4, 0xb4), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_HIGH3, 0x7b), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_HIGH2, 0x5c), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_HIGH, 0xdc), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_LOW, 0xdc), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_PI_CONTROLS, 0x99), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SB2_THRESH1, 0x048), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SB2_THRESH2, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SB2_GAIN1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SB2_GAIN2, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_FASTLOCK_FO_GAIN, 0x2f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_FASTLOCK_COUNT_LOW, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_FASTLOCK_COUNT_HIGH, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_FO_GAIN, 0x09), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_VGA_CAL_CNTRL1, 0x54), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_VGA_CAL_CNTRL2, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_EQU_ADAPTOR_CNTRL3, 0x4a), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_EQU_ADAPTOR_CNTRL4, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_DFE_EN_TIMER, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x47), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_SIGDET_CNTRL, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_SIGDET_DEGLITCH_CNTRL, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_IDAC_TSETTLE_HIGH, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_IDAC_TSETTLE_LOW, 0xc0), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_DFE_CTLE_POST_CAL_OFFSET, 0x38), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SO_GAIN, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_DCC_CTRL1, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_GM_CAL, 0x1f), +}; + +static const struct qmp_phy_init_tbl sm8350_ufsphy_serdes_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SYSCLK_EN_SEL, 0xd9), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_HSCLK_SEL, 0x11), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_HSCLK_HS_SWITCH_SEL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP_EN, 0x42), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE_MAP, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_IVCO, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE_INITVAL2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_HSCLK_SEL, 0x11), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DEC_START_MODE0, 0x82), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CP_CTRL_MODE0, 0x14), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_RCTRL_MODE0, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_CCTRL_MODE0, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP1_MODE0, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP2_MODE0, 0x19), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE1_MODE0, 0xac), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE2_MODE0, 0x1e), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DEC_START_MODE1, 0x98), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CP_CTRL_MODE1, 0x14), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_RCTRL_MODE1, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_CCTRL_MODE1, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP1_MODE1, 0x65), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP2_MODE1, 0x1e), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE1_MODE1, 0xdd), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE2_MODE1, 0x23), + + /* Rate B */ + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE_MAP, 0x06), +}; + +static const struct qmp_phy_init_tbl sm8350_ufsphy_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PWM_GEAR_1_DIVIDER_BAND0_1, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PWM_GEAR_2_DIVIDER_BAND0_1, 0x03), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PWM_GEAR_3_DIVIDER_BAND0_1, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PWM_GEAR_4_DIVIDER_BAND0_1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_1, 0xf5), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_3, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_TX, 0x09), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_RX, 0x09), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_TRAN_DRVR_EMP_EN, 0x0c), +}; + +static const struct qmp_phy_init_tbl sm8350_ufsphy_rx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_LVL, 0x24), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_CNTRL, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_DEGLITCH_CNTRL, 0x1e), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_BAND, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_FO_GAIN, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x5a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_PI_CONTROLS, 0xf1), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_LOW, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_PI_CTRL2, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FO_GAIN, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SO_GAIN, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_TERM_BW, 0x1b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL1, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL2, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL3, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL4, 0x1a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x17), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_IDAC_MEASURE_TIME, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_IDAC_TSETTLE_LOW, 0xc0), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_IDAC_TSETTLE_HIGH, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_LOW, 0x6d), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH, 0x6d), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH2, 0xed), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH3, 0x3b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH4, 0x3c), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_LOW, 0xe0), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH, 0xc8), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH2, 0xc8), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH3, 0x3b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH4, 0xb7), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_10_LOW, 0xe0), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_10_HIGH, 0xc8), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_10_HIGH2, 0xc8), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_10_HIGH3, 0x3b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_10_HIGH4, 0xb7), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_DCC_CTRL1, 0x0c), +}; + +static const struct qmp_phy_init_tbl sm8350_ufsphy_pcs_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_RX_SIGDET_CTRL2, 0x6d), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TX_LARGE_AMP_DRV_LVL, 0x0a), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TX_SMALL_AMP_DRV_LVL, 0x02), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TX_MID_TERM_CTRL1, 0x43), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_DEBUG_BUS_CLKSEL, 0x1f), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_RX_MIN_HIBERN8_TIME, 0xff), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_PLL_CNTL, 0x03), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TIMER_20US_CORECLK_STEPS_MSB, 0x16), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TIMER_20US_CORECLK_STEPS_LSB, 0xd8), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TX_PWM_GEAR_BAND, 0xaa), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TX_HS_GEAR_BAND, 0x06), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x03), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x03), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_RX_SIGDET_CTRL1, 0x0e), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_MULTI_LANE_CTRL1, 0x02), +}; + +static const struct qmp_phy_init_tbl sm8350_usb3_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_TX, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_RX, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_TX, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_RX, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_1, 0x35), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_3, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_4, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_5, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RCV_DETECT_LVL_2, 0x12), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PI_QEC_CTRL, 0x21), +}; + +static const struct qmp_phy_init_tbl sm8350_usb3_rx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FO_GAIN, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SO_GAIN, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_FO_GAIN, 0x2f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_LOW, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_HIGH, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_PI_CONTROLS, 0x99), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_THRESH1, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_THRESH2, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_GAIN1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_GAIN2, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_VGA_CAL_CNTRL1, 0x54), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_VGA_CAL_CNTRL2, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL3, 0x4a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL4, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_IDAC_TSETTLE_LOW, 0xc0), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_IDAC_TSETTLE_HIGH, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x47), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_CNTRL, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_DEGLITCH_CNTRL, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_LOW, 0xbb), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH, 0x7b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH2, 0xbb), + QMP_PHY_INIT_CFG_LANE(QSERDES_V5_RX_RX_MODE_00_HIGH3, 0x3d, 1), + QMP_PHY_INIT_CFG_LANE(QSERDES_V5_RX_RX_MODE_00_HIGH3, 0x3c, 2), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH4, 0xdb), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_LOW, 0x64), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH, 0x24), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH2, 0xd2), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH3, 0x13), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH4, 0xa9), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_DFE_EN_TIMER, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_DFE_CTLE_POST_CAL_OFFSET, 0x38), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_AUX_DATA_TCOARSE_TFINE, 0xa0), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_DCC_CTRL1, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_GM_CAL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_VTH_CODE, 0x10), +}; + +static const struct qmp_phy_init_tbl sm8350_usb3_pcs_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_RCVR_DTCT_DLY_U3_L, 0x40), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_RCVR_DTCT_DLY_U3_H, 0x00), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RCVR_DTCT_DLY_P1U2_L, 0xe7), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RCVR_DTCT_DLY_P1U2_H, 0x03), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG1, 0xd0), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG2, 0x07), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG3, 0x20), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG6, 0x13), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_REFGEN_REQ_CONFIG1, 0x21), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RX_SIGDET_LVL, 0xaa), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_CDR_RESET_TIME, 0x0a), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_ALIGN_DETECT_CONFIG1, 0x88), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_ALIGN_DETECT_CONFIG2, 0x13), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_PCS_TX_RX_CONFIG, 0x0c), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_EQ_CONFIG1, 0x4b), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_EQ_CONFIG5, 0x10), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_LFPS_DET_HIGH_COUNT_VAL, 0xf8), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_RXEQTRAINING_DFE_TIME_S2, 0x07), +}; + +static const struct qmp_phy_init_tbl sm8350_usb3_uniphy_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_1, 0xa5), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_2, 0x82), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_3, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_4, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PI_QEC_CTRL, 0x21), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_TX, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_RX, 0x0e), +}; + +static const struct qmp_phy_init_tbl sm8350_usb3_uniphy_rx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH4, 0xdc), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH3, 0xbd), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH2, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_LOW, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH4, 0xa9), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH3, 0x7b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH2, 0xe4), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH, 0x24), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_LOW, 0x64), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_PI_CONTROLS, 0x99), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_THRESH1, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_THRESH2, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_GAIN1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_GAIN2, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_FO_GAIN, 0x2f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_LOW, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_HIGH, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FO_GAIN, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_VGA_CAL_CNTRL1, 0x54), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_VGA_CAL_CNTRL2, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL4, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x47), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_CNTRL, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_DEGLITCH_CNTRL, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_DFE_CTLE_POST_CAL_OFFSET, 0x38), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SO_GAIN, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_GM_CAL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_ENABLES, 0x00), +}; + +static const struct qmp_phy_init_tbl sm8350_usb3_uniphy_pcs_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG1, 0xd0), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG2, 0x07), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG3, 0x20), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG6, 0x13), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RCVR_DTCT_DLY_P1U2_L, 0xe7), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RCVR_DTCT_DLY_P1U2_H, 0x03), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RX_SIGDET_LVL, 0xaa), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_PCS_TX_RX_CONFIG, 0x0c), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_UNI_RXEQTRAINING_DFE_TIME_S2, 0x07), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_UNI_LFPS_DET_HIGH_COUNT_VAL, 0xf8), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_CDR_RESET_TIME, 0x0a), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_ALIGN_DETECT_CONFIG1, 0x88), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_ALIGN_DETECT_CONFIG2, 0x13), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_EQ_CONFIG1, 0x4b), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_EQ_CONFIG5, 0x10), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_REFGEN_REQ_CONFIG1, 0x21), +}; + /* struct qmp_phy_cfg - per-PHY initialization config */ struct qmp_phy_cfg { /* phy-type - PCIE/UFS/USB */ @@ -2183,6 +2477,11 @@ static const char * const sdm845_ufs_phy_clk_l[] = { "ref", "ref_aux", }; +/* usb3 phy on sdx55 doesn't have com_aux clock */ +static const char * const qmp_v4_sdx55_usbphy_clk_l[] = { + "aux", "cfg_ahb", "ref" +}; + /* list of resets */ static const char * const msm8996_pciephy_reset_l[] = { "phy", "common", "cfg", @@ -2824,6 +3123,117 @@ static const struct qmp_phy_cfg sm8250_usb3_uniphy_cfg = { .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, }; +static const struct qmp_phy_cfg sdx55_usb3_uniphy_cfg = { + .type = PHY_TYPE_USB3, + .nlanes = 1, + + .serdes_tbl = sm8150_usb3_uniphy_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_uniphy_serdes_tbl), + .tx_tbl = sdx55_usb3_uniphy_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(sdx55_usb3_uniphy_tx_tbl), + .rx_tbl = sdx55_usb3_uniphy_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(sdx55_usb3_uniphy_rx_tbl), + .pcs_tbl = sm8250_usb3_uniphy_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(sm8250_usb3_uniphy_pcs_tbl), + .clk_list = qmp_v4_sdx55_usbphy_clk_l, + .num_clks = ARRAY_SIZE(qmp_v4_sdx55_usbphy_clk_l), + .reset_list = msm8996_usb3phy_reset_l, + .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = qmp_v4_usb3_uniphy_regs_layout, + + .start_ctrl = SERDES_START | PCS_START, + .pwrdn_ctrl = SW_PWRDN, + + .has_pwrdn_delay = true, + .pwrdn_delay_min = POWER_DOWN_DELAY_US_MIN, + .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, +}; + +static const struct qmp_phy_cfg sm8350_ufsphy_cfg = { + .type = PHY_TYPE_UFS, + .nlanes = 2, + + .serdes_tbl = sm8350_ufsphy_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(sm8350_ufsphy_serdes_tbl), + .tx_tbl = sm8350_ufsphy_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(sm8350_ufsphy_tx_tbl), + .rx_tbl = sm8350_ufsphy_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(sm8350_ufsphy_rx_tbl), + .pcs_tbl = sm8350_ufsphy_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(sm8350_ufsphy_pcs_tbl), + .clk_list = sdm845_ufs_phy_clk_l, + .num_clks = ARRAY_SIZE(sdm845_ufs_phy_clk_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = sm8150_ufsphy_regs_layout, + + .start_ctrl = SERDES_START, + .pwrdn_ctrl = SW_PWRDN, + + .is_dual_lane_phy = true, +}; + +static const struct qmp_phy_cfg sm8350_usb3phy_cfg = { + .type = PHY_TYPE_USB3, + .nlanes = 1, + + .serdes_tbl = sm8150_usb3_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_serdes_tbl), + .tx_tbl = sm8350_usb3_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(sm8350_usb3_tx_tbl), + .rx_tbl = sm8350_usb3_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(sm8350_usb3_rx_tbl), + .pcs_tbl = sm8350_usb3_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(sm8350_usb3_pcs_tbl), + .clk_list = qmp_v4_sm8250_usbphy_clk_l, + .num_clks = ARRAY_SIZE(qmp_v4_sm8250_usbphy_clk_l), + .reset_list = msm8996_usb3phy_reset_l, + .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = qmp_v4_usb3phy_regs_layout, + + .start_ctrl = SERDES_START | PCS_START, + .pwrdn_ctrl = SW_PWRDN, + + .has_pwrdn_delay = true, + .pwrdn_delay_min = POWER_DOWN_DELAY_US_MIN, + .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, + + .has_phy_dp_com_ctrl = true, + .is_dual_lane_phy = true, +}; + +static const struct qmp_phy_cfg sm8350_usb3_uniphy_cfg = { + .type = PHY_TYPE_USB3, + .nlanes = 1, + + .serdes_tbl = sm8150_usb3_uniphy_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_uniphy_serdes_tbl), + .tx_tbl = sm8350_usb3_uniphy_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(sm8350_usb3_uniphy_tx_tbl), + .rx_tbl = sm8350_usb3_uniphy_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(sm8350_usb3_uniphy_rx_tbl), + .pcs_tbl = sm8350_usb3_uniphy_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(sm8350_usb3_uniphy_pcs_tbl), + .clk_list = qmp_v4_phy_clk_l, + .num_clks = ARRAY_SIZE(qmp_v4_phy_clk_l), + .reset_list = msm8996_usb3phy_reset_l, + .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = sm8350_usb3_uniphy_regs_layout, + + .start_ctrl = SERDES_START | PCS_START, + .pwrdn_ctrl = SW_PWRDN, + + .has_pwrdn_delay = true, + .pwrdn_delay_min = POWER_DOWN_DELAY_US_MIN, + .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, +}; + static void qcom_qmp_phy_configure_lane(void __iomem *base, const unsigned int *regs, const struct qmp_phy_init_tbl tbl[], @@ -3135,7 +3545,7 @@ static int qcom_qmp_phy_configure_dp_phy(struct qmp_phy *qphy) static int qcom_qmp_dp_phy_calibrate(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); - const u8 cfg1_settings[] = { 0x13, 0x23, 0x1d }; + static const u8 cfg1_settings[] = { 0x13, 0x23, 0x1d }; u8 val; qphy->dp_aux_cfg++; @@ -4129,6 +4539,12 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = { .compatible = "qcom,sc7180-qmp-usb3-dp-phy", /* It's a combo phy */ }, { + .compatible = "qcom,sc8180x-qmp-ufs-phy", + .data = &sm8150_ufsphy_cfg, + }, { + .compatible = "qcom,sc8180x-qmp-usb3-phy", + .data = &sm8150_usb3phy_cfg, + }, { .compatible = "qcom,sdm845-qhp-pcie-phy", .data = &sdm845_qhp_pciephy_cfg, }, { @@ -4171,8 +4587,20 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = { .compatible = "qcom,sm8250-qmp-gen3x2-pcie-phy", .data = &sm8250_qmp_gen3x2_pciephy_cfg, }, { + .compatible = "qcom,sm8350-qmp-ufs-phy", + .data = &sm8350_ufsphy_cfg, + }, { .compatible = "qcom,sm8250-qmp-modem-pcie-phy", .data = &sm8250_qmp_gen3x2_pciephy_cfg, + }, { + .compatible = "qcom,sdx55-qmp-usb3-uni-phy", + .data = &sdx55_usb3_uniphy_cfg, + }, { + .compatible = "qcom,sm8350-qmp-usb3-phy", + .data = &sm8350_usb3phy_cfg, + }, { + .compatible = "qcom,sm8350-qmp-usb3-uni-phy", + .data = &sm8350_usb3_uniphy_cfg, }, { }, }; diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.h b/drivers/phy/qualcomm/phy-qcom-qmp.h index db92a461dd2e..71ce3aa174ae 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.h +++ b/drivers/phy/qualcomm/phy-qcom-qmp.h @@ -824,4 +824,151 @@ #define QPHY_V4_PCS_PCIE_PRESET_P10_PRE 0xbc #define QPHY_V4_PCS_PCIE_PRESET_P10_POST 0xe0 +/* Only for QMP V5 PHY - QSERDES COM registers */ +#define QSERDES_V5_COM_PLL_IVCO 0x058 +#define QSERDES_V5_COM_CP_CTRL_MODE0 0x074 +#define QSERDES_V5_COM_CP_CTRL_MODE1 0x078 +#define QSERDES_V5_COM_PLL_RCTRL_MODE0 0x07c +#define QSERDES_V5_COM_PLL_RCTRL_MODE1 0x080 +#define QSERDES_V5_COM_PLL_CCTRL_MODE0 0x084 +#define QSERDES_V5_COM_PLL_CCTRL_MODE1 0x088 +#define QSERDES_V5_COM_SYSCLK_EN_SEL 0x094 +#define QSERDES_V5_COM_LOCK_CMP_EN 0x0a4 +#define QSERDES_V5_COM_LOCK_CMP1_MODE0 0x0ac +#define QSERDES_V5_COM_LOCK_CMP2_MODE0 0x0b0 +#define QSERDES_V5_COM_LOCK_CMP1_MODE1 0x0b4 +#define QSERDES_V5_COM_DEC_START_MODE0 0x0bc +#define QSERDES_V5_COM_LOCK_CMP2_MODE1 0x0b8 +#define QSERDES_V5_COM_DEC_START_MODE1 0x0c4 +#define QSERDES_V5_COM_VCO_TUNE_MAP 0x10c +#define QSERDES_V5_COM_VCO_TUNE_INITVAL2 0x124 +#define QSERDES_V5_COM_HSCLK_SEL 0x158 +#define QSERDES_V5_COM_HSCLK_HS_SWITCH_SEL 0x15c +#define QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE1_MODE0 0x1ac +#define QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE2_MODE0 0x1b0 +#define QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE1_MODE1 0x1b4 +#define QSERDES_V5_COM_BIN_VCOCAL_HSCLK_SEL 0x1bc +#define QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE2_MODE1 0x1b8 + +/* Only for QMP V5 PHY - TX registers */ +#define QSERDES_V5_TX_RES_CODE_LANE_TX 0x34 +#define QSERDES_V5_TX_RES_CODE_LANE_RX 0x38 +#define QSERDES_V5_TX_RES_CODE_LANE_OFFSET_TX 0x3c +#define QSERDES_V5_TX_RES_CODE_LANE_OFFSET_RX 0x40 +#define QSERDES_V5_TX_LANE_MODE_1 0x84 +#define QSERDES_V5_TX_LANE_MODE_2 0x88 +#define QSERDES_V5_TX_LANE_MODE_3 0x8c +#define QSERDES_V5_TX_LANE_MODE_4 0x90 +#define QSERDES_V5_TX_LANE_MODE_5 0x94 +#define QSERDES_V5_TX_RCV_DETECT_LVL_2 0xa4 +#define QSERDES_V5_TX_TRAN_DRVR_EMP_EN 0xc0 +#define QSERDES_V5_TX_PI_QEC_CTRL 0xe4 +#define QSERDES_V5_TX_PWM_GEAR_1_DIVIDER_BAND0_1 0x178 +#define QSERDES_V5_TX_PWM_GEAR_2_DIVIDER_BAND0_1 0x17c +#define QSERDES_V5_TX_PWM_GEAR_3_DIVIDER_BAND0_1 0x180 +#define QSERDES_V5_TX_PWM_GEAR_4_DIVIDER_BAND0_1 0x184 + +/* Only for QMP V5 PHY - RX registers */ +#define QSERDES_V5_RX_UCDR_FO_GAIN 0x008 +#define QSERDES_V5_RX_UCDR_SO_GAIN 0x014 +#define QSERDES_V5_RX_UCDR_FASTLOCK_FO_GAIN 0x030 +#define QSERDES_V5_RX_UCDR_SO_SATURATION_AND_ENABLE 0x034 +#define QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_LOW 0x03c +#define QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_HIGH 0x040 +#define QSERDES_V5_RX_UCDR_PI_CONTROLS 0x044 +#define QSERDES_V5_RX_UCDR_PI_CTRL2 0x048 +#define QSERDES_V5_RX_UCDR_SB2_THRESH1 0x04c +#define QSERDES_V5_RX_UCDR_SB2_THRESH2 0x050 +#define QSERDES_V5_RX_UCDR_SB2_GAIN1 0x054 +#define QSERDES_V5_RX_UCDR_SB2_GAIN2 0x058 +#define QSERDES_V5_RX_AUX_DATA_TCOARSE_TFINE 0x060 +#define QSERDES_V5_RX_RCLK_AUXDATA_SEL 0x064 +#define QSERDES_V5_RX_AC_JTAG_ENABLE 0x068 +#define QSERDES_V5_RX_AC_JTAG_MODE 0x078 +#define QSERDES_V5_RX_RX_TERM_BW 0x080 +#define QSERDES_V5_RX_VGA_CAL_CNTRL1 0x0d4 +#define QSERDES_V5_RX_VGA_CAL_CNTRL2 0x0d8 +#define QSERDES_V5_RX_GM_CAL 0x0dc +#define QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL1 0x0e8 +#define QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL2 0x0ec +#define QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL3 0x0f0 +#define QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL4 0x0f4 +#define QSERDES_V5_RX_RX_IDAC_TSETTLE_LOW 0x0f8 +#define QSERDES_V5_RX_RX_IDAC_TSETTLE_HIGH 0x0fc +#define QSERDES_V5_RX_RX_IDAC_MEASURE_TIME 0x100 +#define QSERDES_V5_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1 0x110 +#define QSERDES_V5_RX_RX_OFFSET_ADAPTOR_CNTRL2 0x114 +#define QSERDES_V5_RX_SIGDET_ENABLES 0x118 +#define QSERDES_V5_RX_SIGDET_CNTRL 0x11c +#define QSERDES_V5_RX_SIGDET_LVL 0x120 +#define QSERDES_V5_RX_SIGDET_DEGLITCH_CNTRL 0x124 +#define QSERDES_V5_RX_RX_BAND 0x128 +#define QSERDES_V5_RX_RX_MODE_00_LOW 0x15c +#define QSERDES_V5_RX_RX_MODE_00_HIGH 0x160 +#define QSERDES_V5_RX_RX_MODE_00_HIGH2 0x164 +#define QSERDES_V5_RX_RX_MODE_00_HIGH3 0x168 +#define QSERDES_V5_RX_RX_MODE_00_HIGH4 0x16c +#define QSERDES_V5_RX_RX_MODE_01_LOW 0x170 +#define QSERDES_V5_RX_RX_MODE_01_HIGH 0x174 +#define QSERDES_V5_RX_RX_MODE_01_HIGH2 0x178 +#define QSERDES_V5_RX_RX_MODE_01_HIGH3 0x17c +#define QSERDES_V5_RX_RX_MODE_01_HIGH4 0x180 +#define QSERDES_V5_RX_RX_MODE_10_LOW 0x184 +#define QSERDES_V5_RX_RX_MODE_10_HIGH 0x188 +#define QSERDES_V5_RX_RX_MODE_10_HIGH2 0x18c +#define QSERDES_V5_RX_RX_MODE_10_HIGH3 0x190 +#define QSERDES_V5_RX_RX_MODE_10_HIGH4 0x194 +#define QSERDES_V5_RX_DFE_EN_TIMER 0x1a0 +#define QSERDES_V5_RX_DFE_CTLE_POST_CAL_OFFSET 0x1a4 +#define QSERDES_V5_RX_DCC_CTRL1 0x1a8 +#define QSERDES_V5_RX_VTH_CODE 0x1b0 + +/* Only for QMP V5 PHY - UFS PCS registers */ +#define QPHY_V5_PCS_UFS_TIMER_20US_CORECLK_STEPS_MSB 0x00c +#define QPHY_V5_PCS_UFS_TIMER_20US_CORECLK_STEPS_LSB 0x010 +#define QPHY_V5_PCS_UFS_PLL_CNTL 0x02c +#define QPHY_V5_PCS_UFS_TX_LARGE_AMP_DRV_LVL 0x030 +#define QPHY_V5_PCS_UFS_TX_SMALL_AMP_DRV_LVL 0x038 +#define QPHY_V5_PCS_UFS_TX_HSGEAR_CAPABILITY 0x074 +#define QPHY_V5_PCS_UFS_RX_HSGEAR_CAPABILITY 0x0b4 +#define QPHY_V5_PCS_UFS_DEBUG_BUS_CLKSEL 0x124 +#define QPHY_V5_PCS_UFS_RX_MIN_HIBERN8_TIME 0x150 +#define QPHY_V5_PCS_UFS_RX_SIGDET_CTRL1 0x154 +#define QPHY_V5_PCS_UFS_RX_SIGDET_CTRL2 0x158 +#define QPHY_V5_PCS_UFS_TX_PWM_GEAR_BAND 0x160 +#define QPHY_V5_PCS_UFS_TX_HS_GEAR_BAND 0x168 +#define QPHY_V5_PCS_UFS_TX_MID_TERM_CTRL1 0x1d8 +#define QPHY_V5_PCS_UFS_MULTI_LANE_CTRL1 0x1e0 + +/* Only for QMP V5 PHY - USB3 have different offsets than V4 */ +#define QPHY_V5_PCS_USB3_POWER_STATE_CONFIG1 0x300 +#define QPHY_V5_PCS_USB3_AUTONOMOUS_MODE_STATUS 0x304 +#define QPHY_V5_PCS_USB3_AUTONOMOUS_MODE_CTRL 0x308 +#define QPHY_V5_PCS_USB3_AUTONOMOUS_MODE_CTRL2 0x30c +#define QPHY_V5_PCS_USB3_LFPS_RXTERM_IRQ_SOURCE_STATUS 0x310 +#define QPHY_V5_PCS_USB3_LFPS_RXTERM_IRQ_CLEAR 0x314 +#define QPHY_V5_PCS_USB3_LFPS_DET_HIGH_COUNT_VAL 0x318 +#define QPHY_V5_PCS_USB3_LFPS_TX_ECSTART 0x31c +#define QPHY_V5_PCS_USB3_LFPS_PER_TIMER_VAL 0x320 +#define QPHY_V5_PCS_USB3_LFPS_TX_END_CNT_U3_START 0x324 +#define QPHY_V5_PCS_USB3_LFPS_CONFIG1 0x328 +#define QPHY_V5_PCS_USB3_RXEQTRAINING_LOCK_TIME 0x32c +#define QPHY_V5_PCS_USB3_RXEQTRAINING_WAIT_TIME 0x330 +#define QPHY_V5_PCS_USB3_RXEQTRAINING_CTLE_TIME 0x334 +#define QPHY_V5_PCS_USB3_RXEQTRAINING_WAIT_TIME_S2 0x338 +#define QPHY_V5_PCS_USB3_RXEQTRAINING_DFE_TIME_S2 0x33c +#define QPHY_V5_PCS_USB3_RCVR_DTCT_DLY_U3_L 0x340 +#define QPHY_V5_PCS_USB3_RCVR_DTCT_DLY_U3_H 0x344 +#define QPHY_V5_PCS_USB3_ARCVR_DTCT_EN_PERIOD 0x348 +#define QPHY_V5_PCS_USB3_ARCVR_DTCT_CM_DLY 0x34c +#define QPHY_V5_PCS_USB3_TXONESZEROS_RUN_LENGTH 0x350 +#define QPHY_V5_PCS_USB3_ALFPS_DEGLITCH_VAL 0x354 +#define QPHY_V5_PCS_USB3_SIGDET_STARTUP_TIMER_VAL 0x358 +#define QPHY_V5_PCS_USB3_TEST_CONTROL 0x35c +#define QPHY_V5_PCS_USB3_RXTERMINATION_DLY_SEL 0x360 + +/* Only for QMP V5 PHY - UNI has 0x1000 offset for PCS_USB3 regs */ +#define QPHY_V5_PCS_USB3_UNI_LFPS_DET_HIGH_COUNT_VAL 0x1018 +#define QPHY_V5_PCS_USB3_UNI_RXEQTRAINING_DFE_TIME_S2 0x103c + #endif diff --git a/drivers/phy/qualcomm/phy-qcom-qusb2.c b/drivers/phy/qualcomm/phy-qcom-qusb2.c index 109792203baf..8f1bf7e2186b 100644 --- a/drivers/phy/qualcomm/phy-qcom-qusb2.c +++ b/drivers/phy/qualcomm/phy-qcom-qusb2.c @@ -22,6 +22,7 @@ #include <dt-bindings/phy/phy-qcom-qusb2.h> +#define QUSB2PHY_PLL 0x0 #define QUSB2PHY_PLL_TEST 0x04 #define CLK_REF_SEL BIT(7) @@ -135,6 +136,35 @@ enum qusb2phy_reg_layout { QUSB2PHY_INTR_CTRL, }; +static const struct qusb2_phy_init_tbl ipq6018_init_tbl[] = { + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL, 0x14), + QUSB2_PHY_INIT_CFG_L(QUSB2PHY_PORT_TUNE1, 0xF8), + QUSB2_PHY_INIT_CFG_L(QUSB2PHY_PORT_TUNE2, 0xB3), + QUSB2_PHY_INIT_CFG_L(QUSB2PHY_PORT_TUNE3, 0x83), + QUSB2_PHY_INIT_CFG_L(QUSB2PHY_PORT_TUNE4, 0xC0), + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_TUNE, 0x30), + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_USER_CTL1, 0x79), + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_USER_CTL2, 0x21), + QUSB2_PHY_INIT_CFG_L(QUSB2PHY_PORT_TUNE5, 0x00), + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_PWR_CTRL, 0x00), + QUSB2_PHY_INIT_CFG_L(QUSB2PHY_PORT_TEST2, 0x14), + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_TEST, 0x80), + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_AUTOPGM_CTL1, 0x9F), +}; + +static const unsigned int ipq6018_regs_layout[] = { + [QUSB2PHY_PLL_STATUS] = 0x38, + [QUSB2PHY_PORT_TUNE1] = 0x80, + [QUSB2PHY_PORT_TUNE2] = 0x84, + [QUSB2PHY_PORT_TUNE3] = 0x88, + [QUSB2PHY_PORT_TUNE4] = 0x8C, + [QUSB2PHY_PORT_TUNE5] = 0x90, + [QUSB2PHY_PORT_TEST1] = 0x98, + [QUSB2PHY_PORT_TEST2] = 0x9C, + [QUSB2PHY_PORT_POWERDOWN] = 0xB4, + [QUSB2PHY_INTR_CTRL] = 0xBC, +}; + static const unsigned int msm8996_regs_layout[] = { [QUSB2PHY_PLL_STATUS] = 0x38, [QUSB2PHY_PORT_TUNE1] = 0x80, @@ -245,6 +275,9 @@ struct qusb2_phy_cfg { /* true if PHY has PLL_CORE_INPUT_OVERRIDE register to reset PLL */ bool has_pll_override; + + /* true if PHY default clk scheme is single-ended */ + bool se_clk_scheme_default; }; static const struct qusb2_phy_cfg msm8996_phy_cfg = { @@ -253,6 +286,7 @@ static const struct qusb2_phy_cfg msm8996_phy_cfg = { .regs = msm8996_regs_layout, .has_pll_test = true, + .se_clk_scheme_default = true, .disable_ctrl = (CLAMP_N_EN | FREEZIO_N | POWER_DOWN), .mask_core_ready = PLL_LOCKED, .autoresume_en = BIT(3), @@ -266,10 +300,22 @@ static const struct qusb2_phy_cfg msm8998_phy_cfg = { .disable_ctrl = POWER_DOWN, .mask_core_ready = CORE_READY_STATUS, .has_pll_override = true, + .se_clk_scheme_default = true, .autoresume_en = BIT(0), .update_tune1_with_efuse = true, }; +static const struct qusb2_phy_cfg ipq6018_phy_cfg = { + .tbl = ipq6018_init_tbl, + .tbl_num = ARRAY_SIZE(ipq6018_init_tbl), + .regs = ipq6018_regs_layout, + + .disable_ctrl = POWER_DOWN, + .mask_core_ready = PLL_LOCKED, + /* autoresume not used */ + .autoresume_en = BIT(0), +}; + static const struct qusb2_phy_cfg qusb2_v2_phy_cfg = { .tbl = qusb2_v2_init_tbl, .tbl_num = ARRAY_SIZE(qusb2_v2_init_tbl), @@ -279,10 +325,23 @@ static const struct qusb2_phy_cfg qusb2_v2_phy_cfg = { POWER_DOWN), .mask_core_ready = CORE_READY_STATUS, .has_pll_override = true, + .se_clk_scheme_default = true, .autoresume_en = BIT(0), .update_tune1_with_efuse = true, }; +static const struct qusb2_phy_cfg sdm660_phy_cfg = { + .tbl = msm8996_init_tbl, + .tbl_num = ARRAY_SIZE(msm8996_init_tbl), + .regs = msm8996_regs_layout, + + .has_pll_test = true, + .se_clk_scheme_default = false, + .disable_ctrl = (CLAMP_N_EN | FREEZIO_N | POWER_DOWN), + .mask_core_ready = PLL_LOCKED, + .autoresume_en = BIT(3), +}; + static const char * const qusb2_phy_vreg_names[] = { "vdda-pll", "vdda-phy-dpdm", }; @@ -701,8 +760,13 @@ static int qusb2_phy_init(struct phy *phy) /* Required to get phy pll lock successfully */ usleep_range(150, 160); - /* Default is single-ended clock on msm8996 */ - qphy->has_se_clk_scheme = true; + /* + * Not all the SoCs have got a readable TCSR_PHY_CLK_SCHEME + * register in the TCSR so, if there's none, use the default + * value hardcoded in the configuration. + */ + qphy->has_se_clk_scheme = cfg->se_clk_scheme_default; + /* * read TCSR_PHY_CLK_SCHEME register to check if single-ended * clock scheme is selected. If yes, then disable differential @@ -810,6 +874,9 @@ static const struct phy_ops qusb2_phy_gen_ops = { static const struct of_device_id qusb2_phy_of_match_table[] = { { + .compatible = "qcom,ipq6018-qusb2-phy", + .data = &ipq6018_phy_cfg, + }, { .compatible = "qcom,ipq8074-qusb2-phy", .data = &msm8996_phy_cfg, }, { @@ -819,6 +886,9 @@ static const struct of_device_id qusb2_phy_of_match_table[] = { .compatible = "qcom,msm8998-qusb2-phy", .data = &msm8998_phy_cfg, }, { + .compatible = "qcom,sdm660-qusb2-phy", + .data = &sdm660_phy_cfg, + }, { /* * Deprecated. Only here to support legacy device * trees that didn't include "qcom,qusb2-v2-phy" diff --git a/drivers/phy/qualcomm/phy-qcom-usb-hs-28nm.c b/drivers/phy/qualcomm/phy-qcom-usb-hs-28nm.c index a52a9bf13b75..8807e59a1162 100644 --- a/drivers/phy/qualcomm/phy-qcom-usb-hs-28nm.c +++ b/drivers/phy/qualcomm/phy-qcom-usb-hs-28nm.c @@ -401,13 +401,26 @@ static const struct hsphy_init_seq init_seq_femtophy[] = { HSPHY_INIT_CFG(0x90, 0x60, 0), }; +static const struct hsphy_init_seq init_seq_mdm9607[] = { + HSPHY_INIT_CFG(0x80, 0x44, 0), + HSPHY_INIT_CFG(0x81, 0x38, 0), + HSPHY_INIT_CFG(0x82, 0x24, 0), + HSPHY_INIT_CFG(0x83, 0x13, 0), +}; + static const struct hsphy_data hsphy_data_femtophy = { .init_seq = init_seq_femtophy, .init_seq_num = ARRAY_SIZE(init_seq_femtophy), }; +static const struct hsphy_data hsphy_data_mdm9607 = { + .init_seq = init_seq_mdm9607, + .init_seq_num = ARRAY_SIZE(init_seq_mdm9607), +}; + static const struct of_device_id qcom_snps_hsphy_match[] = { { .compatible = "qcom,usb-hs-28nm-femtophy", .data = &hsphy_data_femtophy, }, + { .compatible = "qcom,usb-hs-28nm-mdm9607", .data = &hsphy_data_mdm9607, }, { }, }; MODULE_DEVICE_TABLE(of, qcom_snps_hsphy_match); diff --git a/drivers/phy/rockchip/phy-rockchip-emmc.c b/drivers/phy/rockchip/phy-rockchip-emmc.c index 1e424f263e7a..20023f6eb994 100644 --- a/drivers/phy/rockchip/phy-rockchip-emmc.c +++ b/drivers/phy/rockchip/phy-rockchip-emmc.c @@ -248,15 +248,17 @@ static int rockchip_emmc_phy_init(struct phy *phy) * - SDHCI driver to get the PHY * - SDHCI driver to init the PHY * - * The clock is optional, so upon any error we just set to NULL. + * The clock is optional, using clk_get_optional() to get the clock + * and do error processing if the return value != NULL * * NOTE: we don't do anything special for EPROBE_DEFER here. Given the * above expected use case, EPROBE_DEFER isn't sensible to expect, so * it's just like any other error. */ - rk_phy->emmcclk = clk_get(&phy->dev, "emmcclk"); + rk_phy->emmcclk = clk_get_optional(&phy->dev, "emmcclk"); if (IS_ERR(rk_phy->emmcclk)) { - dev_dbg(&phy->dev, "Error getting emmcclk: %d\n", ret); + ret = PTR_ERR(rk_phy->emmcclk); + dev_err(&phy->dev, "Error getting emmcclk: %d\n", ret); rk_phy->emmcclk = NULL; } @@ -380,10 +382,10 @@ static int rockchip_emmc_phy_probe(struct platform_device *pdev) if (!of_property_read_u32(dev->of_node, "drive-impedance-ohm", &val)) rk_phy->drive_impedance = convert_drive_impedance_ohm(pdev, val); - if (of_property_read_bool(dev->of_node, "enable-strobe-pulldown")) + if (of_property_read_bool(dev->of_node, "rockchip,enable-strobe-pulldown")) rk_phy->enable_strobe_pulldown = PHYCTRL_REN_STRB_ENABLE; - if (!of_property_read_u32(dev->of_node, "output-tapdelay-select", &val)) { + if (!of_property_read_u32(dev->of_node, "rockchip,output-tapdelay-select", &val)) { if (val <= PHYCTRL_OTAPDLYSEL_MAXVALUE) rk_phy->output_tapdelay_select = val; else diff --git a/drivers/phy/st/phy-stm32-usbphyc.c b/drivers/phy/st/phy-stm32-usbphyc.c index a54317e96c41..d08fbb180e43 100644 --- a/drivers/phy/st/phy-stm32-usbphyc.c +++ b/drivers/phy/st/phy-stm32-usbphyc.c @@ -8,7 +8,7 @@ #include <linux/bitfield.h> #include <linux/clk.h> #include <linux/delay.h> -#include <linux/io.h> +#include <linux/iopoll.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/of_platform.h> @@ -17,6 +17,7 @@ #define STM32_USBPHYC_PLL 0x0 #define STM32_USBPHYC_MISC 0x8 +#define STM32_USBPHYC_MONITOR(X) (0x108 + ((X) * 0x100)) #define STM32_USBPHYC_VERSION 0x3F4 /* STM32_USBPHYC_PLL bit fields */ @@ -32,19 +33,16 @@ /* STM32_USBPHYC_MISC bit fields */ #define SWITHOST BIT(0) +/* STM32_USBPHYC_MONITOR bit fields */ +#define STM32_USBPHYC_MON_OUT GENMASK(3, 0) +#define STM32_USBPHYC_MON_SEL GENMASK(8, 4) +#define STM32_USBPHYC_MON_SEL_LOCKP 0x1F +#define STM32_USBPHYC_MON_OUT_LOCKP BIT(3) + /* STM32_USBPHYC_VERSION bit fields */ #define MINREV GENMASK(3, 0) #define MAJREV GENMASK(7, 4) -static const char * const supplies_names[] = { - "vdda1v1", /* 1V1 */ - "vdda1v8", /* 1V8 */ -}; - -#define NUM_SUPPLIES ARRAY_SIZE(supplies_names) - -#define PLL_LOCK_TIME_US 100 -#define PLL_PWR_DOWN_TIME_US 5 #define PLL_FVCO_MHZ 2880 #define PLL_INFF_MIN_RATE_HZ 19200000 #define PLL_INFF_MAX_RATE_HZ 38400000 @@ -58,7 +56,6 @@ struct pll_params { struct stm32_usbphyc_phy { struct phy *phy; struct stm32_usbphyc *usbphyc; - struct regulator_bulk_data supplies[NUM_SUPPLIES]; u32 index; bool active; }; @@ -70,6 +67,9 @@ struct stm32_usbphyc { struct reset_control *rst; struct stm32_usbphyc_phy **phys; int nphys; + struct regulator *vdda1v1; + struct regulator *vdda1v8; + atomic_t n_pll_cons; int switch_setup; }; @@ -83,6 +83,41 @@ static inline void stm32_usbphyc_clr_bits(void __iomem *reg, u32 bits) writel_relaxed(readl_relaxed(reg) & ~bits, reg); } +static int stm32_usbphyc_regulators_enable(struct stm32_usbphyc *usbphyc) +{ + int ret; + + ret = regulator_enable(usbphyc->vdda1v1); + if (ret) + return ret; + + ret = regulator_enable(usbphyc->vdda1v8); + if (ret) + goto vdda1v1_disable; + + return 0; + +vdda1v1_disable: + regulator_disable(usbphyc->vdda1v1); + + return ret; +} + +static int stm32_usbphyc_regulators_disable(struct stm32_usbphyc *usbphyc) +{ + int ret; + + ret = regulator_disable(usbphyc->vdda1v8); + if (ret) + return ret; + + ret = regulator_disable(usbphyc->vdda1v1); + if (ret) + return ret; + + return 0; +} + static void stm32_usbphyc_get_pll_params(u32 clk_rate, struct pll_params *pll_params) { @@ -142,83 +177,106 @@ static int stm32_usbphyc_pll_init(struct stm32_usbphyc *usbphyc) return 0; } -static bool stm32_usbphyc_has_one_phy_active(struct stm32_usbphyc *usbphyc) +static int __stm32_usbphyc_pll_disable(struct stm32_usbphyc *usbphyc) { - int i; + void __iomem *pll_reg = usbphyc->base + STM32_USBPHYC_PLL; + u32 pllen; + + stm32_usbphyc_clr_bits(pll_reg, PLLEN); - for (i = 0; i < usbphyc->nphys; i++) - if (usbphyc->phys[i]->active) - return true; + /* Wait for minimum width of powerdown pulse (ENABLE = Low) */ + if (readl_relaxed_poll_timeout(pll_reg, pllen, !(pllen & PLLEN), 5, 50)) + dev_err(usbphyc->dev, "PLL not reset\n"); - return false; + return stm32_usbphyc_regulators_disable(usbphyc); +} + +static int stm32_usbphyc_pll_disable(struct stm32_usbphyc *usbphyc) +{ + /* Check if a phy port is still active or clk48 in use */ + if (atomic_dec_return(&usbphyc->n_pll_cons) > 0) + return 0; + + return __stm32_usbphyc_pll_disable(usbphyc); } static int stm32_usbphyc_pll_enable(struct stm32_usbphyc *usbphyc) { void __iomem *pll_reg = usbphyc->base + STM32_USBPHYC_PLL; - bool pllen = (readl_relaxed(pll_reg) & PLLEN); + bool pllen = readl_relaxed(pll_reg) & PLLEN; int ret; - /* Check if one phy port has already configured the pll */ - if (pllen && stm32_usbphyc_has_one_phy_active(usbphyc)) + /* + * Check if a phy port or clk48 prepare has configured the pll + * and ensure the PLL is enabled + */ + if (atomic_inc_return(&usbphyc->n_pll_cons) > 1 && pllen) return 0; if (pllen) { - stm32_usbphyc_clr_bits(pll_reg, PLLEN); - /* Wait for minimum width of powerdown pulse (ENABLE = Low) */ - udelay(PLL_PWR_DOWN_TIME_US); + /* + * PLL shouldn't be enabled without known consumer, + * disable it and reinit n_pll_cons + */ + dev_warn(usbphyc->dev, "PLL enabled without known consumers\n"); + + ret = __stm32_usbphyc_pll_disable(usbphyc); + if (ret) + return ret; } + ret = stm32_usbphyc_regulators_enable(usbphyc); + if (ret) + goto dec_n_pll_cons; + ret = stm32_usbphyc_pll_init(usbphyc); if (ret) - return ret; + goto reg_disable; stm32_usbphyc_set_bits(pll_reg, PLLEN); - /* Wait for maximum lock time */ - udelay(PLL_LOCK_TIME_US); - - if (!(readl_relaxed(pll_reg) & PLLEN)) { - dev_err(usbphyc->dev, "PLLEN not set\n"); - return -EIO; - } - return 0; -} - -static int stm32_usbphyc_pll_disable(struct stm32_usbphyc *usbphyc) -{ - void __iomem *pll_reg = usbphyc->base + STM32_USBPHYC_PLL; - /* Check if other phy port active */ - if (stm32_usbphyc_has_one_phy_active(usbphyc)) - return 0; - - stm32_usbphyc_clr_bits(pll_reg, PLLEN); - /* Wait for minimum width of powerdown pulse (ENABLE = Low) */ - udelay(PLL_PWR_DOWN_TIME_US); +reg_disable: + stm32_usbphyc_regulators_disable(usbphyc); - if (readl_relaxed(pll_reg) & PLLEN) { - dev_err(usbphyc->dev, "PLL not reset\n"); - return -EIO; - } +dec_n_pll_cons: + atomic_dec(&usbphyc->n_pll_cons); - return 0; + return ret; } static int stm32_usbphyc_phy_init(struct phy *phy) { struct stm32_usbphyc_phy *usbphyc_phy = phy_get_drvdata(phy); struct stm32_usbphyc *usbphyc = usbphyc_phy->usbphyc; + u32 reg_mon = STM32_USBPHYC_MONITOR(usbphyc_phy->index); + u32 monsel = FIELD_PREP(STM32_USBPHYC_MON_SEL, + STM32_USBPHYC_MON_SEL_LOCKP); + u32 monout; int ret; ret = stm32_usbphyc_pll_enable(usbphyc); if (ret) return ret; + /* Check that PLL Lock input to PHY is High */ + writel_relaxed(monsel, usbphyc->base + reg_mon); + ret = readl_relaxed_poll_timeout(usbphyc->base + reg_mon, monout, + (monout & STM32_USBPHYC_MON_OUT_LOCKP), + 100, 1000); + if (ret) { + dev_err(usbphyc->dev, "PLL Lock input to PHY is Low (val=%x)\n", + (u32)(monout & STM32_USBPHYC_MON_OUT)); + goto pll_disable; + } + usbphyc_phy->active = true; return 0; + +pll_disable: + return stm32_usbphyc_pll_disable(usbphyc); } static int stm32_usbphyc_phy_exit(struct phy *phy) @@ -231,25 +289,9 @@ static int stm32_usbphyc_phy_exit(struct phy *phy) return stm32_usbphyc_pll_disable(usbphyc); } -static int stm32_usbphyc_phy_power_on(struct phy *phy) -{ - struct stm32_usbphyc_phy *usbphyc_phy = phy_get_drvdata(phy); - - return regulator_bulk_enable(NUM_SUPPLIES, usbphyc_phy->supplies); -} - -static int stm32_usbphyc_phy_power_off(struct phy *phy) -{ - struct stm32_usbphyc_phy *usbphyc_phy = phy_get_drvdata(phy); - - return regulator_bulk_disable(NUM_SUPPLIES, usbphyc_phy->supplies); -} - static const struct phy_ops stm32_usbphyc_phy_ops = { .init = stm32_usbphyc_phy_init, .exit = stm32_usbphyc_phy_exit, - .power_on = stm32_usbphyc_phy_power_on, - .power_off = stm32_usbphyc_phy_power_off, .owner = THIS_MODULE, }; @@ -312,7 +354,7 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *child, *np = dev->of_node; struct phy_provider *phy_provider; - u32 version; + u32 pllen, version; int ret, port = 0; usbphyc = devm_kzalloc(dev, sizeof(*usbphyc), GFP_KERNEL); @@ -344,6 +386,19 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) ret = PTR_ERR(usbphyc->rst); if (ret == -EPROBE_DEFER) goto clk_disable; + + stm32_usbphyc_clr_bits(usbphyc->base + STM32_USBPHYC_PLL, PLLEN); + } + + /* + * Wait for minimum width of powerdown pulse (ENABLE = Low): + * we have to ensure the PLL is disabled before phys initialization. + */ + if (readl_relaxed_poll_timeout(usbphyc->base + STM32_USBPHYC_PLL, + pllen, !(pllen & PLLEN), 5, 50)) { + dev_warn(usbphyc->dev, "PLL not reset\n"); + ret = -EPROBE_DEFER; + goto clk_disable; } usbphyc->switch_setup = -EINVAL; @@ -355,11 +410,26 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) goto clk_disable; } + usbphyc->vdda1v1 = devm_regulator_get(dev, "vdda1v1"); + if (IS_ERR(usbphyc->vdda1v1)) { + ret = PTR_ERR(usbphyc->vdda1v1); + if (ret != -EPROBE_DEFER) + dev_err(dev, "failed to get vdda1v1 supply: %d\n", ret); + goto clk_disable; + } + + usbphyc->vdda1v8 = devm_regulator_get(dev, "vdda1v8"); + if (IS_ERR(usbphyc->vdda1v8)) { + ret = PTR_ERR(usbphyc->vdda1v8); + if (ret != -EPROBE_DEFER) + dev_err(dev, "failed to get vdda1v8 supply: %d\n", ret); + goto clk_disable; + } + for_each_child_of_node(np, child) { struct stm32_usbphyc_phy *usbphyc_phy; struct phy *phy; u32 index; - int i; phy = devm_phy_create(dev, child, &stm32_usbphyc_phy_ops); if (IS_ERR(phy)) { @@ -377,18 +447,6 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) goto put_child; } - for (i = 0; i < NUM_SUPPLIES; i++) - usbphyc_phy->supplies[i].supply = supplies_names[i]; - - ret = devm_regulator_bulk_get(&phy->dev, NUM_SUPPLIES, - usbphyc_phy->supplies); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(&phy->dev, - "failed to get regulators: %d\n", ret); - goto put_child; - } - ret = of_property_read_u32(child, "reg", &index); if (ret || index > usbphyc->nphys) { dev_err(&phy->dev, "invalid reg property: %d\n", ret); @@ -432,6 +490,12 @@ clk_disable: static int stm32_usbphyc_remove(struct platform_device *pdev) { struct stm32_usbphyc *usbphyc = dev_get_drvdata(&pdev->dev); + int port; + + /* Ensure PHYs are not active, to allow PLL disabling */ + for (port = 0; port < usbphyc->nphys; port++) + if (usbphyc->phys[port]->active) + stm32_usbphyc_phy_exit(usbphyc->phys[port]->phy); clk_disable_unprepare(usbphyc->clk); diff --git a/drivers/phy/xilinx/phy-zynqmp.c b/drivers/phy/xilinx/phy-zynqmp.c index 2b0f921b6ee3..2b65f84a5f89 100644 --- a/drivers/phy/xilinx/phy-zynqmp.c +++ b/drivers/phy/xilinx/phy-zynqmp.c @@ -874,13 +874,10 @@ static int xpsgtr_get_ref_clocks(struct xpsgtr_dev *gtr_dev) snprintf(name, sizeof(name), "ref%u", refclk); clk = devm_clk_get_optional(gtr_dev->dev, name); - if (IS_ERR(clk)) { - if (PTR_ERR(clk) != -EPROBE_DEFER) - dev_err(gtr_dev->dev, - "Failed to get reference clock %u: %ld\n", - refclk, PTR_ERR(clk)); - return PTR_ERR(clk); - } + if (IS_ERR(clk)) + return dev_err_probe(gtr_dev->dev, PTR_ERR(clk), + "Failed to get reference clock %u\n", + refclk); if (!clk) continue; diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig index 03c62e1cb395..b7675cce0027 100644 --- a/drivers/pinctrl/Kconfig +++ b/drivers/pinctrl/Kconfig @@ -370,6 +370,19 @@ config PINCTRL_MICROCHIP_SGPIO connect control signals from SFP modules and to act as an LED controller. +config PINCTRL_K210 + bool "Pinctrl driver for the Canaan Kendryte K210 SoC" + depends on RISCV && SOC_CANAAN && OF + select GENERIC_PINMUX_FUNCTIONS + select GENERIC_PINCONF + select GPIOLIB + select OF_GPIO + select REGMAP_MMIO + default SOC_CANAAN + help + Add support for the Canaan Kendryte K210 RISC-V SOC Field + Programmable IO Array (FPIOA) controller. + source "drivers/pinctrl/actions/Kconfig" source "drivers/pinctrl/aspeed/Kconfig" source "drivers/pinctrl/bcm/Kconfig" diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile index efc96f25c8db..8bf459c32a76 100644 --- a/drivers/pinctrl/Makefile +++ b/drivers/pinctrl/Makefile @@ -45,6 +45,7 @@ obj-$(CONFIG_PINCTRL_RK805) += pinctrl-rk805.o obj-$(CONFIG_PINCTRL_OCELOT) += pinctrl-ocelot.o obj-$(CONFIG_PINCTRL_MICROCHIP_SGPIO) += pinctrl-microchip-sgpio.o obj-$(CONFIG_PINCTRL_EQUILIBRIUM) += pinctrl-equilibrium.o +obj-$(CONFIG_PINCTRL_K210) += pinctrl-k210.o obj-y += actions/ obj-$(CONFIG_ARCH_ASPEED) += aspeed/ diff --git a/drivers/pinctrl/pinctrl-k210.c b/drivers/pinctrl/pinctrl-k210.c new file mode 100644 index 000000000000..8a733cf77ba0 --- /dev/null +++ b/drivers/pinctrl/pinctrl-k210.c @@ -0,0 +1,985 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2020 Sean Anderson <seanga2@gmail.com> + * Copyright (c) 2020 Western Digital Corporation or its affiliates. + */ +#include <linux/io.h> +#include <linux/of_device.h> +#include <linux/clk.h> +#include <linux/mfd/syscon.h> +#include <linux/platform_device.h> +#include <linux/bitfield.h> +#include <linux/regmap.h> +#include <linux/slab.h> +#include <linux/pinctrl/pinctrl.h> +#include <linux/pinctrl/pinmux.h> +#include <linux/pinctrl/pinconf.h> +#include <linux/pinctrl/pinconf-generic.h> +#include <linux/io.h> + +#include <dt-bindings/pinctrl/k210-fpioa.h> + +#include "core.h" +#include "pinconf.h" +#include "pinctrl-utils.h" + +/* + * The K210 only implements 8 drive levels, even though + * there is register space for 16 + */ +#define K210_PC_DRIVE_MASK GENMASK(11, 8) +#define K210_PC_DRIVE_SHIFT 8 +#define K210_PC_DRIVE_0 (0 << K210_PC_DRIVE_SHIFT) +#define K210_PC_DRIVE_1 (1 << K210_PC_DRIVE_SHIFT) +#define K210_PC_DRIVE_2 (2 << K210_PC_DRIVE_SHIFT) +#define K210_PC_DRIVE_3 (3 << K210_PC_DRIVE_SHIFT) +#define K210_PC_DRIVE_4 (4 << K210_PC_DRIVE_SHIFT) +#define K210_PC_DRIVE_5 (5 << K210_PC_DRIVE_SHIFT) +#define K210_PC_DRIVE_6 (6 << K210_PC_DRIVE_SHIFT) +#define K210_PC_DRIVE_7 (7 << K210_PC_DRIVE_SHIFT) +#define K210_PC_DRIVE_MAX 7 +#define K210_PC_MODE_MASK GENMASK(23, 12) + +/* + * output enabled == PC_OE & (PC_OE_INV ^ FUNCTION_OE) + * where FUNCTION_OE is a physical signal from the function. + */ +#define K210_PC_OE BIT(12) /* Output Enable */ +#define K210_PC_OE_INV BIT(13) /* INVert Output Enable */ +#define K210_PC_DO_OE BIT(14) /* set Data Out to Output Enable sig */ +#define K210_PC_DO_INV BIT(15) /* INVert final Data Output */ +#define K210_PC_PU BIT(16) /* Pull Up */ +#define K210_PC_PD BIT(17) /* Pull Down */ +/* Strong pull up not implemented on K210 */ +#define K210_PC_SL BIT(19) /* reduce SLew rate */ +/* Same semantics as OE above */ +#define K210_PC_IE BIT(20) /* Input Enable */ +#define K210_PC_IE_INV BIT(21) /* INVert Input Enable */ +#define K210_PC_DI_INV BIT(22) /* INVert Data Input */ +#define K210_PC_ST BIT(23) /* Schmitt Trigger */ +#define K210_PC_DI BIT(31) /* raw Data Input */ + +#define K210_PC_BIAS_MASK (K210_PC_PU & K210_PC_PD) + +#define K210_PC_MODE_IN (K210_PC_IE | K210_PC_ST) +#define K210_PC_MODE_OUT (K210_PC_DRIVE_7 | K210_PC_OE) +#define K210_PC_MODE_I2C (K210_PC_MODE_IN | K210_PC_SL | \ + K210_PC_OE | K210_PC_PU) +#define K210_PC_MODE_SCCB (K210_PC_MODE_I2C | \ + K210_PC_OE_INV | K210_PC_IE_INV) +#define K210_PC_MODE_SPI (K210_PC_MODE_IN | K210_PC_IE_INV | \ + K210_PC_MODE_OUT | K210_PC_OE_INV) +#define K210_PC_MODE_GPIO (K210_PC_MODE_IN | K210_PC_MODE_OUT) + +#define K210_PG_FUNC GENMASK(7, 0) +#define K210_PG_DO BIT(8) +#define K210_PG_PIN GENMASK(22, 16) + +/* + * struct k210_fpioa: Kendryte K210 FPIOA memory mapped registers + * @pins: 48 32-bits IO pin registers + * @tie_en: 256 (one per function) input tie enable bits + * @tie_val: 256 (one per function) input tie value bits + */ +struct k210_fpioa { + u32 pins[48]; + u32 tie_en[8]; + u32 tie_val[8]; +}; + +struct k210_fpioa_data { + + struct device *dev; + struct pinctrl_dev *pctl; + + struct k210_fpioa __iomem *fpioa; + struct regmap *sysctl_map; + u32 power_offset; + struct clk *clk; + struct clk *pclk; +}; + +#define K210_PIN_NAME(i) ("IO_" #i) +#define K210_PIN(i) [(i)] = PINCTRL_PIN((i), K210_PIN_NAME(i)) + +static const struct pinctrl_pin_desc k210_pins[] = { + K210_PIN(0), K210_PIN(1), K210_PIN(2), + K210_PIN(3), K210_PIN(4), K210_PIN(5), + K210_PIN(6), K210_PIN(7), K210_PIN(8), + K210_PIN(9), K210_PIN(10), K210_PIN(11), + K210_PIN(12), K210_PIN(13), K210_PIN(14), + K210_PIN(15), K210_PIN(16), K210_PIN(17), + K210_PIN(18), K210_PIN(19), K210_PIN(20), + K210_PIN(21), K210_PIN(22), K210_PIN(23), + K210_PIN(24), K210_PIN(25), K210_PIN(26), + K210_PIN(27), K210_PIN(28), K210_PIN(29), + K210_PIN(30), K210_PIN(31), K210_PIN(32), + K210_PIN(33), K210_PIN(34), K210_PIN(35), + K210_PIN(36), K210_PIN(37), K210_PIN(38), + K210_PIN(39), K210_PIN(40), K210_PIN(41), + K210_PIN(42), K210_PIN(43), K210_PIN(44), + K210_PIN(45), K210_PIN(46), K210_PIN(47) +}; + +#define K210_NPINS ARRAY_SIZE(k210_pins) + +/* + * Pin groups: each of the 48 programmable pins is a group. + * To this are added 8 power domain groups, which for the purposes of + * the pin subsystem, contain no pins. The power domain groups only exist + * to set the power level. The id should never be used (since there are + * no pins 48-55). + */ +static const char *const k210_group_names[] = { + /* The first 48 groups are for pins, one each */ + K210_PIN_NAME(0), K210_PIN_NAME(1), K210_PIN_NAME(2), + K210_PIN_NAME(3), K210_PIN_NAME(4), K210_PIN_NAME(5), + K210_PIN_NAME(6), K210_PIN_NAME(7), K210_PIN_NAME(8), + K210_PIN_NAME(9), K210_PIN_NAME(10), K210_PIN_NAME(11), + K210_PIN_NAME(12), K210_PIN_NAME(13), K210_PIN_NAME(14), + K210_PIN_NAME(15), K210_PIN_NAME(16), K210_PIN_NAME(17), + K210_PIN_NAME(18), K210_PIN_NAME(19), K210_PIN_NAME(20), + K210_PIN_NAME(21), K210_PIN_NAME(22), K210_PIN_NAME(23), + K210_PIN_NAME(24), K210_PIN_NAME(25), K210_PIN_NAME(26), + K210_PIN_NAME(27), K210_PIN_NAME(28), K210_PIN_NAME(29), + K210_PIN_NAME(30), K210_PIN_NAME(31), K210_PIN_NAME(32), + K210_PIN_NAME(33), K210_PIN_NAME(34), K210_PIN_NAME(35), + K210_PIN_NAME(36), K210_PIN_NAME(37), K210_PIN_NAME(38), + K210_PIN_NAME(39), K210_PIN_NAME(40), K210_PIN_NAME(41), + K210_PIN_NAME(42), K210_PIN_NAME(43), K210_PIN_NAME(44), + K210_PIN_NAME(45), K210_PIN_NAME(46), K210_PIN_NAME(47), + [48] = "A0", [49] = "A1", [50] = "A2", + [51] = "B3", [52] = "B4", [53] = "B5", + [54] = "C6", [55] = "C7" +}; + +#define K210_NGROUPS ARRAY_SIZE(k210_group_names) + +enum k210_pinctrl_mode_id { + K210_PC_DEFAULT_DISABLED, + K210_PC_DEFAULT_IN, + K210_PC_DEFAULT_IN_TIE, + K210_PC_DEFAULT_OUT, + K210_PC_DEFAULT_I2C, + K210_PC_DEFAULT_SCCB, + K210_PC_DEFAULT_SPI, + K210_PC_DEFAULT_GPIO, + K210_PC_DEFAULT_INT13, +}; + +#define K210_PC_DEFAULT(mode) \ + [K210_PC_DEFAULT_##mode] = K210_PC_MODE_##mode + +static const u32 k210_pinconf_mode_id_to_mode[] = { + [K210_PC_DEFAULT_DISABLED] = 0, + K210_PC_DEFAULT(IN), + [K210_PC_DEFAULT_IN_TIE] = K210_PC_MODE_IN, + K210_PC_DEFAULT(OUT), + K210_PC_DEFAULT(I2C), + K210_PC_DEFAULT(SCCB), + K210_PC_DEFAULT(SPI), + K210_PC_DEFAULT(GPIO), + [K210_PC_DEFAULT_INT13] = K210_PC_MODE_IN | K210_PC_PU, +}; + +#undef DEFAULT + +/* + * Pin functions configuration information. + */ +struct k210_pcf_info { + char name[15]; + u8 mode_id; +}; + +#define K210_FUNC(id, mode) \ + [K210_PCF_##id] = { \ + .name = #id, \ + .mode_id = K210_PC_DEFAULT_##mode \ + } + +static const struct k210_pcf_info k210_pcf_infos[] = { + K210_FUNC(JTAG_TCLK, IN), + K210_FUNC(JTAG_TDI, IN), + K210_FUNC(JTAG_TMS, IN), + K210_FUNC(JTAG_TDO, OUT), + K210_FUNC(SPI0_D0, SPI), + K210_FUNC(SPI0_D1, SPI), + K210_FUNC(SPI0_D2, SPI), + K210_FUNC(SPI0_D3, SPI), + K210_FUNC(SPI0_D4, SPI), + K210_FUNC(SPI0_D5, SPI), + K210_FUNC(SPI0_D6, SPI), + K210_FUNC(SPI0_D7, SPI), + K210_FUNC(SPI0_SS0, OUT), + K210_FUNC(SPI0_SS1, OUT), + K210_FUNC(SPI0_SS2, OUT), + K210_FUNC(SPI0_SS3, OUT), + K210_FUNC(SPI0_ARB, IN_TIE), + K210_FUNC(SPI0_SCLK, OUT), + K210_FUNC(UARTHS_RX, IN), + K210_FUNC(UARTHS_TX, OUT), + K210_FUNC(RESV6, IN), + K210_FUNC(RESV7, IN), + K210_FUNC(CLK_SPI1, OUT), + K210_FUNC(CLK_I2C1, OUT), + K210_FUNC(GPIOHS0, GPIO), + K210_FUNC(GPIOHS1, GPIO), + K210_FUNC(GPIOHS2, GPIO), + K210_FUNC(GPIOHS3, GPIO), + K210_FUNC(GPIOHS4, GPIO), + K210_FUNC(GPIOHS5, GPIO), + K210_FUNC(GPIOHS6, GPIO), + K210_FUNC(GPIOHS7, GPIO), + K210_FUNC(GPIOHS8, GPIO), + K210_FUNC(GPIOHS9, GPIO), + K210_FUNC(GPIOHS10, GPIO), + K210_FUNC(GPIOHS11, GPIO), + K210_FUNC(GPIOHS12, GPIO), + K210_FUNC(GPIOHS13, GPIO), + K210_FUNC(GPIOHS14, GPIO), + K210_FUNC(GPIOHS15, GPIO), + K210_FUNC(GPIOHS16, GPIO), + K210_FUNC(GPIOHS17, GPIO), + K210_FUNC(GPIOHS18, GPIO), + K210_FUNC(GPIOHS19, GPIO), + K210_FUNC(GPIOHS20, GPIO), + K210_FUNC(GPIOHS21, GPIO), + K210_FUNC(GPIOHS22, GPIO), + K210_FUNC(GPIOHS23, GPIO), + K210_FUNC(GPIOHS24, GPIO), + K210_FUNC(GPIOHS25, GPIO), + K210_FUNC(GPIOHS26, GPIO), + K210_FUNC(GPIOHS27, GPIO), + K210_FUNC(GPIOHS28, GPIO), + K210_FUNC(GPIOHS29, GPIO), + K210_FUNC(GPIOHS30, GPIO), + K210_FUNC(GPIOHS31, GPIO), + K210_FUNC(GPIO0, GPIO), + K210_FUNC(GPIO1, GPIO), + K210_FUNC(GPIO2, GPIO), + K210_FUNC(GPIO3, GPIO), + K210_FUNC(GPIO4, GPIO), + K210_FUNC(GPIO5, GPIO), + K210_FUNC(GPIO6, GPIO), + K210_FUNC(GPIO7, GPIO), + K210_FUNC(UART1_RX, IN), + K210_FUNC(UART1_TX, OUT), + K210_FUNC(UART2_RX, IN), + K210_FUNC(UART2_TX, OUT), + K210_FUNC(UART3_RX, IN), + K210_FUNC(UART3_TX, OUT), + K210_FUNC(SPI1_D0, SPI), + K210_FUNC(SPI1_D1, SPI), + K210_FUNC(SPI1_D2, SPI), + K210_FUNC(SPI1_D3, SPI), + K210_FUNC(SPI1_D4, SPI), + K210_FUNC(SPI1_D5, SPI), + K210_FUNC(SPI1_D6, SPI), + K210_FUNC(SPI1_D7, SPI), + K210_FUNC(SPI1_SS0, OUT), + K210_FUNC(SPI1_SS1, OUT), + K210_FUNC(SPI1_SS2, OUT), + K210_FUNC(SPI1_SS3, OUT), + K210_FUNC(SPI1_ARB, IN_TIE), + K210_FUNC(SPI1_SCLK, OUT), + K210_FUNC(SPI2_D0, SPI), + K210_FUNC(SPI2_SS, IN), + K210_FUNC(SPI2_SCLK, IN), + K210_FUNC(I2S0_MCLK, OUT), + K210_FUNC(I2S0_SCLK, OUT), + K210_FUNC(I2S0_WS, OUT), + K210_FUNC(I2S0_IN_D0, IN), + K210_FUNC(I2S0_IN_D1, IN), + K210_FUNC(I2S0_IN_D2, IN), + K210_FUNC(I2S0_IN_D3, IN), + K210_FUNC(I2S0_OUT_D0, OUT), + K210_FUNC(I2S0_OUT_D1, OUT), + K210_FUNC(I2S0_OUT_D2, OUT), + K210_FUNC(I2S0_OUT_D3, OUT), + K210_FUNC(I2S1_MCLK, OUT), + K210_FUNC(I2S1_SCLK, OUT), + K210_FUNC(I2S1_WS, OUT), + K210_FUNC(I2S1_IN_D0, IN), + K210_FUNC(I2S1_IN_D1, IN), + K210_FUNC(I2S1_IN_D2, IN), + K210_FUNC(I2S1_IN_D3, IN), + K210_FUNC(I2S1_OUT_D0, OUT), + K210_FUNC(I2S1_OUT_D1, OUT), + K210_FUNC(I2S1_OUT_D2, OUT), + K210_FUNC(I2S1_OUT_D3, OUT), + K210_FUNC(I2S2_MCLK, OUT), + K210_FUNC(I2S2_SCLK, OUT), + K210_FUNC(I2S2_WS, OUT), + K210_FUNC(I2S2_IN_D0, IN), + K210_FUNC(I2S2_IN_D1, IN), + K210_FUNC(I2S2_IN_D2, IN), + K210_FUNC(I2S2_IN_D3, IN), + K210_FUNC(I2S2_OUT_D0, OUT), + K210_FUNC(I2S2_OUT_D1, OUT), + K210_FUNC(I2S2_OUT_D2, OUT), + K210_FUNC(I2S2_OUT_D3, OUT), + K210_FUNC(RESV0, DISABLED), + K210_FUNC(RESV1, DISABLED), + K210_FUNC(RESV2, DISABLED), + K210_FUNC(RESV3, DISABLED), + K210_FUNC(RESV4, DISABLED), + K210_FUNC(RESV5, DISABLED), + K210_FUNC(I2C0_SCLK, I2C), + K210_FUNC(I2C0_SDA, I2C), + K210_FUNC(I2C1_SCLK, I2C), + K210_FUNC(I2C1_SDA, I2C), + K210_FUNC(I2C2_SCLK, I2C), + K210_FUNC(I2C2_SDA, I2C), + K210_FUNC(DVP_XCLK, OUT), + K210_FUNC(DVP_RST, OUT), + K210_FUNC(DVP_PWDN, OUT), + K210_FUNC(DVP_VSYNC, IN), + K210_FUNC(DVP_HSYNC, IN), + K210_FUNC(DVP_PCLK, IN), + K210_FUNC(DVP_D0, IN), + K210_FUNC(DVP_D1, IN), + K210_FUNC(DVP_D2, IN), + K210_FUNC(DVP_D3, IN), + K210_FUNC(DVP_D4, IN), + K210_FUNC(DVP_D5, IN), + K210_FUNC(DVP_D6, IN), + K210_FUNC(DVP_D7, IN), + K210_FUNC(SCCB_SCLK, SCCB), + K210_FUNC(SCCB_SDA, SCCB), + K210_FUNC(UART1_CTS, IN), + K210_FUNC(UART1_DSR, IN), + K210_FUNC(UART1_DCD, IN), + K210_FUNC(UART1_RI, IN), + K210_FUNC(UART1_SIR_IN, IN), + K210_FUNC(UART1_DTR, OUT), + K210_FUNC(UART1_RTS, OUT), + K210_FUNC(UART1_OUT2, OUT), + K210_FUNC(UART1_OUT1, OUT), + K210_FUNC(UART1_SIR_OUT, OUT), + K210_FUNC(UART1_BAUD, OUT), + K210_FUNC(UART1_RE, OUT), + K210_FUNC(UART1_DE, OUT), + K210_FUNC(UART1_RS485_EN, OUT), + K210_FUNC(UART2_CTS, IN), + K210_FUNC(UART2_DSR, IN), + K210_FUNC(UART2_DCD, IN), + K210_FUNC(UART2_RI, IN), + K210_FUNC(UART2_SIR_IN, IN), + K210_FUNC(UART2_DTR, OUT), + K210_FUNC(UART2_RTS, OUT), + K210_FUNC(UART2_OUT2, OUT), + K210_FUNC(UART2_OUT1, OUT), + K210_FUNC(UART2_SIR_OUT, OUT), + K210_FUNC(UART2_BAUD, OUT), + K210_FUNC(UART2_RE, OUT), + K210_FUNC(UART2_DE, OUT), + K210_FUNC(UART2_RS485_EN, OUT), + K210_FUNC(UART3_CTS, IN), + K210_FUNC(UART3_DSR, IN), + K210_FUNC(UART3_DCD, IN), + K210_FUNC(UART3_RI, IN), + K210_FUNC(UART3_SIR_IN, IN), + K210_FUNC(UART3_DTR, OUT), + K210_FUNC(UART3_RTS, OUT), + K210_FUNC(UART3_OUT2, OUT), + K210_FUNC(UART3_OUT1, OUT), + K210_FUNC(UART3_SIR_OUT, OUT), + K210_FUNC(UART3_BAUD, OUT), + K210_FUNC(UART3_RE, OUT), + K210_FUNC(UART3_DE, OUT), + K210_FUNC(UART3_RS485_EN, OUT), + K210_FUNC(TIMER0_TOGGLE1, OUT), + K210_FUNC(TIMER0_TOGGLE2, OUT), + K210_FUNC(TIMER0_TOGGLE3, OUT), + K210_FUNC(TIMER0_TOGGLE4, OUT), + K210_FUNC(TIMER1_TOGGLE1, OUT), + K210_FUNC(TIMER1_TOGGLE2, OUT), + K210_FUNC(TIMER1_TOGGLE3, OUT), + K210_FUNC(TIMER1_TOGGLE4, OUT), + K210_FUNC(TIMER2_TOGGLE1, OUT), + K210_FUNC(TIMER2_TOGGLE2, OUT), + K210_FUNC(TIMER2_TOGGLE3, OUT), + K210_FUNC(TIMER2_TOGGLE4, OUT), + K210_FUNC(CLK_SPI2, OUT), + K210_FUNC(CLK_I2C2, OUT), + K210_FUNC(INTERNAL0, OUT), + K210_FUNC(INTERNAL1, OUT), + K210_FUNC(INTERNAL2, OUT), + K210_FUNC(INTERNAL3, OUT), + K210_FUNC(INTERNAL4, OUT), + K210_FUNC(INTERNAL5, OUT), + K210_FUNC(INTERNAL6, OUT), + K210_FUNC(INTERNAL7, OUT), + K210_FUNC(INTERNAL8, OUT), + K210_FUNC(INTERNAL9, IN), + K210_FUNC(INTERNAL10, IN), + K210_FUNC(INTERNAL11, IN), + K210_FUNC(INTERNAL12, IN), + K210_FUNC(INTERNAL13, INT13), + K210_FUNC(INTERNAL14, I2C), + K210_FUNC(INTERNAL15, IN), + K210_FUNC(INTERNAL16, IN), + K210_FUNC(INTERNAL17, IN), + K210_FUNC(CONSTANT, DISABLED), + K210_FUNC(INTERNAL18, IN), + K210_FUNC(DEBUG0, OUT), + K210_FUNC(DEBUG1, OUT), + K210_FUNC(DEBUG2, OUT), + K210_FUNC(DEBUG3, OUT), + K210_FUNC(DEBUG4, OUT), + K210_FUNC(DEBUG5, OUT), + K210_FUNC(DEBUG6, OUT), + K210_FUNC(DEBUG7, OUT), + K210_FUNC(DEBUG8, OUT), + K210_FUNC(DEBUG9, OUT), + K210_FUNC(DEBUG10, OUT), + K210_FUNC(DEBUG11, OUT), + K210_FUNC(DEBUG12, OUT), + K210_FUNC(DEBUG13, OUT), + K210_FUNC(DEBUG14, OUT), + K210_FUNC(DEBUG15, OUT), + K210_FUNC(DEBUG16, OUT), + K210_FUNC(DEBUG17, OUT), + K210_FUNC(DEBUG18, OUT), + K210_FUNC(DEBUG19, OUT), + K210_FUNC(DEBUG20, OUT), + K210_FUNC(DEBUG21, OUT), + K210_FUNC(DEBUG22, OUT), + K210_FUNC(DEBUG23, OUT), + K210_FUNC(DEBUG24, OUT), + K210_FUNC(DEBUG25, OUT), + K210_FUNC(DEBUG26, OUT), + K210_FUNC(DEBUG27, OUT), + K210_FUNC(DEBUG28, OUT), + K210_FUNC(DEBUG29, OUT), + K210_FUNC(DEBUG30, OUT), + K210_FUNC(DEBUG31, OUT), +}; + +#define PIN_CONFIG_OUTPUT_INVERT (PIN_CONFIG_END + 1) +#define PIN_CONFIG_INPUT_INVERT (PIN_CONFIG_END + 2) + +static const struct pinconf_generic_params k210_pinconf_custom_params[] = { + { "output-polarity-invert", PIN_CONFIG_OUTPUT_INVERT, 1 }, + { "input-polarity-invert", PIN_CONFIG_INPUT_INVERT, 1 }, +}; + +/* + * Max drive strength in uA. + */ +static const int k210_pinconf_drive_strength[] = { + [0] = 11200, + [1] = 16800, + [2] = 22300, + [3] = 27800, + [4] = 33300, + [5] = 38700, + [6] = 44100, + [7] = 49500, +}; + +static int k210_pinconf_get_drive(unsigned int max_strength_ua) +{ + int i; + + for (i = K210_PC_DRIVE_MAX; i; i--) { + if (k210_pinconf_drive_strength[i] <= max_strength_ua) + return i; + } + + return -EINVAL; +} + +static void k210_pinmux_set_pin_function(struct pinctrl_dev *pctldev, + u32 pin, u32 func) +{ + struct k210_fpioa_data *pdata = pinctrl_dev_get_drvdata(pctldev); + const struct k210_pcf_info *info = &k210_pcf_infos[func]; + u32 mode = k210_pinconf_mode_id_to_mode[info->mode_id]; + u32 val = func | mode; + + dev_dbg(pdata->dev, "set pin %u function %s (%u) -> 0x%08x\n", + pin, info->name, func, val); + + writel(val, &pdata->fpioa->pins[pin]); +} + +static int k210_pinconf_set_param(struct pinctrl_dev *pctldev, + unsigned int pin, + unsigned int param, unsigned int arg) +{ + struct k210_fpioa_data *pdata = pinctrl_dev_get_drvdata(pctldev); + u32 val = readl(&pdata->fpioa->pins[pin]); + int drive; + + dev_dbg(pdata->dev, "set pin %u param %u, arg 0x%x\n", + pin, param, arg); + + switch (param) { + case PIN_CONFIG_BIAS_DISABLE: + val &= ~K210_PC_BIAS_MASK; + break; + case PIN_CONFIG_BIAS_PULL_DOWN: + if (!arg) + return -EINVAL; + val |= K210_PC_PD; + break; + case PIN_CONFIG_BIAS_PULL_UP: + if (!arg) + return -EINVAL; + val |= K210_PC_PD; + break; + case PIN_CONFIG_DRIVE_STRENGTH: + arg *= 1000; + fallthrough; + case PIN_CONFIG_DRIVE_STRENGTH_UA: + drive = k210_pinconf_get_drive(arg); + if (drive < 0) + return drive; + val &= ~K210_PC_DRIVE_MASK; + val |= FIELD_PREP(K210_PC_DRIVE_MASK, drive); + break; + case PIN_CONFIG_INPUT_ENABLE: + if (arg) + val |= K210_PC_IE; + else + val &= ~K210_PC_IE; + break; + case PIN_CONFIG_INPUT_SCHMITT_ENABLE: + if (arg) + val |= K210_PC_ST; + else + val &= ~K210_PC_ST; + break; + case PIN_CONFIG_OUTPUT: + k210_pinmux_set_pin_function(pctldev, pin, K210_PCF_CONSTANT); + val = readl(&pdata->fpioa->pins[pin]); + val |= K210_PC_MODE_OUT; + if (!arg) + val |= K210_PC_DO_INV; + break; + case PIN_CONFIG_OUTPUT_ENABLE: + if (arg) + val |= K210_PC_OE; + else + val &= ~K210_PC_OE; + break; + case PIN_CONFIG_SLEW_RATE: + if (arg) + val |= K210_PC_SL; + else + val &= ~K210_PC_SL; + break; + case PIN_CONFIG_OUTPUT_INVERT: + if (arg) + val |= K210_PC_DO_INV; + else + val &= ~K210_PC_DO_INV; + break; + case PIN_CONFIG_INPUT_INVERT: + if (arg) + val |= K210_PC_DI_INV; + else + val &= ~K210_PC_DI_INV; + break; + default: + return -EINVAL; + } + + writel(val, &pdata->fpioa->pins[pin]); + + return 0; +} + +static int k210_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin, + unsigned long *configs, unsigned int num_configs) +{ + unsigned int param, arg; + int i, ret; + + if (WARN_ON(pin >= K210_NPINS)) + return -EINVAL; + + for (i = 0; i < num_configs; i++) { + param = pinconf_to_config_param(configs[i]); + arg = pinconf_to_config_argument(configs[i]); + ret = k210_pinconf_set_param(pctldev, pin, param, arg); + if (ret) + return ret; + } + + return 0; +} + +static void k210_pinconf_dbg_show(struct pinctrl_dev *pctldev, + struct seq_file *s, unsigned int pin) +{ + struct k210_fpioa_data *pdata = pinctrl_dev_get_drvdata(pctldev); + + seq_printf(s, "%#x", readl(&pdata->fpioa->pins[pin])); +} + +static int k210_pinconf_group_set(struct pinctrl_dev *pctldev, + unsigned int selector, unsigned long *configs, + unsigned int num_configs) +{ + struct k210_fpioa_data *pdata = pinctrl_dev_get_drvdata(pctldev); + unsigned int param, arg; + u32 bit; + int i; + + /* Pins should be configured with pinmux, not groups*/ + if (selector < K210_NPINS) + return -EINVAL; + + /* Otherwise it's a power domain */ + for (i = 0; i < num_configs; i++) { + param = pinconf_to_config_param(configs[i]); + if (param != PIN_CONFIG_POWER_SOURCE) + return -EINVAL; + + arg = pinconf_to_config_argument(configs[i]); + bit = BIT(selector - K210_NPINS); + regmap_update_bits(pdata->sysctl_map, + pdata->power_offset, + bit, arg ? bit : 0); + } + + return 0; +} + +static void k210_pinconf_group_dbg_show(struct pinctrl_dev *pctldev, + struct seq_file *s, + unsigned int selector) +{ + struct k210_fpioa_data *pdata = pinctrl_dev_get_drvdata(pctldev); + int ret; + u32 val; + + if (selector < K210_NPINS) + return k210_pinconf_dbg_show(pctldev, s, selector); + + ret = regmap_read(pdata->sysctl_map, pdata->power_offset, &val); + if (ret) { + dev_err(pdata->dev, "Failed to read power reg\n"); + return; + } + + seq_printf(s, "%s: %s V", k210_group_names[selector], + val & BIT(selector - K210_NPINS) ? "1.8" : "3.3"); +} + +static const struct pinconf_ops k210_pinconf_ops = { + .is_generic = true, + .pin_config_set = k210_pinconf_set, + .pin_config_group_set = k210_pinconf_group_set, + .pin_config_dbg_show = k210_pinconf_dbg_show, + .pin_config_group_dbg_show = k210_pinconf_group_dbg_show, +}; + +static int k210_pinmux_get_function_count(struct pinctrl_dev *pctldev) +{ + return ARRAY_SIZE(k210_pcf_infos); +} + +static const char *k210_pinmux_get_function_name(struct pinctrl_dev *pctldev, + unsigned int selector) +{ + return k210_pcf_infos[selector].name; +} + +static int k210_pinmux_get_function_groups(struct pinctrl_dev *pctldev, + unsigned int selector, + const char * const **groups, + unsigned int * const num_groups) +{ + /* Any function can be mapped to any pin */ + *groups = k210_group_names; + *num_groups = K210_NPINS; + + return 0; +} + +static int k210_pinmux_set_mux(struct pinctrl_dev *pctldev, + unsigned int function, + unsigned int group) +{ + /* Can't mux power domains */ + if (group >= K210_NPINS) + return -EINVAL; + + k210_pinmux_set_pin_function(pctldev, group, function); + + return 0; +} + +static const struct pinmux_ops k210_pinmux_ops = { + .get_functions_count = k210_pinmux_get_function_count, + .get_function_name = k210_pinmux_get_function_name, + .get_function_groups = k210_pinmux_get_function_groups, + .set_mux = k210_pinmux_set_mux, + .strict = true, +}; + +static int k210_pinctrl_get_groups_count(struct pinctrl_dev *pctldev) +{ + return K210_NGROUPS; +} + +static const char *k210_pinctrl_get_group_name(struct pinctrl_dev *pctldev, + unsigned int group) +{ + return k210_group_names[group]; +} + +static int k210_pinctrl_get_group_pins(struct pinctrl_dev *pctldev, + unsigned int group, + const unsigned int **pins, + unsigned int *npins) +{ + if (group >= K210_NPINS) { + *pins = NULL; + *npins = 0; + return 0; + } + + *pins = &k210_pins[group].number; + *npins = 1; + + return 0; +} + +static void k210_pinctrl_pin_dbg_show(struct pinctrl_dev *pctldev, + struct seq_file *s, unsigned int offset) +{ + seq_printf(s, "%s", dev_name(pctldev->dev)); +} + +static int k210_pinctrl_dt_subnode_to_map(struct pinctrl_dev *pctldev, + struct device_node *np, + struct pinctrl_map **map, + unsigned int *reserved_maps, + unsigned int *num_maps) +{ + struct property *prop; + const __be32 *p; + int ret, pinmux_groups; + u32 pinmux_group; + unsigned long *configs = NULL; + unsigned int num_configs = 0; + unsigned int reserve = 0; + + ret = of_property_count_strings(np, "groups"); + if (!ret) + return pinconf_generic_dt_subnode_to_map(pctldev, np, map, + reserved_maps, num_maps, + PIN_MAP_TYPE_CONFIGS_GROUP); + + pinmux_groups = of_property_count_u32_elems(np, "pinmux"); + if (pinmux_groups <= 0) { + /* Ignore this node */ + return 0; + } + + ret = pinconf_generic_parse_dt_config(np, pctldev, &configs, + &num_configs); + if (ret < 0) { + dev_err(pctldev->dev, "%pOF: could not parse node property\n", + np); + return ret; + } + + reserve = pinmux_groups * (1 + num_configs); + ret = pinctrl_utils_reserve_map(pctldev, map, reserved_maps, num_maps, + reserve); + if (ret < 0) + goto exit; + + of_property_for_each_u32(np, "pinmux", prop, p, pinmux_group) { + const char *group_name, *func_name; + u32 pin = FIELD_GET(K210_PG_PIN, pinmux_group); + u32 func = FIELD_GET(K210_PG_FUNC, pinmux_group); + + if (pin >= K210_NPINS) { + ret = -EINVAL; + goto exit; + } + + group_name = k210_group_names[pin]; + func_name = k210_pcf_infos[func].name; + + dev_dbg(pctldev->dev, "Pinmux %s: pin %u func %s\n", + np->name, pin, func_name); + + ret = pinctrl_utils_add_map_mux(pctldev, map, reserved_maps, + num_maps, group_name, + func_name); + if (ret < 0) { + dev_err(pctldev->dev, "%pOF add mux map failed %d\n", + np, ret); + goto exit; + } + + if (num_configs) { + ret = pinctrl_utils_add_map_configs(pctldev, map, + reserved_maps, num_maps, group_name, + configs, num_configs, + PIN_MAP_TYPE_CONFIGS_PIN); + if (ret < 0) { + dev_err(pctldev->dev, + "%pOF add configs map failed %d\n", + np, ret); + goto exit; + } + } + } + + ret = 0; + +exit: + kfree(configs); + return ret; +} + +static int k210_pinctrl_dt_node_to_map(struct pinctrl_dev *pctldev, + struct device_node *np_config, + struct pinctrl_map **map, + unsigned int *num_maps) +{ + unsigned int reserved_maps; + struct device_node *np; + int ret; + + reserved_maps = 0; + *map = NULL; + *num_maps = 0; + + ret = k210_pinctrl_dt_subnode_to_map(pctldev, np_config, map, + &reserved_maps, num_maps); + if (ret < 0) + goto err; + + for_each_available_child_of_node(np_config, np) { + ret = k210_pinctrl_dt_subnode_to_map(pctldev, np, map, + &reserved_maps, num_maps); + if (ret < 0) + goto err; + } + return 0; + +err: + pinctrl_utils_free_map(pctldev, *map, *num_maps); + return ret; +} + + +static const struct pinctrl_ops k210_pinctrl_ops = { + .get_groups_count = k210_pinctrl_get_groups_count, + .get_group_name = k210_pinctrl_get_group_name, + .get_group_pins = k210_pinctrl_get_group_pins, + .pin_dbg_show = k210_pinctrl_pin_dbg_show, + .dt_node_to_map = k210_pinctrl_dt_node_to_map, + .dt_free_map = pinconf_generic_dt_free_map, +}; + +static struct pinctrl_desc k210_pinctrl_desc = { + .name = "k210-pinctrl", + .pins = k210_pins, + .npins = K210_NPINS, + .pctlops = &k210_pinctrl_ops, + .pmxops = &k210_pinmux_ops, + .confops = &k210_pinconf_ops, + .custom_params = k210_pinconf_custom_params, + .num_custom_params = ARRAY_SIZE(k210_pinconf_custom_params), +}; + +static void k210_fpioa_init_ties(struct k210_fpioa_data *pdata) +{ + struct k210_fpioa __iomem *fpioa = pdata->fpioa; + u32 val; + int i, j; + + dev_dbg(pdata->dev, "Init pin ties\n"); + + /* Init pin functions input ties */ + for (i = 0; i < ARRAY_SIZE(fpioa->tie_en); i++) { + val = 0; + for (j = 0; j < 32; j++) { + if (k210_pcf_infos[i * 32 + j].mode_id == + K210_PC_DEFAULT_IN_TIE) { + dev_dbg(pdata->dev, + "tie_en function %d (%s)\n", + i * 32 + j, + k210_pcf_infos[i * 32 + j].name); + val |= BIT(j); + } + } + + /* Set value before enable */ + writel(val, &fpioa->tie_val[i]); + writel(val, &fpioa->tie_en[i]); + } +} + +static int k210_fpioa_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct k210_fpioa_data *pdata; + int ret; + + dev_info(dev, "K210 FPIOA pin controller\n"); + + pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return -ENOMEM; + + pdata->dev = dev; + platform_set_drvdata(pdev, pdata); + + pdata->fpioa = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(pdata->fpioa)) + return PTR_ERR(pdata->fpioa); + + pdata->clk = devm_clk_get(dev, "ref"); + if (IS_ERR(pdata->clk)) + return PTR_ERR(pdata->clk); + + ret = clk_prepare_enable(pdata->clk); + if (ret) + return ret; + + pdata->pclk = devm_clk_get_optional(dev, "pclk"); + if (!IS_ERR(pdata->pclk)) + clk_prepare_enable(pdata->pclk); + + pdata->sysctl_map = + syscon_regmap_lookup_by_phandle_args(np, + "canaan,k210-sysctl-power", + 1, &pdata->power_offset); + if (IS_ERR(pdata->sysctl_map)) + return PTR_ERR(pdata->sysctl_map); + + k210_fpioa_init_ties(pdata); + + pdata->pctl = pinctrl_register(&k210_pinctrl_desc, dev, (void *)pdata); + if (IS_ERR(pdata->pctl)) + return PTR_ERR(pdata->pctl); + + return 0; +} + +static const struct of_device_id k210_fpioa_dt_ids[] = { + { .compatible = "canaan,k210-fpioa" }, + { /* sentinel */ }, +}; + +static struct platform_driver k210_fpioa_driver = { + .probe = k210_fpioa_probe, + .driver = { + .name = "k210-fpioa", + .of_match_table = k210_fpioa_dt_ids, + }, +}; +builtin_platform_driver(k210_fpioa_driver); diff --git a/drivers/platform/goldfish/goldfish_pipe.c b/drivers/platform/goldfish/goldfish_pipe.c index 1ab207ec9c94..b67539f9848c 100644 --- a/drivers/platform/goldfish/goldfish_pipe.c +++ b/drivers/platform/goldfish/goldfish_pipe.c @@ -212,9 +212,6 @@ struct goldfish_pipe_dev { int version; unsigned char __iomem *base; - /* an irq tasklet to run goldfish_interrupt_task */ - struct tasklet_struct irq_tasklet; - struct miscdevice miscdev; }; @@ -577,10 +574,10 @@ static struct goldfish_pipe *signalled_pipes_pop_front( return pipe; } -static void goldfish_interrupt_task(unsigned long dev_addr) +static irqreturn_t goldfish_interrupt_task(int irq, void *dev_addr) { /* Iterate over the signalled pipes and wake them one by one */ - struct goldfish_pipe_dev *dev = (struct goldfish_pipe_dev *)dev_addr; + struct goldfish_pipe_dev *dev = dev_addr; struct goldfish_pipe *pipe; int wakes; @@ -599,13 +596,14 @@ static void goldfish_interrupt_task(unsigned long dev_addr) */ wake_up_interruptible(&pipe->wake_queue); } + return IRQ_HANDLED; } static void goldfish_pipe_device_deinit(struct platform_device *pdev, struct goldfish_pipe_dev *dev); /* - * The general idea of the interrupt handling: + * The general idea of the (threaded) interrupt handling: * * 1. device raises an interrupt if there's at least one signalled pipe * 2. IRQ handler reads the signalled pipes and their count from the device @@ -614,8 +612,8 @@ static void goldfish_pipe_device_deinit(struct platform_device *pdev, * otherwise it leaves it raised, so IRQ handler will be called * again for the next chunk * 4. IRQ handler adds all returned pipes to the device's signalled pipes list - * 5. IRQ handler launches a tasklet to process the signalled pipes from the - * list in a separate context + * 5. IRQ handler defers processing the signalled pipes from the list in a + * separate context */ static irqreturn_t goldfish_pipe_interrupt(int irq, void *dev_id) { @@ -645,8 +643,7 @@ static irqreturn_t goldfish_pipe_interrupt(int irq, void *dev_id) spin_unlock_irqrestore(&dev->lock, flags); - tasklet_schedule(&dev->irq_tasklet); - return IRQ_HANDLED; + return IRQ_WAKE_THREAD; } static int get_free_pipe_id_locked(struct goldfish_pipe_dev *dev) @@ -811,12 +808,10 @@ static int goldfish_pipe_device_init(struct platform_device *pdev, { int err; - tasklet_init(&dev->irq_tasklet, &goldfish_interrupt_task, - (unsigned long)dev); - - err = devm_request_irq(&pdev->dev, dev->irq, - goldfish_pipe_interrupt, - IRQF_SHARED, "goldfish_pipe", dev); + err = devm_request_threaded_irq(&pdev->dev, dev->irq, + goldfish_pipe_interrupt, + goldfish_interrupt_task, + IRQF_SHARED, "goldfish_pipe", dev); if (err) { dev_err(&pdev->dev, "unable to allocate IRQ for v2\n"); return err; @@ -874,7 +869,6 @@ static void goldfish_pipe_device_deinit(struct platform_device *pdev, struct goldfish_pipe_dev *dev) { misc_deregister(&dev->miscdev); - tasklet_kill(&dev->irq_tasklet); kfree(dev->pipes); free_page((unsigned long)dev->buffers); } diff --git a/drivers/platform/surface/Kconfig b/drivers/platform/surface/Kconfig index 0847b2dc97bf..179b8c93d7fd 100644 --- a/drivers/platform/surface/Kconfig +++ b/drivers/platform/surface/Kconfig @@ -77,6 +77,33 @@ config SURFACE_AGGREGATOR_CDEV The provided interface is intended for debugging and development only, and should not be used otherwise. +config SURFACE_AGGREGATOR_REGISTRY + tristate "Surface System Aggregator Module Device Registry" + depends on SURFACE_AGGREGATOR + depends on SURFACE_AGGREGATOR_BUS + help + Device-registry and device-hubs for Surface System Aggregator Module + (SSAM) devices. + + Provides a module and driver which act as a device-registry for SSAM + client devices that cannot be detected automatically, e.g. via ACPI. + Such devices are instead provided via this registry and attached via + device hubs, also provided in this module. + + Devices provided via this registry are: + - Platform profile (performance-/cooling-mode) device (5th- and later + generations). + - Battery/AC devices (7th-generation). + - HID input devices (7th-generation). + + Select M (recommended) or Y here if you want support for the above + mentioned devices on the corresponding Surface models. Without this + module, the respective devices will not be instantiated and thus any + functionality provided by them will be missing, even when drivers for + these devices are present. In other words, this module only provides + the respective client devices. Drivers for these devices still need to + be selected via the other options. + config SURFACE_GPE tristate "Surface GPE/Lid Support Driver" depends on DMI diff --git a/drivers/platform/surface/Makefile b/drivers/platform/surface/Makefile index 990424c5f0c9..80035ee540bf 100644 --- a/drivers/platform/surface/Makefile +++ b/drivers/platform/surface/Makefile @@ -10,6 +10,7 @@ obj-$(CONFIG_SURFACE_3_POWER_OPREGION) += surface3_power.o obj-$(CONFIG_SURFACE_ACPI_NOTIFY) += surface_acpi_notify.o obj-$(CONFIG_SURFACE_AGGREGATOR) += aggregator/ obj-$(CONFIG_SURFACE_AGGREGATOR_CDEV) += surface_aggregator_cdev.o +obj-$(CONFIG_SURFACE_AGGREGATOR_REGISTRY) += surface_aggregator_registry.o obj-$(CONFIG_SURFACE_GPE) += surface_gpe.o obj-$(CONFIG_SURFACE_HOTPLUG) += surface_hotplug.o obj-$(CONFIG_SURFACE_PRO3_BUTTON) += surfacepro3_button.o diff --git a/drivers/platform/surface/surface_aggregator_registry.c b/drivers/platform/surface/surface_aggregator_registry.c new file mode 100644 index 000000000000..caee90d135c5 --- /dev/null +++ b/drivers/platform/surface/surface_aggregator_registry.c @@ -0,0 +1,641 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Surface System Aggregator Module (SSAM) client device registry. + * + * Registry for non-platform/non-ACPI SSAM client devices, i.e. devices that + * cannot be auto-detected. Provides device-hubs and performs instantiation + * for these devices. + * + * Copyright (C) 2020-2021 Maximilian Luz <luzmaximilian@gmail.com> + */ + +#include <linux/acpi.h> +#include <linux/kernel.h> +#include <linux/limits.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/platform_device.h> +#include <linux/property.h> +#include <linux/types.h> + +#include <linux/surface_aggregator/controller.h> +#include <linux/surface_aggregator/device.h> + + +/* -- Device registry. ------------------------------------------------------ */ + +/* + * SSAM device names follow the SSAM module alias, meaning they are prefixed + * with 'ssam:', followed by domain, category, target ID, instance ID, and + * function, each encoded as two-digit hexadecimal, separated by ':'. In other + * words, it follows the scheme + * + * ssam:dd:cc:tt:ii:ff + * + * Where, 'dd', 'cc', 'tt', 'ii', and 'ff' are the two-digit hexadecimal + * values mentioned above, respectively. + */ + +/* Root node. */ +static const struct software_node ssam_node_root = { + .name = "ssam_platform_hub", +}; + +/* Base device hub (devices attached to Surface Book 3 base). */ +static const struct software_node ssam_node_hub_base = { + .name = "ssam:00:00:02:00:00", + .parent = &ssam_node_root, +}; + +/* AC adapter. */ +static const struct software_node ssam_node_bat_ac = { + .name = "ssam:01:02:01:01:01", + .parent = &ssam_node_root, +}; + +/* Primary battery. */ +static const struct software_node ssam_node_bat_main = { + .name = "ssam:01:02:01:01:00", + .parent = &ssam_node_root, +}; + +/* Secondary battery (Surface Book 3). */ +static const struct software_node ssam_node_bat_sb3base = { + .name = "ssam:01:02:02:01:00", + .parent = &ssam_node_hub_base, +}; + +/* Platform profile / performance-mode device. */ +static const struct software_node ssam_node_tmp_pprof = { + .name = "ssam:01:03:01:00:01", + .parent = &ssam_node_root, +}; + +/* DTX / detachment-system device (Surface Book 3). */ +static const struct software_node ssam_node_bas_dtx = { + .name = "ssam:01:11:01:00:00", + .parent = &ssam_node_root, +}; + +/* HID keyboard. */ +static const struct software_node ssam_node_hid_main_keyboard = { + .name = "ssam:01:15:02:01:00", + .parent = &ssam_node_root, +}; + +/* HID touchpad. */ +static const struct software_node ssam_node_hid_main_touchpad = { + .name = "ssam:01:15:02:03:00", + .parent = &ssam_node_root, +}; + +/* HID device instance 5 (unknown HID device). */ +static const struct software_node ssam_node_hid_main_iid5 = { + .name = "ssam:01:15:02:05:00", + .parent = &ssam_node_root, +}; + +/* HID keyboard (base hub). */ +static const struct software_node ssam_node_hid_base_keyboard = { + .name = "ssam:01:15:02:01:00", + .parent = &ssam_node_hub_base, +}; + +/* HID touchpad (base hub). */ +static const struct software_node ssam_node_hid_base_touchpad = { + .name = "ssam:01:15:02:03:00", + .parent = &ssam_node_hub_base, +}; + +/* HID device instance 5 (unknown HID device, base hub). */ +static const struct software_node ssam_node_hid_base_iid5 = { + .name = "ssam:01:15:02:05:00", + .parent = &ssam_node_hub_base, +}; + +/* HID device instance 6 (unknown HID device, base hub). */ +static const struct software_node ssam_node_hid_base_iid6 = { + .name = "ssam:01:15:02:06:00", + .parent = &ssam_node_hub_base, +}; + +/* Devices for Surface Book 2. */ +static const struct software_node *ssam_node_group_sb2[] = { + &ssam_node_root, + &ssam_node_tmp_pprof, + NULL, +}; + +/* Devices for Surface Book 3. */ +static const struct software_node *ssam_node_group_sb3[] = { + &ssam_node_root, + &ssam_node_hub_base, + &ssam_node_bat_ac, + &ssam_node_bat_main, + &ssam_node_bat_sb3base, + &ssam_node_tmp_pprof, + &ssam_node_bas_dtx, + &ssam_node_hid_base_keyboard, + &ssam_node_hid_base_touchpad, + &ssam_node_hid_base_iid5, + &ssam_node_hid_base_iid6, + NULL, +}; + +/* Devices for Surface Laptop 1. */ +static const struct software_node *ssam_node_group_sl1[] = { + &ssam_node_root, + &ssam_node_tmp_pprof, + NULL, +}; + +/* Devices for Surface Laptop 2. */ +static const struct software_node *ssam_node_group_sl2[] = { + &ssam_node_root, + &ssam_node_tmp_pprof, + NULL, +}; + +/* Devices for Surface Laptop 3. */ +static const struct software_node *ssam_node_group_sl3[] = { + &ssam_node_root, + &ssam_node_bat_ac, + &ssam_node_bat_main, + &ssam_node_tmp_pprof, + &ssam_node_hid_main_keyboard, + &ssam_node_hid_main_touchpad, + &ssam_node_hid_main_iid5, + NULL, +}; + +/* Devices for Surface Laptop Go. */ +static const struct software_node *ssam_node_group_slg1[] = { + &ssam_node_root, + &ssam_node_bat_ac, + &ssam_node_bat_main, + &ssam_node_tmp_pprof, + NULL, +}; + +/* Devices for Surface Pro 5. */ +static const struct software_node *ssam_node_group_sp5[] = { + &ssam_node_root, + &ssam_node_tmp_pprof, + NULL, +}; + +/* Devices for Surface Pro 6. */ +static const struct software_node *ssam_node_group_sp6[] = { + &ssam_node_root, + &ssam_node_tmp_pprof, + NULL, +}; + +/* Devices for Surface Pro 7. */ +static const struct software_node *ssam_node_group_sp7[] = { + &ssam_node_root, + &ssam_node_bat_ac, + &ssam_node_bat_main, + &ssam_node_tmp_pprof, + NULL, +}; + + +/* -- Device registry helper functions. ------------------------------------- */ + +static int ssam_uid_from_string(const char *str, struct ssam_device_uid *uid) +{ + u8 d, tc, tid, iid, fn; + int n; + + n = sscanf(str, "ssam:%hhx:%hhx:%hhx:%hhx:%hhx", &d, &tc, &tid, &iid, &fn); + if (n != 5) + return -EINVAL; + + uid->domain = d; + uid->category = tc; + uid->target = tid; + uid->instance = iid; + uid->function = fn; + + return 0; +} + +static int ssam_hub_remove_devices_fn(struct device *dev, void *data) +{ + if (!is_ssam_device(dev)) + return 0; + + ssam_device_remove(to_ssam_device(dev)); + return 0; +} + +static void ssam_hub_remove_devices(struct device *parent) +{ + device_for_each_child_reverse(parent, NULL, ssam_hub_remove_devices_fn); +} + +static int ssam_hub_add_device(struct device *parent, struct ssam_controller *ctrl, + struct fwnode_handle *node) +{ + struct ssam_device_uid uid; + struct ssam_device *sdev; + int status; + + status = ssam_uid_from_string(fwnode_get_name(node), &uid); + if (status) + return status; + + sdev = ssam_device_alloc(ctrl, uid); + if (!sdev) + return -ENOMEM; + + sdev->dev.parent = parent; + sdev->dev.fwnode = node; + + status = ssam_device_add(sdev); + if (status) + ssam_device_put(sdev); + + return status; +} + +static int ssam_hub_add_devices(struct device *parent, struct ssam_controller *ctrl, + struct fwnode_handle *node) +{ + struct fwnode_handle *child; + int status; + + fwnode_for_each_child_node(node, child) { + /* + * Try to add the device specified in the firmware node. If + * this fails with -EINVAL, the node does not specify any SSAM + * device, so ignore it and continue with the next one. + */ + + status = ssam_hub_add_device(parent, ctrl, child); + if (status && status != -EINVAL) + goto err; + } + + return 0; +err: + ssam_hub_remove_devices(parent); + return status; +} + + +/* -- SSAM base-hub driver. ------------------------------------------------- */ + +enum ssam_base_hub_state { + SSAM_BASE_HUB_UNINITIALIZED, + SSAM_BASE_HUB_CONNECTED, + SSAM_BASE_HUB_DISCONNECTED, +}; + +struct ssam_base_hub { + struct ssam_device *sdev; + + struct mutex lock; /* Guards state update checks and transitions. */ + enum ssam_base_hub_state state; + + struct ssam_event_notifier notif; +}; + +static SSAM_DEFINE_SYNC_REQUEST_R(ssam_bas_query_opmode, u8, { + .target_category = SSAM_SSH_TC_BAS, + .target_id = 0x01, + .command_id = 0x0d, + .instance_id = 0x00, +}); + +#define SSAM_BAS_OPMODE_TABLET 0x00 +#define SSAM_EVENT_BAS_CID_CONNECTION 0x0c + +static int ssam_base_hub_query_state(struct ssam_base_hub *hub, enum ssam_base_hub_state *state) +{ + u8 opmode; + int status; + + status = ssam_retry(ssam_bas_query_opmode, hub->sdev->ctrl, &opmode); + if (status < 0) { + dev_err(&hub->sdev->dev, "failed to query base state: %d\n", status); + return status; + } + + if (opmode != SSAM_BAS_OPMODE_TABLET) + *state = SSAM_BASE_HUB_CONNECTED; + else + *state = SSAM_BASE_HUB_DISCONNECTED; + + return 0; +} + +static ssize_t ssam_base_hub_state_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct ssam_base_hub *hub = dev_get_drvdata(dev); + bool connected; + + mutex_lock(&hub->lock); + connected = hub->state == SSAM_BASE_HUB_CONNECTED; + mutex_unlock(&hub->lock); + + return sysfs_emit(buf, "%d\n", connected); +} + +static struct device_attribute ssam_base_hub_attr_state = + __ATTR(state, 0444, ssam_base_hub_state_show, NULL); + +static struct attribute *ssam_base_hub_attrs[] = { + &ssam_base_hub_attr_state.attr, + NULL, +}; + +const struct attribute_group ssam_base_hub_group = { + .attrs = ssam_base_hub_attrs, +}; + +static int __ssam_base_hub_update(struct ssam_base_hub *hub, enum ssam_base_hub_state new) +{ + struct fwnode_handle *node = dev_fwnode(&hub->sdev->dev); + int status = 0; + + lockdep_assert_held(&hub->lock); + + if (hub->state == new) + return 0; + hub->state = new; + + if (hub->state == SSAM_BASE_HUB_CONNECTED) + status = ssam_hub_add_devices(&hub->sdev->dev, hub->sdev->ctrl, node); + else + ssam_hub_remove_devices(&hub->sdev->dev); + + if (status) + dev_err(&hub->sdev->dev, "failed to update base-hub devices: %d\n", status); + + return status; +} + +static int ssam_base_hub_update(struct ssam_base_hub *hub) +{ + enum ssam_base_hub_state state; + int status; + + mutex_lock(&hub->lock); + + status = ssam_base_hub_query_state(hub, &state); + if (!status) + status = __ssam_base_hub_update(hub, state); + + mutex_unlock(&hub->lock); + return status; +} + +static u32 ssam_base_hub_notif(struct ssam_event_notifier *nf, const struct ssam_event *event) +{ + struct ssam_base_hub *hub; + struct ssam_device *sdev; + enum ssam_base_hub_state new; + + hub = container_of(nf, struct ssam_base_hub, notif); + sdev = hub->sdev; + + if (event->command_id != SSAM_EVENT_BAS_CID_CONNECTION) + return 0; + + if (event->length < 1) { + dev_err(&sdev->dev, "unexpected payload size: %u\n", + event->length); + return 0; + } + + if (event->data[0]) + new = SSAM_BASE_HUB_CONNECTED; + else + new = SSAM_BASE_HUB_DISCONNECTED; + + mutex_lock(&hub->lock); + __ssam_base_hub_update(hub, new); + mutex_unlock(&hub->lock); + + /* + * Do not return SSAM_NOTIF_HANDLED: The event should be picked up and + * consumed by the detachment system driver. We're just a (more or less) + * silent observer. + */ + return 0; +} + +static int __maybe_unused ssam_base_hub_resume(struct device *dev) +{ + return ssam_base_hub_update(dev_get_drvdata(dev)); +} +static SIMPLE_DEV_PM_OPS(ssam_base_hub_pm_ops, NULL, ssam_base_hub_resume); + +static int ssam_base_hub_probe(struct ssam_device *sdev) +{ + struct ssam_base_hub *hub; + int status; + + hub = devm_kzalloc(&sdev->dev, sizeof(*hub), GFP_KERNEL); + if (!hub) + return -ENOMEM; + + mutex_init(&hub->lock); + + hub->sdev = sdev; + hub->state = SSAM_BASE_HUB_UNINITIALIZED; + + hub->notif.base.priority = INT_MAX; /* This notifier should run first. */ + hub->notif.base.fn = ssam_base_hub_notif; + hub->notif.event.reg = SSAM_EVENT_REGISTRY_SAM; + hub->notif.event.id.target_category = SSAM_SSH_TC_BAS, + hub->notif.event.id.instance = 0, + hub->notif.event.mask = SSAM_EVENT_MASK_NONE; + hub->notif.event.flags = SSAM_EVENT_SEQUENCED; + + ssam_device_set_drvdata(sdev, hub); + + status = ssam_notifier_register(sdev->ctrl, &hub->notif); + if (status) + goto err_register; + + status = ssam_base_hub_update(hub); + if (status) + goto err_update; + + status = sysfs_create_group(&sdev->dev.kobj, &ssam_base_hub_group); + if (status) + goto err_update; + + return 0; + +err_update: + ssam_notifier_unregister(sdev->ctrl, &hub->notif); + ssam_hub_remove_devices(&sdev->dev); +err_register: + mutex_destroy(&hub->lock); + return status; +} + +static void ssam_base_hub_remove(struct ssam_device *sdev) +{ + struct ssam_base_hub *hub = ssam_device_get_drvdata(sdev); + + sysfs_remove_group(&sdev->dev.kobj, &ssam_base_hub_group); + + ssam_notifier_unregister(sdev->ctrl, &hub->notif); + ssam_hub_remove_devices(&sdev->dev); + + mutex_destroy(&hub->lock); +} + +static const struct ssam_device_id ssam_base_hub_match[] = { + { SSAM_VDEV(HUB, 0x02, SSAM_ANY_IID, 0x00) }, + { }, +}; + +static struct ssam_device_driver ssam_base_hub_driver = { + .probe = ssam_base_hub_probe, + .remove = ssam_base_hub_remove, + .match_table = ssam_base_hub_match, + .driver = { + .name = "surface_aggregator_base_hub", + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + .pm = &ssam_base_hub_pm_ops, + }, +}; + + +/* -- SSAM platform/meta-hub driver. ---------------------------------------- */ + +static const struct acpi_device_id ssam_platform_hub_match[] = { + /* Surface Pro 4, 5, and 6 (OMBR < 0x10) */ + { "MSHW0081", (unsigned long)ssam_node_group_sp5 }, + + /* Surface Pro 6 (OMBR >= 0x10) */ + { "MSHW0111", (unsigned long)ssam_node_group_sp6 }, + + /* Surface Pro 7 */ + { "MSHW0116", (unsigned long)ssam_node_group_sp7 }, + + /* Surface Book 2 */ + { "MSHW0107", (unsigned long)ssam_node_group_sb2 }, + + /* Surface Book 3 */ + { "MSHW0117", (unsigned long)ssam_node_group_sb3 }, + + /* Surface Laptop 1 */ + { "MSHW0086", (unsigned long)ssam_node_group_sl1 }, + + /* Surface Laptop 2 */ + { "MSHW0112", (unsigned long)ssam_node_group_sl2 }, + + /* Surface Laptop 3 (13", Intel) */ + { "MSHW0114", (unsigned long)ssam_node_group_sl3 }, + + /* Surface Laptop 3 (15", AMD) */ + { "MSHW0110", (unsigned long)ssam_node_group_sl3 }, + + /* Surface Laptop Go 1 */ + { "MSHW0118", (unsigned long)ssam_node_group_slg1 }, + + { }, +}; +MODULE_DEVICE_TABLE(acpi, ssam_platform_hub_match); + +static int ssam_platform_hub_probe(struct platform_device *pdev) +{ + const struct software_node **nodes; + struct ssam_controller *ctrl; + struct fwnode_handle *root; + int status; + + nodes = (const struct software_node **)acpi_device_get_match_data(&pdev->dev); + if (!nodes) + return -ENODEV; + + /* + * As we're adding the SSAM client devices as children under this device + * and not the SSAM controller, we need to add a device link to the + * controller to ensure that we remove all of our devices before the + * controller is removed. This also guarantees proper ordering for + * suspend/resume of the devices on this hub. + */ + ctrl = ssam_client_bind(&pdev->dev); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl) == -ENODEV ? -EPROBE_DEFER : PTR_ERR(ctrl); + + status = software_node_register_node_group(nodes); + if (status) + return status; + + root = software_node_fwnode(&ssam_node_root); + if (!root) { + software_node_unregister_node_group(nodes); + return -ENOENT; + } + + set_secondary_fwnode(&pdev->dev, root); + + status = ssam_hub_add_devices(&pdev->dev, ctrl, root); + if (status) { + set_secondary_fwnode(&pdev->dev, NULL); + software_node_unregister_node_group(nodes); + } + + platform_set_drvdata(pdev, nodes); + return status; +} + +static int ssam_platform_hub_remove(struct platform_device *pdev) +{ + const struct software_node **nodes = platform_get_drvdata(pdev); + + ssam_hub_remove_devices(&pdev->dev); + set_secondary_fwnode(&pdev->dev, NULL); + software_node_unregister_node_group(nodes); + return 0; +} + +static struct platform_driver ssam_platform_hub_driver = { + .probe = ssam_platform_hub_probe, + .remove = ssam_platform_hub_remove, + .driver = { + .name = "surface_aggregator_platform_hub", + .acpi_match_table = ssam_platform_hub_match, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; + + +/* -- Module initialization. ------------------------------------------------ */ + +static int __init ssam_device_hub_init(void) +{ + int status; + + status = platform_driver_register(&ssam_platform_hub_driver); + if (status) + return status; + + status = ssam_device_driver_register(&ssam_base_hub_driver); + if (status) + platform_driver_unregister(&ssam_platform_hub_driver); + + return status; +} +module_init(ssam_device_hub_init); + +static void __exit ssam_device_hub_exit(void) +{ + ssam_device_driver_unregister(&ssam_base_hub_driver); + platform_driver_unregister(&ssam_platform_hub_driver); +} +module_exit(ssam_device_hub_exit); + +MODULE_AUTHOR("Maximilian Luz <luzmaximilian@gmail.com>"); +MODULE_DESCRIPTION("Device-registry for Surface System Aggregator Module"); +MODULE_LICENSE("GPL"); diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig index 56353e8c792a..ad4e630e73e2 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig @@ -450,7 +450,7 @@ config IDEAPAD_LAPTOP depends on BACKLIGHT_CLASS_DEVICE depends on ACPI_VIDEO || ACPI_VIDEO = n depends on ACPI_WMI || ACPI_WMI = n - depends on ACPI_PLATFORM_PROFILE + select ACPI_PLATFORM_PROFILE select INPUT_SPARSEKMAP select NEW_LEDS select LEDS_CLASS @@ -484,7 +484,7 @@ config THINKPAD_ACPI depends on RFKILL || RFKILL = n depends on ACPI_VIDEO || ACPI_VIDEO = n depends on BACKLIGHT_CLASS_DEVICE - depends on ACPI_PLATFORM_PROFILE + select ACPI_PLATFORM_PROFILE select HWMON select NVRAM select NEW_LEDS diff --git a/drivers/platform/x86/intel_scu_pcidrv.c b/drivers/platform/x86/intel_scu_pcidrv.c index 8c5fd8240da9..80abc708e4f2 100644 --- a/drivers/platform/x86/intel_scu_pcidrv.c +++ b/drivers/platform/x86/intel_scu_pcidrv.c @@ -17,7 +17,6 @@ static int intel_scu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { - void (*setup_fn)(void) = (void (*)(void))id->driver_data; struct intel_scu_ipc_data scu_data = {}; struct intel_scu_ipc_dev *scu; int ret; @@ -30,27 +29,14 @@ static int intel_scu_pci_probe(struct pci_dev *pdev, scu_data.irq = pdev->irq; scu = intel_scu_ipc_register(&pdev->dev, &scu_data); - if (IS_ERR(scu)) - return PTR_ERR(scu); - - if (setup_fn) - setup_fn(); - return 0; -} - -static void intel_mid_scu_setup(void) -{ - intel_scu_devices_create(); + return PTR_ERR_OR_ZERO(scu); } static const struct pci_device_id pci_ids[] = { - { PCI_VDEVICE(INTEL, 0x080e), - .driver_data = (kernel_ulong_t)intel_mid_scu_setup }, - { PCI_VDEVICE(INTEL, 0x08ea), - .driver_data = (kernel_ulong_t)intel_mid_scu_setup }, + { PCI_VDEVICE(INTEL, 0x080e) }, + { PCI_VDEVICE(INTEL, 0x08ea) }, { PCI_VDEVICE(INTEL, 0x0a94) }, - { PCI_VDEVICE(INTEL, 0x11a0), - .driver_data = (kernel_ulong_t)intel_mid_scu_setup }, + { PCI_VDEVICE(INTEL, 0x11a0) }, { PCI_VDEVICE(INTEL, 0x1a94) }, { PCI_VDEVICE(INTEL, 0x5a94) }, {} diff --git a/drivers/powercap/Kconfig b/drivers/powercap/Kconfig index 20b4325c6161..8242e8c5ed77 100644 --- a/drivers/powercap/Kconfig +++ b/drivers/powercap/Kconfig @@ -45,7 +45,7 @@ config IDLE_INJECT on a per CPU basis. config DTPM - bool "Power capping for Dynamic Thermal Power Management" + bool "Power capping for Dynamic Thermal Power Management (EXPERIMENTAL)" help This enables support for the power capping for the dynamic thermal power management userspace engine. diff --git a/drivers/powercap/dtpm.c b/drivers/powercap/dtpm.c index 5a51cd34a7e8..c2185ec5f887 100644 --- a/drivers/powercap/dtpm.c +++ b/drivers/powercap/dtpm.c @@ -207,6 +207,9 @@ int dtpm_release_zone(struct powercap_zone *pcz) if (dtpm->ops) dtpm->ops->release(dtpm); + if (root == dtpm) + root = NULL; + kfree(dtpm); return 0; diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig index 0937e1c047ac..9a4f66ae8070 100644 --- a/drivers/pwm/Kconfig +++ b/drivers/pwm/Kconfig @@ -611,14 +611,4 @@ config PWM_VT8500 To compile this driver as a module, choose M here: the module will be called pwm-vt8500. -config PWM_ZX - tristate "ZTE ZX PWM support" - depends on ARCH_ZX || COMPILE_TEST - depends on HAS_IOMEM - help - Generic PWM framework driver for ZTE ZX family SoCs. - - To compile this driver as a module, choose M here: the module - will be called pwm-zx. - endif diff --git a/drivers/pwm/Makefile b/drivers/pwm/Makefile index 18b89d7fd092..6374d3b1d6f3 100644 --- a/drivers/pwm/Makefile +++ b/drivers/pwm/Makefile @@ -57,4 +57,3 @@ obj-$(CONFIG_PWM_TIEHRPWM) += pwm-tiehrpwm.o obj-$(CONFIG_PWM_TWL) += pwm-twl.o obj-$(CONFIG_PWM_TWL_LED) += pwm-twl-led.o obj-$(CONFIG_PWM_VT8500) += pwm-vt8500.o -obj-$(CONFIG_PWM_ZX) += pwm-zx.o diff --git a/drivers/pwm/pwm-iqs620a.c b/drivers/pwm/pwm-iqs620a.c index 5ede8255926e..957b972c458b 100644 --- a/drivers/pwm/pwm-iqs620a.c +++ b/drivers/pwm/pwm-iqs620a.c @@ -37,16 +37,34 @@ struct iqs620_pwm_private { struct pwm_chip chip; struct notifier_block notifier; struct mutex lock; - bool out_en; - u8 duty_val; + unsigned int duty_scale; }; +static int iqs620_pwm_init(struct iqs620_pwm_private *iqs620_pwm, + unsigned int duty_scale) +{ + struct iqs62x_core *iqs62x = iqs620_pwm->iqs62x; + int ret; + + if (!duty_scale) + return regmap_update_bits(iqs62x->regmap, IQS620_PWR_SETTINGS, + IQS620_PWR_SETTINGS_PWM_OUT, 0); + + ret = regmap_write(iqs62x->regmap, IQS620_PWM_DUTY_CYCLE, + duty_scale - 1); + if (ret) + return ret; + + return regmap_update_bits(iqs62x->regmap, IQS620_PWR_SETTINGS, + IQS620_PWR_SETTINGS_PWM_OUT, 0xff); +} + static int iqs620_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { struct iqs620_pwm_private *iqs620_pwm; - struct iqs62x_core *iqs62x; - u64 duty_scale; + unsigned int duty_cycle; + unsigned int duty_scale; int ret; if (state->polarity != PWM_POLARITY_NORMAL) @@ -56,7 +74,6 @@ static int iqs620_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, return -EINVAL; iqs620_pwm = container_of(chip, struct iqs620_pwm_private, chip); - iqs62x = iqs620_pwm->iqs62x; /* * The duty cycle generated by the device is calculated as follows: @@ -70,38 +87,18 @@ static int iqs620_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, * For lower duty cycles (e.g. 0), the PWM output is simply disabled to * allow an external pull-down resistor to hold the GPIO3/LTX pin low. */ - duty_scale = div_u64(state->duty_cycle * 256, IQS620_PWM_PERIOD_NS); - - mutex_lock(&iqs620_pwm->lock); - - if (!state->enabled || !duty_scale) { - ret = regmap_update_bits(iqs62x->regmap, IQS620_PWR_SETTINGS, - IQS620_PWR_SETTINGS_PWM_OUT, 0); - if (ret) - goto err_mutex; - } + duty_cycle = min_t(u64, state->duty_cycle, IQS620_PWM_PERIOD_NS); + duty_scale = duty_cycle * 256 / IQS620_PWM_PERIOD_NS; - if (duty_scale) { - u8 duty_val = min_t(u64, duty_scale - 1, 0xff); + if (!state->enabled) + duty_scale = 0; - ret = regmap_write(iqs62x->regmap, IQS620_PWM_DUTY_CYCLE, - duty_val); - if (ret) - goto err_mutex; - - iqs620_pwm->duty_val = duty_val; - } - - if (state->enabled && duty_scale) { - ret = regmap_update_bits(iqs62x->regmap, IQS620_PWR_SETTINGS, - IQS620_PWR_SETTINGS_PWM_OUT, 0xff); - if (ret) - goto err_mutex; - } + mutex_lock(&iqs620_pwm->lock); - iqs620_pwm->out_en = state->enabled; + ret = iqs620_pwm_init(iqs620_pwm, duty_scale); + if (!ret) + iqs620_pwm->duty_scale = duty_scale; -err_mutex: mutex_unlock(&iqs620_pwm->lock); return ret; @@ -119,12 +116,11 @@ static void iqs620_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, /* * Since the device cannot generate a 0% duty cycle, requests to do so * cause subsequent calls to iqs620_pwm_get_state to report the output - * as disabled with duty cycle equal to that which was in use prior to - * the request. This is not ideal, but is the best compromise based on + * as disabled. This is not ideal, but is the best compromise based on * the capabilities of the device. */ - state->enabled = iqs620_pwm->out_en; - state->duty_cycle = DIV_ROUND_UP((iqs620_pwm->duty_val + 1) * + state->enabled = iqs620_pwm->duty_scale > 0; + state->duty_cycle = DIV_ROUND_UP(iqs620_pwm->duty_scale * IQS620_PWM_PERIOD_NS, 256); mutex_unlock(&iqs620_pwm->lock); @@ -136,7 +132,6 @@ static int iqs620_pwm_notifier(struct notifier_block *notifier, unsigned long event_flags, void *context) { struct iqs620_pwm_private *iqs620_pwm; - struct iqs62x_core *iqs62x; int ret; if (!(event_flags & BIT(IQS62X_EVENT_SYS_RESET))) @@ -144,7 +139,6 @@ static int iqs620_pwm_notifier(struct notifier_block *notifier, iqs620_pwm = container_of(notifier, struct iqs620_pwm_private, notifier); - iqs62x = iqs620_pwm->iqs62x; mutex_lock(&iqs620_pwm->lock); @@ -153,16 +147,8 @@ static int iqs620_pwm_notifier(struct notifier_block *notifier, * of a device reset, so nothing else is printed here unless there is * an additional failure. */ - ret = regmap_write(iqs62x->regmap, IQS620_PWM_DUTY_CYCLE, - iqs620_pwm->duty_val); - if (ret) - goto err_mutex; + ret = iqs620_pwm_init(iqs620_pwm, iqs620_pwm->duty_scale); - ret = regmap_update_bits(iqs62x->regmap, IQS620_PWR_SETTINGS, - IQS620_PWR_SETTINGS_PWM_OUT, - iqs620_pwm->out_en ? 0xff : 0); - -err_mutex: mutex_unlock(&iqs620_pwm->lock); if (ret) { @@ -209,12 +195,14 @@ static int iqs620_pwm_probe(struct platform_device *pdev) ret = regmap_read(iqs62x->regmap, IQS620_PWR_SETTINGS, &val); if (ret) return ret; - iqs620_pwm->out_en = val & IQS620_PWR_SETTINGS_PWM_OUT; - ret = regmap_read(iqs62x->regmap, IQS620_PWM_DUTY_CYCLE, &val); - if (ret) - return ret; - iqs620_pwm->duty_val = val; + if (val & IQS620_PWR_SETTINGS_PWM_OUT) { + ret = regmap_read(iqs62x->regmap, IQS620_PWM_DUTY_CYCLE, &val); + if (ret) + return ret; + + iqs620_pwm->duty_scale = val + 1; + } iqs620_pwm->chip.dev = &pdev->dev; iqs620_pwm->chip.ops = &iqs620_pwm_ops; diff --git a/drivers/pwm/pwm-lpc18xx-sct.c b/drivers/pwm/pwm-lpc18xx-sct.c index dc5133bec3e7..7ef40243eb6c 100644 --- a/drivers/pwm/pwm-lpc18xx-sct.c +++ b/drivers/pwm/pwm-lpc18xx-sct.c @@ -289,7 +289,7 @@ static int lpc18xx_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) dev_err(lpc18xx_pwm->dev, "maximum number of simultaneous channels reached\n"); return -EBUSY; - }; + } set_bit(event, &lpc18xx_pwm->event_map); lpc18xx_data->duty_event = event; diff --git a/drivers/pwm/pwm-rockchip.c b/drivers/pwm/pwm-rockchip.c index 389a5e140412..6ad7d0a50aed 100644 --- a/drivers/pwm/pwm-rockchip.c +++ b/drivers/pwm/pwm-rockchip.c @@ -72,6 +72,10 @@ static void rockchip_pwm_get_state(struct pwm_chip *chip, if (ret) return; + ret = clk_enable(pc->clk); + if (ret) + return; + clk_rate = clk_get_rate(pc->clk); tmp = readl_relaxed(pc->base + pc->data->regs.period); @@ -90,6 +94,7 @@ static void rockchip_pwm_get_state(struct pwm_chip *chip, else state->polarity = PWM_POLARITY_NORMAL; + clk_disable(pc->clk); clk_disable(pc->pclk); } @@ -189,6 +194,10 @@ static int rockchip_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (ret) return ret; + ret = clk_enable(pc->clk); + if (ret) + return ret; + pwm_get_state(pwm, &curstate); enabled = curstate.enabled; @@ -208,6 +217,7 @@ static int rockchip_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, } out: + clk_disable(pc->clk); clk_disable(pc->pclk); return ret; @@ -288,6 +298,7 @@ static int rockchip_pwm_probe(struct platform_device *pdev) const struct of_device_id *id; struct rockchip_pwm_chip *pc; u32 enable_conf, ctrl; + bool enabled; int ret, count; id = of_match_device(rockchip_pwm_dt_ids, &pdev->dev); @@ -307,7 +318,7 @@ static int rockchip_pwm_probe(struct platform_device *pdev) pc->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(pc->clk)) return dev_err_probe(&pdev->dev, PTR_ERR(pc->clk), - "Can't get bus clk\n"); + "Can't get PWM clk\n"); } count = of_count_phandle_with_args(pdev->dev.of_node, @@ -326,13 +337,13 @@ static int rockchip_pwm_probe(struct platform_device *pdev) ret = clk_prepare_enable(pc->clk); if (ret) { - dev_err(&pdev->dev, "Can't prepare enable bus clk: %d\n", ret); + dev_err(&pdev->dev, "Can't prepare enable PWM clk: %d\n", ret); return ret; } - ret = clk_prepare(pc->pclk); + ret = clk_prepare_enable(pc->pclk); if (ret) { - dev_err(&pdev->dev, "Can't prepare APB clk: %d\n", ret); + dev_err(&pdev->dev, "Can't prepare enable APB clk: %d\n", ret); goto err_clk; } @@ -349,23 +360,26 @@ static int rockchip_pwm_probe(struct platform_device *pdev) pc->chip.of_pwm_n_cells = 3; } + enable_conf = pc->data->enable_conf; + ctrl = readl_relaxed(pc->base + pc->data->regs.ctrl); + enabled = (ctrl & enable_conf) == enable_conf; + ret = pwmchip_add(&pc->chip); if (ret < 0) { - clk_unprepare(pc->clk); dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret); goto err_pclk; } /* Keep the PWM clk enabled if the PWM appears to be up and running. */ - enable_conf = pc->data->enable_conf; - ctrl = readl_relaxed(pc->base + pc->data->regs.ctrl); - if ((ctrl & enable_conf) != enable_conf) + if (!enabled) clk_disable(pc->clk); + clk_disable(pc->pclk); + return 0; err_pclk: - clk_unprepare(pc->pclk); + clk_disable_unprepare(pc->pclk); err_clk: clk_disable_unprepare(pc->clk); diff --git a/drivers/pwm/pwm-zx.c b/drivers/pwm/pwm-zx.c deleted file mode 100644 index 34e91195ce98..000000000000 --- a/drivers/pwm/pwm-zx.c +++ /dev/null @@ -1,278 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2017 Sanechips Technology Co., Ltd. - * Copyright 2017 Linaro Ltd. - */ - -#include <linux/clk.h> -#include <linux/err.h> -#include <linux/io.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/platform_device.h> -#include <linux/pwm.h> -#include <linux/slab.h> - -#define ZX_PWM_MODE 0x0 -#define ZX_PWM_CLKDIV_SHIFT 2 -#define ZX_PWM_CLKDIV_MASK GENMASK(11, 2) -#define ZX_PWM_CLKDIV(x) (((x) << ZX_PWM_CLKDIV_SHIFT) & \ - ZX_PWM_CLKDIV_MASK) -#define ZX_PWM_POLAR BIT(1) -#define ZX_PWM_EN BIT(0) -#define ZX_PWM_PERIOD 0x4 -#define ZX_PWM_DUTY 0x8 - -#define ZX_PWM_CLKDIV_MAX 1023 -#define ZX_PWM_PERIOD_MAX 65535 - -struct zx_pwm_chip { - struct pwm_chip chip; - struct clk *pclk; - struct clk *wclk; - void __iomem *base; -}; - -static inline struct zx_pwm_chip *to_zx_pwm_chip(struct pwm_chip *chip) -{ - return container_of(chip, struct zx_pwm_chip, chip); -} - -static inline u32 zx_pwm_readl(struct zx_pwm_chip *zpc, unsigned int hwpwm, - unsigned int offset) -{ - return readl(zpc->base + (hwpwm + 1) * 0x10 + offset); -} - -static inline void zx_pwm_writel(struct zx_pwm_chip *zpc, unsigned int hwpwm, - unsigned int offset, u32 value) -{ - writel(value, zpc->base + (hwpwm + 1) * 0x10 + offset); -} - -static void zx_pwm_set_mask(struct zx_pwm_chip *zpc, unsigned int hwpwm, - unsigned int offset, u32 mask, u32 value) -{ - u32 data; - - data = zx_pwm_readl(zpc, hwpwm, offset); - data &= ~mask; - data |= value & mask; - zx_pwm_writel(zpc, hwpwm, offset, data); -} - -static void zx_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, - struct pwm_state *state) -{ - struct zx_pwm_chip *zpc = to_zx_pwm_chip(chip); - unsigned long rate; - unsigned int div; - u32 value; - u64 tmp; - - value = zx_pwm_readl(zpc, pwm->hwpwm, ZX_PWM_MODE); - - if (value & ZX_PWM_POLAR) - state->polarity = PWM_POLARITY_NORMAL; - else - state->polarity = PWM_POLARITY_INVERSED; - - if (value & ZX_PWM_EN) - state->enabled = true; - else - state->enabled = false; - - div = (value & ZX_PWM_CLKDIV_MASK) >> ZX_PWM_CLKDIV_SHIFT; - rate = clk_get_rate(zpc->wclk); - - tmp = zx_pwm_readl(zpc, pwm->hwpwm, ZX_PWM_PERIOD); - tmp *= div * NSEC_PER_SEC; - state->period = DIV_ROUND_CLOSEST_ULL(tmp, rate); - - tmp = zx_pwm_readl(zpc, pwm->hwpwm, ZX_PWM_DUTY); - tmp *= div * NSEC_PER_SEC; - state->duty_cycle = DIV_ROUND_CLOSEST_ULL(tmp, rate); -} - -static int zx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, - unsigned int duty_ns, unsigned int period_ns) -{ - struct zx_pwm_chip *zpc = to_zx_pwm_chip(chip); - unsigned int period_cycles, duty_cycles; - unsigned long long c; - unsigned int div = 1; - unsigned long rate; - - /* Find out the best divider */ - rate = clk_get_rate(zpc->wclk); - - while (1) { - c = rate / div; - c = c * period_ns; - do_div(c, NSEC_PER_SEC); - - if (c < ZX_PWM_PERIOD_MAX) - break; - - div++; - - if (div > ZX_PWM_CLKDIV_MAX) - return -ERANGE; - } - - /* Calculate duty cycles */ - period_cycles = c; - c *= duty_ns; - do_div(c, period_ns); - duty_cycles = c; - - /* - * If the PWM is being enabled, we have to temporarily disable it - * before configuring the registers. - */ - if (pwm_is_enabled(pwm)) - zx_pwm_set_mask(zpc, pwm->hwpwm, ZX_PWM_MODE, ZX_PWM_EN, 0); - - /* Set up registers */ - zx_pwm_set_mask(zpc, pwm->hwpwm, ZX_PWM_MODE, ZX_PWM_CLKDIV_MASK, - ZX_PWM_CLKDIV(div)); - zx_pwm_writel(zpc, pwm->hwpwm, ZX_PWM_PERIOD, period_cycles); - zx_pwm_writel(zpc, pwm->hwpwm, ZX_PWM_DUTY, duty_cycles); - - /* Re-enable the PWM if needed */ - if (pwm_is_enabled(pwm)) - zx_pwm_set_mask(zpc, pwm->hwpwm, ZX_PWM_MODE, - ZX_PWM_EN, ZX_PWM_EN); - - return 0; -} - -static int zx_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, - const struct pwm_state *state) -{ - struct zx_pwm_chip *zpc = to_zx_pwm_chip(chip); - struct pwm_state cstate; - int ret; - - pwm_get_state(pwm, &cstate); - - if (state->polarity != cstate.polarity) - zx_pwm_set_mask(zpc, pwm->hwpwm, ZX_PWM_MODE, ZX_PWM_POLAR, - (state->polarity == PWM_POLARITY_INVERSED) ? - 0 : ZX_PWM_POLAR); - - if (state->period != cstate.period || - state->duty_cycle != cstate.duty_cycle) { - ret = zx_pwm_config(chip, pwm, state->duty_cycle, - state->period); - if (ret) - return ret; - } - - if (state->enabled != cstate.enabled) { - if (state->enabled) { - ret = clk_prepare_enable(zpc->wclk); - if (ret) - return ret; - - zx_pwm_set_mask(zpc, pwm->hwpwm, ZX_PWM_MODE, - ZX_PWM_EN, ZX_PWM_EN); - } else { - zx_pwm_set_mask(zpc, pwm->hwpwm, ZX_PWM_MODE, - ZX_PWM_EN, 0); - clk_disable_unprepare(zpc->wclk); - } - } - - return 0; -} - -static const struct pwm_ops zx_pwm_ops = { - .apply = zx_pwm_apply, - .get_state = zx_pwm_get_state, - .owner = THIS_MODULE, -}; - -static int zx_pwm_probe(struct platform_device *pdev) -{ - struct zx_pwm_chip *zpc; - unsigned int i; - int ret; - - zpc = devm_kzalloc(&pdev->dev, sizeof(*zpc), GFP_KERNEL); - if (!zpc) - return -ENOMEM; - - zpc->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(zpc->base)) - return PTR_ERR(zpc->base); - - zpc->pclk = devm_clk_get(&pdev->dev, "pclk"); - if (IS_ERR(zpc->pclk)) - return PTR_ERR(zpc->pclk); - - zpc->wclk = devm_clk_get(&pdev->dev, "wclk"); - if (IS_ERR(zpc->wclk)) - return PTR_ERR(zpc->wclk); - - ret = clk_prepare_enable(zpc->pclk); - if (ret) - return ret; - - zpc->chip.dev = &pdev->dev; - zpc->chip.ops = &zx_pwm_ops; - zpc->chip.base = -1; - zpc->chip.npwm = 4; - zpc->chip.of_xlate = of_pwm_xlate_with_flags; - zpc->chip.of_pwm_n_cells = 3; - - /* - * PWM devices may be enabled by firmware, and let's disable all of - * them initially to save power. - */ - for (i = 0; i < zpc->chip.npwm; i++) - zx_pwm_set_mask(zpc, i, ZX_PWM_MODE, ZX_PWM_EN, 0); - - ret = pwmchip_add(&zpc->chip); - if (ret < 0) { - dev_err(&pdev->dev, "failed to add PWM chip: %d\n", ret); - clk_disable_unprepare(zpc->pclk); - return ret; - } - - platform_set_drvdata(pdev, zpc); - - return 0; -} - -static int zx_pwm_remove(struct platform_device *pdev) -{ - struct zx_pwm_chip *zpc = platform_get_drvdata(pdev); - int ret; - - ret = pwmchip_remove(&zpc->chip); - clk_disable_unprepare(zpc->pclk); - - return ret; -} - -static const struct of_device_id zx_pwm_dt_ids[] = { - { .compatible = "zte,zx296718-pwm", }, - { /* sentinel */ } -}; -MODULE_DEVICE_TABLE(of, zx_pwm_dt_ids); - -static struct platform_driver zx_pwm_driver = { - .driver = { - .name = "zx-pwm", - .of_match_table = zx_pwm_dt_ids, - }, - .probe = zx_pwm_probe, - .remove = zx_pwm_remove, -}; -module_platform_driver(zx_pwm_driver); - -MODULE_ALIAS("platform:zx-pwm"); -MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>"); -MODULE_DESCRIPTION("ZTE ZX PWM Driver"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig index 9e7efe542f69..15d1574d129b 100644 --- a/drivers/remoteproc/Kconfig +++ b/drivers/remoteproc/Kconfig @@ -155,6 +155,7 @@ config QCOM_Q6V5_ADSP depends on RPMSG_QCOM_SMD || (COMPILE_TEST && RPMSG_QCOM_SMD=n) depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n depends on QCOM_SYSMON || QCOM_SYSMON=n + depends on RPMSG_QCOM_GLINK || RPMSG_QCOM_GLINK=n select MFD_SYSCON select QCOM_PIL_INFO select QCOM_MDT_LOADER @@ -162,7 +163,9 @@ config QCOM_Q6V5_ADSP select QCOM_RPROC_COMMON help Say y here to support the Peripheral Image Loader - for the Qualcomm Technology Inc. ADSP remote processors. + for the non-TrustZone part of Qualcomm Technology Inc. ADSP and CDSP + remote processors. The TrustZone part is handled by QCOM_Q6V5_PAS + driver. config QCOM_Q6V5_MSS tristate "Qualcomm Hexagon V5 self-authenticating modem subsystem support" @@ -171,6 +174,7 @@ config QCOM_Q6V5_MSS depends on RPMSG_QCOM_SMD || (COMPILE_TEST && RPMSG_QCOM_SMD=n) depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n depends on QCOM_SYSMON || QCOM_SYSMON=n + depends on RPMSG_QCOM_GLINK || RPMSG_QCOM_GLINK=n select MFD_SYSCON select QCOM_MDT_LOADER select QCOM_PIL_INFO @@ -179,7 +183,8 @@ config QCOM_Q6V5_MSS select QCOM_SCM help Say y here to support the Qualcomm self-authenticating modem - subsystem based on Hexagon V5. + subsystem based on Hexagon V5. The TrustZone based system is + handled by QCOM_Q6V5_PAS driver. config QCOM_Q6V5_PAS tristate "Qualcomm Hexagon v5 Peripheral Authentication Service support" @@ -188,6 +193,7 @@ config QCOM_Q6V5_PAS depends on RPMSG_QCOM_SMD || (COMPILE_TEST && RPMSG_QCOM_SMD=n) depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n depends on QCOM_SYSMON || QCOM_SYSMON=n + depends on RPMSG_QCOM_GLINK || RPMSG_QCOM_GLINK=n select MFD_SYSCON select QCOM_PIL_INFO select QCOM_MDT_LOADER @@ -197,7 +203,9 @@ config QCOM_Q6V5_PAS help Say y here to support the TrustZone based Peripheral Image Loader for the Qualcomm Hexagon v5 based remote processors. This is commonly - used to control subsystems such as ADSP, Compute and Sensor. + used to control subsystems such as ADSP (Audio DSP), + CDSP (Compute DSP), MPSS (Modem Peripheral SubSystem), and + SLPI (Sensor Low Power Island). config QCOM_Q6V5_WCSS tristate "Qualcomm Hexagon based WCSS Peripheral Image Loader" @@ -206,6 +214,7 @@ config QCOM_Q6V5_WCSS depends on RPMSG_QCOM_SMD || (COMPILE_TEST && RPMSG_QCOM_SMD=n) depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n depends on QCOM_SYSMON || QCOM_SYSMON=n + depends on RPMSG_QCOM_GLINK || RPMSG_QCOM_GLINK=n select MFD_SYSCON select QCOM_MDT_LOADER select QCOM_PIL_INFO @@ -214,7 +223,8 @@ config QCOM_Q6V5_WCSS select QCOM_SCM help Say y here to support the Qualcomm Peripheral Image Loader for the - Hexagon V5 based WCSS remote processors. + Hexagon V5 based WCSS remote processors on e.g. IPQ8074. This is + a non-TrustZone wireless subsystem. config QCOM_SYSMON tristate "Qualcomm sysmon driver" @@ -238,13 +248,16 @@ config QCOM_WCNSS_PIL depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n depends on QCOM_SMEM depends on QCOM_SYSMON || QCOM_SYSMON=n + depends on RPMSG_QCOM_GLINK || RPMSG_QCOM_GLINK=n select QCOM_MDT_LOADER select QCOM_PIL_INFO select QCOM_RPROC_COMMON select QCOM_SCM help - Say y here to support the Peripheral Image Loader for the Qualcomm - Wireless Connectivity Subsystem. + Say y here to support the Peripheral Image Loader for loading WCNSS + firmware and boot the core on e.g. MSM8974, MSM8916. The firmware is + verified and booted with the help of the Peripheral Authentication + System (PAS) in TrustZone. config ST_REMOTEPROC tristate "ST remoteproc support" diff --git a/drivers/remoteproc/ingenic_rproc.c b/drivers/remoteproc/ingenic_rproc.c index 26e19e6143b7..e2618c36eaab 100644 --- a/drivers/remoteproc/ingenic_rproc.c +++ b/drivers/remoteproc/ingenic_rproc.c @@ -27,6 +27,11 @@ #define AUX_CTRL_NMI BIT(1) #define AUX_CTRL_SW_RESET BIT(0) +static bool auto_boot; +module_param(auto_boot, bool, 0400); +MODULE_PARM_DESC(auto_boot, + "Auto-boot the remote processor [default=false]"); + struct vpu_mem_map { const char *name; unsigned int da; @@ -172,6 +177,8 @@ static int ingenic_rproc_probe(struct platform_device *pdev) if (!rproc) return -ENOMEM; + rproc->auto_boot = auto_boot; + vpu = rproc->priv; vpu->dev = &pdev->dev; platform_set_drvdata(pdev, vpu); diff --git a/drivers/remoteproc/mtk_common.h b/drivers/remoteproc/mtk_common.h index 988edb4977c3..61901f5efa05 100644 --- a/drivers/remoteproc/mtk_common.h +++ b/drivers/remoteproc/mtk_common.h @@ -47,6 +47,8 @@ #define MT8192_CORE0_SW_RSTN_CLR 0x10000 #define MT8192_CORE0_SW_RSTN_SET 0x10004 +#define MT8192_CORE0_MEM_ATT_PREDEF 0x10008 +#define MT8192_CORE0_WDT_IRQ 0x10030 #define MT8192_CORE0_WDT_CFG 0x10034 #define SCP_FW_VER_LEN 32 @@ -75,6 +77,7 @@ struct mtk_scp_of_data { void (*scp_reset_assert)(struct mtk_scp *scp); void (*scp_reset_deassert)(struct mtk_scp *scp); void (*scp_stop)(struct mtk_scp *scp); + void *(*scp_da_to_va)(struct mtk_scp *scp, u64 da, size_t len); u32 host_to_scp_reg; u32 host_to_scp_int_bit; @@ -89,6 +92,10 @@ struct mtk_scp { void __iomem *reg_base; void __iomem *sram_base; size_t sram_size; + phys_addr_t sram_phys; + void __iomem *l1tcm_base; + size_t l1tcm_size; + phys_addr_t l1tcm_phys; const struct mtk_scp_of_data *data; diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index e0c235690361..ce727598c41c 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -197,17 +197,19 @@ static void mt8192_scp_irq_handler(struct mtk_scp *scp) scp_to_host = readl(scp->reg_base + MT8192_SCP2APMCU_IPC_SET); - if (scp_to_host & MT8192_SCP_IPC_INT_BIT) + if (scp_to_host & MT8192_SCP_IPC_INT_BIT) { scp_ipi_handler(scp); - else - scp_wdt_handler(scp, scp_to_host); - /* - * SCP won't send another interrupt until we clear - * MT8192_SCP2APMCU_IPC. - */ - writel(MT8192_SCP_IPC_INT_BIT, - scp->reg_base + MT8192_SCP2APMCU_IPC_CLR); + /* + * SCP won't send another interrupt until we clear + * MT8192_SCP2APMCU_IPC. + */ + writel(MT8192_SCP_IPC_INT_BIT, + scp->reg_base + MT8192_SCP2APMCU_IPC_CLR); + } else { + scp_wdt_handler(scp, scp_to_host); + writel(1, scp->reg_base + MT8192_CORE0_WDT_IRQ); + } } static irqreturn_t scp_irq_handler(int irq, void *priv) @@ -369,6 +371,9 @@ static int mt8192_scp_before_load(struct mtk_scp *scp) mt8192_power_on_sram(scp->reg_base + MT8192_L1TCM_SRAM_PDN); mt8192_power_on_sram(scp->reg_base + MT8192_CPU0_SRAM_PD); + /* enable MPU for all memory regions */ + writel(0xff, scp->reg_base + MT8192_CORE0_MEM_ATT_PREDEF); + return 0; } @@ -458,9 +463,8 @@ stop: return ret; } -static void *scp_da_to_va(struct rproc *rproc, u64 da, size_t len) +static void *mt8183_scp_da_to_va(struct mtk_scp *scp, u64 da, size_t len) { - struct mtk_scp *scp = (struct mtk_scp *)rproc->priv; int offset; if (da < scp->sram_size) { @@ -476,6 +480,42 @@ static void *scp_da_to_va(struct rproc *rproc, u64 da, size_t len) return NULL; } +static void *mt8192_scp_da_to_va(struct mtk_scp *scp, u64 da, size_t len) +{ + int offset; + + if (da >= scp->sram_phys && + (da + len) <= scp->sram_phys + scp->sram_size) { + offset = da - scp->sram_phys; + return (void __force *)scp->sram_base + offset; + } + + /* optional memory region */ + if (scp->l1tcm_size && + da >= scp->l1tcm_phys && + (da + len) <= scp->l1tcm_phys + scp->l1tcm_size) { + offset = da - scp->l1tcm_phys; + return (void __force *)scp->l1tcm_base + offset; + } + + /* optional memory region */ + if (scp->dram_size && + da >= scp->dma_addr && + (da + len) <= scp->dma_addr + scp->dram_size) { + offset = da - scp->dma_addr; + return scp->cpu_addr + offset; + } + + return NULL; +} + +static void *scp_da_to_va(struct rproc *rproc, u64 da, size_t len) +{ + struct mtk_scp *scp = (struct mtk_scp *)rproc->priv; + + return scp->data->scp_da_to_va(scp, da, len); +} + static void mt8183_scp_stop(struct mtk_scp *scp) { /* Disable SCP watchdog */ @@ -714,13 +754,27 @@ static int scp_probe(struct platform_device *pdev) goto free_rproc; } scp->sram_size = resource_size(res); + scp->sram_phys = res->start; + + /* l1tcm is an optional memory region */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "l1tcm"); + scp->l1tcm_base = devm_ioremap_resource(dev, res); + if (IS_ERR((__force void *)scp->l1tcm_base)) { + ret = PTR_ERR((__force void *)scp->l1tcm_base); + if (ret != -EINVAL) { + dev_err(dev, "Failed to map l1tcm memory\n"); + goto free_rproc; + } + } else { + scp->l1tcm_size = resource_size(res); + scp->l1tcm_phys = res->start; + } mutex_init(&scp->send_lock); for (i = 0; i < SCP_IPI_MAX; i++) mutex_init(&scp->ipi_desc[i].lock); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); - scp->reg_base = devm_ioremap_resource(dev, res); + scp->reg_base = devm_platform_ioremap_resource_byname(pdev, "cfg"); if (IS_ERR((__force void *)scp->reg_base)) { dev_err(dev, "Failed to parse and map cfg memory\n"); ret = PTR_ERR((__force void *)scp->reg_base); @@ -803,6 +857,7 @@ static const struct mtk_scp_of_data mt8183_of_data = { .scp_reset_assert = mt8183_scp_reset_assert, .scp_reset_deassert = mt8183_scp_reset_deassert, .scp_stop = mt8183_scp_stop, + .scp_da_to_va = mt8183_scp_da_to_va, .host_to_scp_reg = MT8183_HOST_TO_SCP, .host_to_scp_int_bit = MT8183_HOST_IPC_INT_BIT, .ipi_buf_offset = 0x7bdb0, @@ -814,6 +869,7 @@ static const struct mtk_scp_of_data mt8192_of_data = { .scp_reset_assert = mt8192_scp_reset_assert, .scp_reset_deassert = mt8192_scp_reset_deassert, .scp_stop = mt8192_scp_stop, + .scp_da_to_va = mt8192_scp_da_to_va, .host_to_scp_reg = MT8192_GIPC_IN_SET, .host_to_scp_int_bit = MT8192_HOST_IPC_INT_BIT, }; diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index ee586226e438..e635454d6170 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -565,6 +565,26 @@ static const struct adsp_data sm8250_adsp_resource = { .ssctl_id = 0x14, }; +static const struct adsp_data sm8350_adsp_resource = { + .crash_reason_smem = 423, + .firmware_name = "adsp.mdt", + .pas_id = 1, + .has_aggre2_clk = false, + .auto_boot = true, + .active_pd_names = (char*[]){ + "load_state", + NULL + }, + .proxy_pd_names = (char*[]){ + "lcx", + "lmx", + NULL + }, + .ssr_name = "lpass", + .sysmon_name = "adsp", + .ssctl_id = 0x14, +}; + static const struct adsp_data msm8998_adsp_resource = { .crash_reason_smem = 423, .firmware_name = "adsp.mdt", @@ -629,6 +649,25 @@ static const struct adsp_data sm8250_cdsp_resource = { .ssctl_id = 0x17, }; +static const struct adsp_data sm8350_cdsp_resource = { + .crash_reason_smem = 601, + .firmware_name = "cdsp.mdt", + .pas_id = 18, + .has_aggre2_clk = false, + .auto_boot = true, + .active_pd_names = (char*[]){ + "load_state", + NULL + }, + .proxy_pd_names = (char*[]){ + "cx", + NULL + }, + .ssr_name = "cdsp", + .sysmon_name = "cdsp", + .ssctl_id = 0x17, +}; + static const struct adsp_data mpss_resource_init = { .crash_reason_smem = 421, .firmware_name = "modem.mdt", @@ -701,6 +740,26 @@ static const struct adsp_data sm8250_slpi_resource = { .ssctl_id = 0x16, }; +static const struct adsp_data sm8350_slpi_resource = { + .crash_reason_smem = 424, + .firmware_name = "slpi.mdt", + .pas_id = 12, + .has_aggre2_clk = false, + .auto_boot = true, + .active_pd_names = (char*[]){ + "load_state", + NULL + }, + .proxy_pd_names = (char*[]){ + "lcx", + "lmx", + NULL + }, + .ssr_name = "dsps", + .sysmon_name = "slpi", + .ssctl_id = 0x16, +}; + static const struct adsp_data msm8998_slpi_resource = { .crash_reason_smem = 424, .firmware_name = "slpi.mdt", @@ -745,6 +804,10 @@ static const struct of_device_id adsp_of_match[] = { { .compatible = "qcom,sm8250-adsp-pas", .data = &sm8250_adsp_resource}, { .compatible = "qcom,sm8250-cdsp-pas", .data = &sm8250_cdsp_resource}, { .compatible = "qcom,sm8250-slpi-pas", .data = &sm8250_slpi_resource}, + { .compatible = "qcom,sm8350-adsp-pas", .data = &sm8350_adsp_resource}, + { .compatible = "qcom,sm8350-cdsp-pas", .data = &sm8350_cdsp_resource}, + { .compatible = "qcom,sm8350-slpi-pas", .data = &sm8350_slpi_resource}, + { .compatible = "qcom,sm8350-mpss-pas", .data = &mpss_resource_init}, { }, }; MODULE_DEVICE_TABLE(of, adsp_of_match); diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c index f95854255c70..2a6a23cb14ca 100644 --- a/drivers/remoteproc/qcom_wcnss.c +++ b/drivers/remoteproc/qcom_wcnss.c @@ -570,7 +570,7 @@ static int wcnss_probe(struct platform_device *pdev) if (IS_ERR(mmio)) { ret = PTR_ERR(mmio); goto free_rproc; - }; + } ret = wcnss_alloc_memory_region(wcnss); if (ret) diff --git a/drivers/remoteproc/qcom_wcnss_iris.c b/drivers/remoteproc/qcom_wcnss_iris.c index 0e0ae1e764ea..169acd305ae3 100644 --- a/drivers/remoteproc/qcom_wcnss_iris.c +++ b/drivers/remoteproc/qcom_wcnss_iris.c @@ -160,6 +160,7 @@ static int qcom_iris_remove(struct platform_device *pdev) static const struct of_device_id iris_of_match[] = { { .compatible = "qcom,wcn3620", .data = &wcn3620_data }, { .compatible = "qcom,wcn3660", .data = &wcn3660_data }, + { .compatible = "qcom,wcn3660b", .data = &wcn3680_data }, { .compatible = "qcom,wcn3680", .data = &wcn3680_data }, {} }; diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 2394eef383e3..ab150765d124 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -1988,7 +1988,7 @@ int rproc_set_firmware(struct rproc *rproc, const char *fw_name) goto out; } - kfree(rproc->firmware); + kfree_const(rproc->firmware); rproc->firmware = p; out: diff --git a/drivers/remoteproc/stm32_rproc.c b/drivers/remoteproc/stm32_rproc.c index a180aeae9675..ccb3c14a0023 100644 --- a/drivers/remoteproc/stm32_rproc.c +++ b/drivers/remoteproc/stm32_rproc.c @@ -370,8 +370,13 @@ static int stm32_rproc_request_mbox(struct rproc *rproc) ddata->mb[i].chan = mbox_request_channel_byname(cl, name); if (IS_ERR(ddata->mb[i].chan)) { - if (PTR_ERR(ddata->mb[i].chan) == -EPROBE_DEFER) + if (PTR_ERR(ddata->mb[i].chan) == -EPROBE_DEFER) { + dev_err_probe(dev->parent, + PTR_ERR(ddata->mb[i].chan), + "failed to request mailbox %s\n", + name); goto err_probe; + } dev_warn(dev, "cannot get %s mbox\n", name); ddata->mb[i].chan = NULL; } @@ -592,15 +597,14 @@ static int stm32_rproc_parse_dt(struct platform_device *pdev, irq = platform_get_irq(pdev, 0); if (irq == -EPROBE_DEFER) - return -EPROBE_DEFER; + return dev_err_probe(dev, irq, "failed to get interrupt\n"); if (irq > 0) { err = devm_request_irq(dev, irq, stm32_rproc_wdg, 0, dev_name(dev), pdev); - if (err) { - dev_err(dev, "failed to request wdg irq\n"); - return err; - } + if (err) + return dev_err_probe(dev, err, + "failed to request wdg irq\n"); ddata->wdg_irq = irq; @@ -613,10 +617,9 @@ static int stm32_rproc_parse_dt(struct platform_device *pdev, } ddata->rst = devm_reset_control_get_by_index(dev, 0); - if (IS_ERR(ddata->rst)) { - dev_err(dev, "failed to get mcu reset\n"); - return PTR_ERR(ddata->rst); - } + if (IS_ERR(ddata->rst)) + return dev_err_probe(dev, PTR_ERR(ddata->rst), + "failed to get mcu_reset\n"); /* * if platform is secured the hold boot bit must be written by diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig index 8ac5627564f0..4171c6f76385 100644 --- a/drivers/reset/Kconfig +++ b/drivers/reset/Kconfig @@ -89,6 +89,16 @@ config RESET_INTEL_GW Say Y to control the reset signals provided by reset controller. Otherwise, say N. +config RESET_K210 + bool "Reset controller driver for Canaan Kendryte K210 SoC" + depends on (SOC_CANAAN || COMPILE_TEST) && OF + select MFD_SYSCON + default SOC_CANAAN + help + Support for the Canaan Kendryte K210 RISC-V SoC reset controller. + Say Y if you want to control reset signals provided by this + controller. + config RESET_LANTIQ bool "Lantiq XWAY Reset Driver" if COMPILE_TEST default SOC_TYPE_XWAY diff --git a/drivers/reset/Makefile b/drivers/reset/Makefile index 1054123fd187..65a118a91b27 100644 --- a/drivers/reset/Makefile +++ b/drivers/reset/Makefile @@ -13,6 +13,7 @@ obj-$(CONFIG_RESET_BRCMSTB_RESCAL) += reset-brcmstb-rescal.o obj-$(CONFIG_RESET_HSDK) += reset-hsdk.o obj-$(CONFIG_RESET_IMX7) += reset-imx7.o obj-$(CONFIG_RESET_INTEL_GW) += reset-intel-gw.o +obj-$(CONFIG_RESET_K210) += reset-k210.o obj-$(CONFIG_RESET_LANTIQ) += reset-lantiq.o obj-$(CONFIG_RESET_LPC18XX) += reset-lpc18xx.o obj-$(CONFIG_RESET_MESON) += reset-meson.o diff --git a/drivers/reset/reset-k210.c b/drivers/reset/reset-k210.c new file mode 100644 index 000000000000..1b6e03522b40 --- /dev/null +++ b/drivers/reset/reset-k210.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020 Western Digital Corporation or its affiliates. + */ +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/reset-controller.h> +#include <linux/delay.h> +#include <linux/mfd/syscon.h> +#include <linux/regmap.h> +#include <soc/canaan/k210-sysctl.h> + +#include <dt-bindings/reset/k210-rst.h> + +#define K210_RST_MASK 0x27FFFFFF + +struct k210_rst { + struct regmap *map; + struct reset_controller_dev rcdev; +}; + +static inline struct k210_rst * +to_k210_rst(struct reset_controller_dev *rcdev) +{ + return container_of(rcdev, struct k210_rst, rcdev); +} + +static inline int k210_rst_assert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct k210_rst *ksr = to_k210_rst(rcdev); + + return regmap_update_bits(ksr->map, K210_SYSCTL_PERI_RESET, BIT(id), 1); +} + +static inline int k210_rst_deassert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct k210_rst *ksr = to_k210_rst(rcdev); + + return regmap_update_bits(ksr->map, K210_SYSCTL_PERI_RESET, BIT(id), 0); +} + +static int k210_rst_reset(struct reset_controller_dev *rcdev, + unsigned long id) +{ + int ret; + + ret = k210_rst_assert(rcdev, id); + if (ret == 0) { + udelay(10); + ret = k210_rst_deassert(rcdev, id); + } + + return ret; +} + +static int k210_rst_status(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct k210_rst *ksr = to_k210_rst(rcdev); + u32 reg, bit = BIT(id); + int ret; + + ret = regmap_read(ksr->map, K210_SYSCTL_PERI_RESET, ®); + if (ret) + return ret; + + return reg & bit; +} + +static int k210_rst_xlate(struct reset_controller_dev *rcdev, + const struct of_phandle_args *reset_spec) +{ + unsigned long id = reset_spec->args[0]; + + if (!(BIT(id) & K210_RST_MASK)) + return -EINVAL; + + return id; +} + +static const struct reset_control_ops k210_rst_ops = { + .assert = k210_rst_assert, + .deassert = k210_rst_deassert, + .reset = k210_rst_reset, + .status = k210_rst_status, +}; + +static int k210_rst_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *parent_np = of_get_parent(dev->of_node); + struct k210_rst *ksr; + + dev_info(dev, "K210 reset controller\n"); + + ksr = devm_kzalloc(dev, sizeof(*ksr), GFP_KERNEL); + if (!ksr) + return -ENOMEM; + + ksr->map = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); + if (IS_ERR(ksr->map)) + return PTR_ERR(ksr->map); + + ksr->rcdev.owner = THIS_MODULE; + ksr->rcdev.dev = dev; + ksr->rcdev.of_node = dev->of_node; + ksr->rcdev.ops = &k210_rst_ops; + ksr->rcdev.nr_resets = fls(K210_RST_MASK); + ksr->rcdev.of_reset_n_cells = 1; + ksr->rcdev.of_xlate = k210_rst_xlate; + + return devm_reset_controller_register(dev, &ksr->rcdev); +} + +static const struct of_device_id k210_rst_dt_ids[] = { + { .compatible = "canaan,k210-rst" }, + { /* sentinel */ }, +}; + +static struct platform_driver k210_rst_driver = { + .probe = k210_rst_probe, + .driver = { + .name = "k210-rst", + .of_match_table = k210_rst_dt_ids, + }, +}; +builtin_platform_driver(k210_rst_driver); diff --git a/drivers/rpmsg/qcom_glink_ssr.c b/drivers/rpmsg/qcom_glink_ssr.c index dcd1ce616974..dea929c6045d 100644 --- a/drivers/rpmsg/qcom_glink_ssr.c +++ b/drivers/rpmsg/qcom_glink_ssr.c @@ -8,15 +8,16 @@ #include <linux/module.h> #include <linux/notifier.h> #include <linux/rpmsg.h> +#include <linux/rpmsg/qcom_glink.h> #include <linux/remoteproc/qcom_rproc.h> /** * struct do_cleanup_msg - The data structure for an SSR do_cleanup message - * version: The G-Link SSR protocol version - * command: The G-Link SSR command - do_cleanup - * seq_num: Sequence number - * name_len: Length of the name of the subsystem being restarted - * name: G-Link edge name of the subsystem being restarted + * @version: The G-Link SSR protocol version + * @command: The G-Link SSR command - do_cleanup + * @seq_num: Sequence number + * @name_len: Length of the name of the subsystem being restarted + * @name: G-Link edge name of the subsystem being restarted */ struct do_cleanup_msg { __le32 version; @@ -28,9 +29,9 @@ struct do_cleanup_msg { /** * struct cleanup_done_msg - The data structure for an SSR cleanup_done message - * version: The G-Link SSR protocol version - * response: The G-Link SSR response to a do_cleanup command, cleanup_done - * seq_num: Sequence number + * @version: The G-Link SSR protocol version + * @response: The G-Link SSR response to a do_cleanup command, cleanup_done + * @seq_num: Sequence number */ struct cleanup_done_msg { __le32 version; diff --git a/drivers/rtc/rtc-m41t80.c b/drivers/rtc/rtc-m41t80.c index cce5b5284658..89128fc29ccc 100644 --- a/drivers/rtc/rtc-m41t80.c +++ b/drivers/rtc/rtc-m41t80.c @@ -785,7 +785,7 @@ static long wdt_unlocked_ioctl(struct file *file, unsigned int cmd, */ static int wdt_open(struct inode *inode, struct file *file) { - if (MINOR(inode->i_rdev) == WATCHDOG_MINOR) { + if (iminor(inode) == WATCHDOG_MINOR) { mutex_lock(&m41t80_rtc_mutex); if (test_and_set_bit(0, &wdt_is_open)) { mutex_unlock(&m41t80_rtc_mutex); @@ -809,7 +809,7 @@ static int wdt_open(struct inode *inode, struct file *file) */ static int wdt_release(struct inode *inode, struct file *file) { - if (MINOR(inode->i_rdev) == WATCHDOG_MINOR) + if (iminor(inode) == WATCHDOG_MINOR) clear_bit(0, &wdt_is_open); return 0; } diff --git a/drivers/s390/char/vmur.c b/drivers/s390/char/vmur.c index 1bbf27b98cf6..68f49e2e964c 100644 --- a/drivers/s390/char/vmur.c +++ b/drivers/s390/char/vmur.c @@ -681,7 +681,7 @@ static int ur_open(struct inode *inode, struct file *file) * We treat the minor number as the devno of the ur device * to find in the driver tree. */ - devno = MINOR(file_inode(file)->i_rdev); + devno = iminor(file_inode(file)); urd = urdev_get_from_devno(devno); if (!urd) { diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 5730572b52cd..54e686dca6de 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -117,7 +117,7 @@ struct virtio_rev_info { }; /* the highest virtio-ccw revision we support */ -#define VIRTIO_CCW_REV_MAX 1 +#define VIRTIO_CCW_REV_MAX 2 struct virtio_ccw_vq_info { struct virtqueue *vq; @@ -952,7 +952,7 @@ static u8 virtio_ccw_get_status(struct virtio_device *vdev) u8 old_status = vcdev->dma_area->status; struct ccw1 *ccw; - if (vcdev->revision < 1) + if (vcdev->revision < 2) return vcdev->dma_area->status; ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw)); diff --git a/drivers/sbus/char/display7seg.c b/drivers/sbus/char/display7seg.c index fad936eb845f..00e72b97d0b6 100644 --- a/drivers/sbus/char/display7seg.c +++ b/drivers/sbus/char/display7seg.c @@ -186,7 +186,7 @@ static int d7s_probe(struct platform_device *op) p->regs = of_ioremap(&op->resource[0], 0, sizeof(u8), "d7s"); if (!p->regs) { printk(KERN_ERR PFX "Cannot map chip registers\n"); - goto out_free; + goto out; } err = misc_register(&d7s_miscdev); @@ -228,8 +228,6 @@ out: out_iounmap: of_iounmap(&op->resource[0], p->regs, sizeof(u8)); - -out_free: goto out; } diff --git a/drivers/scsi/aic7xxx/aic79xx.h b/drivers/scsi/aic7xxx/aic79xx.h index 77e1d6bb59a3..59e9321815c8 100644 --- a/drivers/scsi/aic7xxx/aic79xx.h +++ b/drivers/scsi/aic7xxx/aic79xx.h @@ -1175,7 +1175,7 @@ struct ahd_softc { uint8_t tqinfifonext; /* - * Cached verson of the hs_mailbox so we can avoid + * Cached version of the hs_mailbox so we can avoid * pausing the sequencer during mailbox updates. */ uint8_t hs_mailbox; diff --git a/drivers/scsi/aic7xxx/aic7xxx.h b/drivers/scsi/aic7xxx/aic7xxx.h index 11a09798e6b5..9bc755a0a2d3 100644 --- a/drivers/scsi/aic7xxx/aic7xxx.h +++ b/drivers/scsi/aic7xxx/aic7xxx.h @@ -896,8 +896,6 @@ union ahc_bus_softc { typedef void (*ahc_bus_intr_t)(struct ahc_softc *); typedef int (*ahc_bus_chip_init_t)(struct ahc_softc *); -typedef int (*ahc_bus_suspend_t)(struct ahc_softc *); -typedef int (*ahc_bus_resume_t)(struct ahc_softc *); typedef void ahc_callback_t (void *); struct ahc_softc { diff --git a/drivers/scsi/bnx2fc/Kconfig b/drivers/scsi/bnx2fc/Kconfig index 3cf7e08df809..ecdc0f0f4f4e 100644 --- a/drivers/scsi/bnx2fc/Kconfig +++ b/drivers/scsi/bnx2fc/Kconfig @@ -5,6 +5,7 @@ config SCSI_BNX2X_FCOE depends on (IPV6 || IPV6=n) depends on LIBFC depends on LIBFCOE + depends on MMU select NETDEVICES select ETHERNET select NET_VENDOR_BROADCOM diff --git a/drivers/scsi/bnx2i/bnx2i_iscsi.c b/drivers/scsi/bnx2i/bnx2i_iscsi.c index fdd446765311..1e6d8f62ea3c 100644 --- a/drivers/scsi/bnx2i/bnx2i_iscsi.c +++ b/drivers/scsi/bnx2i/bnx2i_iscsi.c @@ -1171,10 +1171,8 @@ static void bnx2i_cleanup_task(struct iscsi_task *task) bnx2i_send_cmd_cleanup_req(hba, task->dd_data); spin_unlock_bh(&conn->session->back_lock); - spin_unlock_bh(&conn->session->frwd_lock); wait_for_completion_timeout(&bnx2i_conn->cmd_cleanup_cmpl, msecs_to_jiffies(ISCSI_CMD_CLEANUP_TIMEOUT)); - spin_lock_bh(&conn->session->frwd_lock); spin_lock_bh(&conn->session->back_lock); } bnx2i_iscsi_unmap_sg_list(task->dd_data); diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c index 337d3aa91945..38369766511c 100644 --- a/drivers/scsi/hpsa.c +++ b/drivers/scsi/hpsa.c @@ -1151,7 +1151,10 @@ static void __enqueue_cmd_and_start_io(struct ctlr_info *h, { dial_down_lockup_detection_during_fw_flash(h, c); atomic_inc(&h->commands_outstanding); - if (c->device) + /* + * Check to see if the command is being retried. + */ + if (c->device && !c->retry_pending) atomic_inc(&c->device->commands_outstanding); reply_queue = h->reply_map[raw_smp_processor_id()]; @@ -5567,7 +5570,8 @@ static inline void hpsa_cmd_partial_init(struct ctlr_info *h, int index, } static int hpsa_ioaccel_submit(struct ctlr_info *h, - struct CommandList *c, struct scsi_cmnd *cmd) + struct CommandList *c, struct scsi_cmnd *cmd, + bool retry) { struct hpsa_scsi_dev_t *dev = cmd->device->hostdata; int rc = IO_ACCEL_INELIGIBLE; @@ -5584,18 +5588,22 @@ static int hpsa_ioaccel_submit(struct ctlr_info *h, cmd->host_scribble = (unsigned char *) c; if (dev->offload_enabled) { - hpsa_cmd_init(h, c->cmdindex, c); + hpsa_cmd_init(h, c->cmdindex, c); /* Zeroes out all fields */ c->cmd_type = CMD_SCSI; c->scsi_cmd = cmd; c->device = dev; + if (retry) /* Resubmit but do not increment device->commands_outstanding. */ + c->retry_pending = true; rc = hpsa_scsi_ioaccel_raid_map(h, c); if (rc < 0) /* scsi_dma_map failed. */ rc = SCSI_MLQUEUE_HOST_BUSY; } else if (dev->hba_ioaccel_enabled) { - hpsa_cmd_init(h, c->cmdindex, c); + hpsa_cmd_init(h, c->cmdindex, c); /* Zeroes out all fields */ c->cmd_type = CMD_SCSI; c->scsi_cmd = cmd; c->device = dev; + if (retry) /* Resubmit but do not increment device->commands_outstanding. */ + c->retry_pending = true; rc = hpsa_scsi_ioaccel_direct_map(h, c); if (rc < 0) /* scsi_dma_map failed. */ rc = SCSI_MLQUEUE_HOST_BUSY; @@ -5628,7 +5636,8 @@ static void hpsa_command_resubmit_worker(struct work_struct *work) if (c2->error_data.serv_response == IOACCEL2_STATUS_SR_TASK_COMP_SET_FULL) { - rc = hpsa_ioaccel_submit(h, c, cmd); + /* Resubmit with the retry_pending flag set. */ + rc = hpsa_ioaccel_submit(h, c, cmd, true); if (rc == 0) return; if (rc == SCSI_MLQUEUE_HOST_BUSY) { @@ -5644,6 +5653,15 @@ static void hpsa_command_resubmit_worker(struct work_struct *work) } } hpsa_cmd_partial_init(c->h, c->cmdindex, c); + /* + * Here we have not come in though queue_command, so we + * can set the retry_pending flag to true for a driver initiated + * retry attempt (I.E. not a SML retry). + * I.E. We are submitting a driver initiated retry. + * Note: hpsa_ciss_submit does not zero out the command fields like + * ioaccel submit does. + */ + c->retry_pending = true; if (hpsa_ciss_submit(c->h, c, cmd, dev)) { /* * If we get here, it means dma mapping failed. Try @@ -5706,11 +5724,16 @@ static int hpsa_scsi_queue_command(struct Scsi_Host *sh, struct scsi_cmnd *cmd) /* * Call alternate submit routine for I/O accelerated commands. * Retries always go down the normal I/O path. + * Note: If cmd->retries is non-zero, then this is a SML + * initiated retry and not a driver initiated retry. + * This command has been obtained from cmd_tagged_alloc + * and is therefore a brand-new command. */ if (likely(cmd->retries == 0 && !blk_rq_is_passthrough(cmd->request) && h->acciopath_status)) { - rc = hpsa_ioaccel_submit(h, c, cmd); + /* Submit with the retry_pending flag unset. */ + rc = hpsa_ioaccel_submit(h, c, cmd, false); if (rc == 0) return 0; if (rc == SCSI_MLQUEUE_HOST_BUSY) { @@ -6105,6 +6128,7 @@ return_reset_status: * at init, and managed by cmd_tagged_alloc() and cmd_tagged_free() using the * block request tag as an index into a table of entries. cmd_tagged_free() is * the complement, although cmd_free() may be called instead. + * This function is only called for new requests from queue_command. */ static struct CommandList *cmd_tagged_alloc(struct ctlr_info *h, struct scsi_cmnd *scmd) @@ -6139,8 +6163,14 @@ static struct CommandList *cmd_tagged_alloc(struct ctlr_info *h, } atomic_inc(&c->refcount); - hpsa_cmd_partial_init(h, idx, c); + + /* + * This is a new command obtained from queue_command so + * there have not been any driver initiated retry attempts. + */ + c->retry_pending = false; + return c; } @@ -6208,6 +6238,13 @@ static struct CommandList *cmd_alloc(struct ctlr_info *h) } hpsa_cmd_partial_init(h, i, c); c->device = NULL; + + /* + * cmd_alloc is for "internal" commands and they are never + * retried. + */ + c->retry_pending = false; + return c; } diff --git a/drivers/scsi/hpsa_cmd.h b/drivers/scsi/hpsa_cmd.h index 46df2e3ff89b..d126bb877250 100644 --- a/drivers/scsi/hpsa_cmd.h +++ b/drivers/scsi/hpsa_cmd.h @@ -448,7 +448,7 @@ struct CommandList { */ struct hpsa_scsi_dev_t *phys_disk; - int abort_pending; + bool retry_pending; struct hpsa_scsi_dev_t *device; atomic_t refcount; /* Must be last to avoid memset in hpsa_cmd_init() */ } __aligned(COMMANDLIST_ALIGNMENT); diff --git a/drivers/scsi/isci/request.c b/drivers/scsi/isci/request.c index bee16850b236..58e62162882f 100644 --- a/drivers/scsi/isci/request.c +++ b/drivers/scsi/isci/request.c @@ -1480,8 +1480,6 @@ static enum sci_status stp_request_pio_await_h2d_completion_tc_event(struct isci_request *ireq, u32 completion_code) { - enum sci_status status = SCI_SUCCESS; - switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) { case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD): ireq->scu_status = SCU_TASK_DONE_GOOD; @@ -1500,7 +1498,7 @@ stp_request_pio_await_h2d_completion_tc_event(struct isci_request *ireq, break; } - return status; + return SCI_SUCCESS; } static enum sci_status @@ -2152,8 +2150,6 @@ static enum sci_status stp_request_udma_await_tc_event(struct isci_request *ireq static enum sci_status atapi_raw_completion(struct isci_request *ireq, u32 completion_code, enum sci_base_request_states next) { - enum sci_status status = SCI_SUCCESS; - switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) { case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD): ireq->scu_status = SCU_TASK_DONE_GOOD; @@ -2172,7 +2168,7 @@ static enum sci_status atapi_raw_completion(struct isci_request *ireq, u32 compl break; } - return status; + return SCI_SUCCESS; } static enum sci_status atapi_data_tc_completion_handler(struct isci_request *ireq, diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c index a9ce6298b935..dd33ce0e3737 100644 --- a/drivers/scsi/iscsi_tcp.c +++ b/drivers/scsi/iscsi_tcp.c @@ -847,6 +847,7 @@ iscsi_sw_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max, struct iscsi_session *session; struct iscsi_sw_tcp_host *tcp_sw_host; struct Scsi_Host *shost; + int rc; if (ep) { printk(KERN_ERR "iscsi_tcp: invalid ep %p.\n", ep); @@ -864,6 +865,11 @@ iscsi_sw_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max, shost->max_channel = 0; shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE; + rc = iscsi_host_get_max_scsi_cmds(shost, cmds_max); + if (rc < 0) + goto free_host; + shost->can_queue = rc; + if (iscsi_host_add(shost, NULL)) goto free_host; @@ -878,7 +884,6 @@ iscsi_sw_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max, tcp_sw_host = iscsi_host_priv(shost); tcp_sw_host->session = session; - shost->can_queue = session->scsi_cmds_max; if (iscsi_tcp_r2tpool_alloc(session)) goto remove_session; return cls_session; @@ -981,7 +986,7 @@ static struct scsi_host_template iscsi_sw_tcp_sht = { .name = "iSCSI Initiator over TCP/IP", .queuecommand = iscsi_queuecommand, .change_queue_depth = scsi_change_queue_depth, - .can_queue = ISCSI_DEF_XMIT_CMDS_MAX - 1, + .can_queue = ISCSI_TOTAL_CMDS_MAX, .sg_tablesize = 4096, .max_sectors = 0xFFFF, .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c index 4e668aafbcca..04633e5157e9 100644 --- a/drivers/scsi/libiscsi.c +++ b/drivers/scsi/libiscsi.c @@ -523,16 +523,6 @@ static void iscsi_complete_task(struct iscsi_task *task, int state) WARN_ON_ONCE(task->state == ISCSI_TASK_FREE); task->state = state; - spin_lock_bh(&conn->taskqueuelock); - if (!list_empty(&task->running)) { - pr_debug_once("%s while task on list", __func__); - list_del_init(&task->running); - } - spin_unlock_bh(&conn->taskqueuelock); - - if (conn->task == task) - conn->task = NULL; - if (READ_ONCE(conn->ping_task) == task) WRITE_ONCE(conn->ping_task, NULL); @@ -564,11 +554,41 @@ void iscsi_complete_scsi_task(struct iscsi_task *task, } EXPORT_SYMBOL_GPL(iscsi_complete_scsi_task); +/* + * Must be called with back and frwd lock + */ +static bool cleanup_queued_task(struct iscsi_task *task) +{ + struct iscsi_conn *conn = task->conn; + bool early_complete = false; + + /* Bad target might have completed task while it was still running */ + if (task->state == ISCSI_TASK_COMPLETED) + early_complete = true; + + if (!list_empty(&task->running)) { + list_del_init(&task->running); + /* + * If it's on a list but still running, this could be from + * a bad target sending a rsp early, cleanup from a TMF, or + * session recovery. + */ + if (task->state == ISCSI_TASK_RUNNING || + task->state == ISCSI_TASK_COMPLETED) + __iscsi_put_task(task); + } + + if (conn->task == task) { + conn->task = NULL; + __iscsi_put_task(task); + } + + return early_complete; +} /* - * session back_lock must be held and if not called for a task that is - * still pending or from the xmit thread, then xmit thread must - * be suspended. + * session frwd lock must be held and if not called for a task that is still + * pending or from the xmit thread, then xmit thread must be suspended */ static void fail_scsi_task(struct iscsi_task *task, int err) { @@ -576,14 +596,11 @@ static void fail_scsi_task(struct iscsi_task *task, int err) struct scsi_cmnd *sc; int state; - /* - * if a command completes and we get a successful tmf response - * we will hit this because the scsi eh abort code does not take - * a ref to the task. - */ - sc = task->sc; - if (!sc) + spin_lock_bh(&conn->session->back_lock); + if (cleanup_queued_task(task)) { + spin_unlock_bh(&conn->session->back_lock); return; + } if (task->state == ISCSI_TASK_PENDING) { /* @@ -598,11 +615,9 @@ static void fail_scsi_task(struct iscsi_task *task, int err) else state = ISCSI_TASK_ABRT_TMF; + sc = task->sc; sc->result = err << 16; scsi_set_resid(sc, scsi_bufflen(sc)); - - /* regular RX path uses back_lock */ - spin_lock_bh(&conn->session->back_lock); iscsi_complete_task(task, state); spin_unlock_bh(&conn->session->back_lock); } @@ -748,9 +763,7 @@ __iscsi_conn_send_pdu(struct iscsi_conn *conn, struct iscsi_hdr *hdr, if (session->tt->xmit_task(task)) goto free_task; } else { - spin_lock_bh(&conn->taskqueuelock); list_add_tail(&task->running, &conn->mgmtqueue); - spin_unlock_bh(&conn->taskqueuelock); iscsi_conn_queue_work(conn); } @@ -1411,31 +1424,61 @@ static int iscsi_check_cmdsn_window_closed(struct iscsi_conn *conn) return 0; } -static int iscsi_xmit_task(struct iscsi_conn *conn) +static int iscsi_xmit_task(struct iscsi_conn *conn, struct iscsi_task *task, + bool was_requeue) { - struct iscsi_task *task = conn->task; int rc; - if (test_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx)) - return -ENODATA; - spin_lock_bh(&conn->session->back_lock); - if (conn->task == NULL) { + + if (!conn->task) { + /* Take a ref so we can access it after xmit_task() */ + __iscsi_get_task(task); + } else { + /* Already have a ref from when we failed to send it last call */ + conn->task = NULL; + } + + /* + * If this was a requeue for a R2T we have an extra ref on the task in + * case a bad target sends a cmd rsp before we have handled the task. + */ + if (was_requeue) + __iscsi_put_task(task); + + /* + * Do this after dropping the extra ref because if this was a requeue + * it's removed from that list and cleanup_queued_task would miss it. + */ + if (test_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx)) { + /* + * Save the task and ref in case we weren't cleaning up this + * task and get woken up again. + */ + conn->task = task; spin_unlock_bh(&conn->session->back_lock); return -ENODATA; } - __iscsi_get_task(task); spin_unlock_bh(&conn->session->back_lock); + spin_unlock_bh(&conn->session->frwd_lock); rc = conn->session->tt->xmit_task(task); spin_lock_bh(&conn->session->frwd_lock); if (!rc) { /* done with this task */ task->last_xfer = jiffies; - conn->task = NULL; } /* regular RX path uses back_lock */ spin_lock(&conn->session->back_lock); + if (rc && task->state == ISCSI_TASK_RUNNING) { + /* + * get an extra ref that is released next time we access it + * as conn->task above. + */ + __iscsi_get_task(task); + conn->task = task; + } + __iscsi_put_task(task); spin_unlock(&conn->session->back_lock); return rc; @@ -1445,9 +1488,7 @@ static int iscsi_xmit_task(struct iscsi_conn *conn) * iscsi_requeue_task - requeue task to run from session workqueue * @task: task to requeue * - * LLDs that need to run a task from the session workqueue should call - * this. The session frwd_lock must be held. This should only be called - * by software drivers. + * Callers must have taken a ref to the task that is going to be requeued. */ void iscsi_requeue_task(struct iscsi_task *task) { @@ -1457,11 +1498,18 @@ void iscsi_requeue_task(struct iscsi_task *task) * this may be on the requeue list already if the xmit_task callout * is handling the r2ts while we are adding new ones */ - spin_lock_bh(&conn->taskqueuelock); - if (list_empty(&task->running)) + spin_lock_bh(&conn->session->frwd_lock); + if (list_empty(&task->running)) { list_add_tail(&task->running, &conn->requeue); - spin_unlock_bh(&conn->taskqueuelock); + } else { + /* + * Don't need the extra ref since it's already requeued and + * has a ref. + */ + iscsi_put_task(task); + } iscsi_conn_queue_work(conn); + spin_unlock_bh(&conn->session->frwd_lock); } EXPORT_SYMBOL_GPL(iscsi_requeue_task); @@ -1487,7 +1535,7 @@ static int iscsi_data_xmit(struct iscsi_conn *conn) } if (conn->task) { - rc = iscsi_xmit_task(conn); + rc = iscsi_xmit_task(conn, conn->task, false); if (rc) goto done; } @@ -1497,54 +1545,41 @@ static int iscsi_data_xmit(struct iscsi_conn *conn) * only have one nop-out as a ping from us and targets should not * overflow us with nop-ins */ - spin_lock_bh(&conn->taskqueuelock); check_mgmt: while (!list_empty(&conn->mgmtqueue)) { - conn->task = list_entry(conn->mgmtqueue.next, - struct iscsi_task, running); - list_del_init(&conn->task->running); - spin_unlock_bh(&conn->taskqueuelock); - if (iscsi_prep_mgmt_task(conn, conn->task)) { + task = list_entry(conn->mgmtqueue.next, struct iscsi_task, + running); + list_del_init(&task->running); + if (iscsi_prep_mgmt_task(conn, task)) { /* regular RX path uses back_lock */ spin_lock_bh(&conn->session->back_lock); - __iscsi_put_task(conn->task); + __iscsi_put_task(task); spin_unlock_bh(&conn->session->back_lock); - conn->task = NULL; - spin_lock_bh(&conn->taskqueuelock); continue; } - rc = iscsi_xmit_task(conn); + rc = iscsi_xmit_task(conn, task, false); if (rc) goto done; - spin_lock_bh(&conn->taskqueuelock); } /* process pending command queue */ while (!list_empty(&conn->cmdqueue)) { - conn->task = list_entry(conn->cmdqueue.next, struct iscsi_task, - running); - list_del_init(&conn->task->running); - spin_unlock_bh(&conn->taskqueuelock); + task = list_entry(conn->cmdqueue.next, struct iscsi_task, + running); + list_del_init(&task->running); if (conn->session->state == ISCSI_STATE_LOGGING_OUT) { - fail_scsi_task(conn->task, DID_IMM_RETRY); - spin_lock_bh(&conn->taskqueuelock); + fail_scsi_task(task, DID_IMM_RETRY); continue; } - rc = iscsi_prep_scsi_cmd_pdu(conn->task); + rc = iscsi_prep_scsi_cmd_pdu(task); if (rc) { - if (rc == -ENOMEM || rc == -EACCES) { - spin_lock_bh(&conn->taskqueuelock); - list_add_tail(&conn->task->running, - &conn->cmdqueue); - conn->task = NULL; - spin_unlock_bh(&conn->taskqueuelock); - goto done; - } else - fail_scsi_task(conn->task, DID_ABORT); - spin_lock_bh(&conn->taskqueuelock); + if (rc == -ENOMEM || rc == -EACCES) + fail_scsi_task(task, DID_IMM_RETRY); + else + fail_scsi_task(task, DID_ABORT); continue; } - rc = iscsi_xmit_task(conn); + rc = iscsi_xmit_task(conn, task, false); if (rc) goto done; /* @@ -1552,7 +1587,6 @@ check_mgmt: * we need to check the mgmt queue for nops that need to * be sent to aviod starvation */ - spin_lock_bh(&conn->taskqueuelock); if (!list_empty(&conn->mgmtqueue)) goto check_mgmt; } @@ -1566,21 +1600,17 @@ check_mgmt: task = list_entry(conn->requeue.next, struct iscsi_task, running); + if (iscsi_check_tmf_restrictions(task, ISCSI_OP_SCSI_DATA_OUT)) break; - conn->task = task; - list_del_init(&conn->task->running); - conn->task->state = ISCSI_TASK_RUNNING; - spin_unlock_bh(&conn->taskqueuelock); - rc = iscsi_xmit_task(conn); + list_del_init(&task->running); + rc = iscsi_xmit_task(conn, task, true); if (rc) goto done; - spin_lock_bh(&conn->taskqueuelock); if (!list_empty(&conn->mgmtqueue)) goto check_mgmt; } - spin_unlock_bh(&conn->taskqueuelock); spin_unlock_bh(&conn->session->frwd_lock); return -ENODATA; @@ -1746,9 +1776,7 @@ int iscsi_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc) goto prepd_reject; } } else { - spin_lock_bh(&conn->taskqueuelock); list_add_tail(&task->running, &conn->cmdqueue); - spin_unlock_bh(&conn->taskqueuelock); iscsi_conn_queue_work(conn); } @@ -1855,27 +1883,39 @@ static int iscsi_exec_task_mgmt_fn(struct iscsi_conn *conn, } /* - * Fail commands. session lock held and recv side suspended and xmit - * thread flushed + * Fail commands. session frwd lock held and xmit thread flushed. */ static void fail_scsi_tasks(struct iscsi_conn *conn, u64 lun, int error) { + struct iscsi_session *session = conn->session; struct iscsi_task *task; int i; - for (i = 0; i < conn->session->cmds_max; i++) { - task = conn->session->cmds[i]; + spin_lock_bh(&session->back_lock); + for (i = 0; i < session->cmds_max; i++) { + task = session->cmds[i]; if (!task->sc || task->state == ISCSI_TASK_FREE) continue; if (lun != -1 && lun != task->sc->device->lun) continue; - ISCSI_DBG_SESSION(conn->session, + __iscsi_get_task(task); + spin_unlock_bh(&session->back_lock); + + ISCSI_DBG_SESSION(session, "failing sc %p itt 0x%x state %d\n", task->sc, task->itt, task->state); fail_scsi_task(task, error); + + spin_unlock_bh(&session->frwd_lock); + iscsi_put_task(task); + spin_lock_bh(&session->frwd_lock); + + spin_lock_bh(&session->back_lock); } + + spin_unlock_bh(&session->back_lock); } /** @@ -1953,6 +1993,7 @@ enum blk_eh_timer_return iscsi_eh_cmd_timed_out(struct scsi_cmnd *sc) ISCSI_DBG_EH(session, "scsi cmd %p timedout\n", sc); spin_lock_bh(&session->frwd_lock); + spin_lock(&session->back_lock); task = (struct iscsi_task *)sc->SCp.ptr; if (!task) { /* @@ -1960,8 +2001,11 @@ enum blk_eh_timer_return iscsi_eh_cmd_timed_out(struct scsi_cmnd *sc) * so let timeout code complete it now. */ rc = BLK_EH_DONE; + spin_unlock(&session->back_lock); goto done; } + __iscsi_get_task(task); + spin_unlock(&session->back_lock); if (session->state != ISCSI_STATE_LOGGED_IN) { /* @@ -2020,6 +2064,7 @@ enum blk_eh_timer_return iscsi_eh_cmd_timed_out(struct scsi_cmnd *sc) goto done; } + spin_lock(&session->back_lock); for (i = 0; i < conn->session->cmds_max; i++) { running_task = conn->session->cmds[i]; if (!running_task->sc || running_task == task || @@ -2052,10 +2097,12 @@ enum blk_eh_timer_return iscsi_eh_cmd_timed_out(struct scsi_cmnd *sc) "last xfer %lu/%lu. Last check %lu.\n", task->last_xfer, running_task->last_xfer, task->last_timeout); + spin_unlock(&session->back_lock); rc = BLK_EH_RESET_TIMER; goto done; } } + spin_unlock(&session->back_lock); /* Assumes nop timeout is shorter than scsi cmd timeout */ if (task->have_checked_conn) @@ -2077,9 +2124,12 @@ enum blk_eh_timer_return iscsi_eh_cmd_timed_out(struct scsi_cmnd *sc) rc = BLK_EH_RESET_TIMER; done: - if (task) - task->last_timeout = jiffies; spin_unlock_bh(&session->frwd_lock); + + if (task) { + task->last_timeout = jiffies; + iscsi_put_task(task); + } ISCSI_DBG_EH(session, "return %s\n", rc == BLK_EH_RESET_TIMER ? "timer reset" : "shutdown or nh"); return rc; @@ -2187,15 +2237,20 @@ int iscsi_eh_abort(struct scsi_cmnd *sc) conn->eh_abort_cnt++; age = session->age; + spin_lock(&session->back_lock); task = (struct iscsi_task *)sc->SCp.ptr; - ISCSI_DBG_EH(session, "aborting [sc %p itt 0x%x]\n", - sc, task->itt); - - /* task completed before time out */ - if (!task->sc) { + if (!task || !task->sc) { + /* task completed before time out */ ISCSI_DBG_EH(session, "sc completed while abort in progress\n"); - goto success; + + spin_unlock(&session->back_lock); + spin_unlock_bh(&session->frwd_lock); + mutex_unlock(&session->eh_mutex); + return SUCCESS; } + ISCSI_DBG_EH(session, "aborting [sc %p itt 0x%x]\n", sc, task->itt); + __iscsi_get_task(task); + spin_unlock(&session->back_lock); if (task->state == ISCSI_TASK_PENDING) { fail_scsi_task(task, DID_ABORT); @@ -2257,6 +2312,7 @@ success: success_unlocked: ISCSI_DBG_EH(session, "abort success [sc %p itt 0x%x]\n", sc, task->itt); + iscsi_put_task(task); mutex_unlock(&session->eh_mutex); return SUCCESS; @@ -2265,6 +2321,7 @@ failed: failed_unlocked: ISCSI_DBG_EH(session, "abort failed [sc %p itt 0x%x]\n", sc, task ? task->itt : 0); + iscsi_put_task(task); mutex_unlock(&session->eh_mutex); return FAILED; } @@ -2591,6 +2648,56 @@ void iscsi_pool_free(struct iscsi_pool *q) } EXPORT_SYMBOL_GPL(iscsi_pool_free); +int iscsi_host_get_max_scsi_cmds(struct Scsi_Host *shost, + uint16_t requested_cmds_max) +{ + int scsi_cmds, total_cmds = requested_cmds_max; + +check: + if (!total_cmds) + total_cmds = ISCSI_DEF_XMIT_CMDS_MAX; + /* + * The iscsi layer needs some tasks for nop handling and tmfs, + * so the cmds_max must at least be greater than ISCSI_MGMT_CMDS_MAX + * + 1 command for scsi IO. + */ + if (total_cmds < ISCSI_TOTAL_CMDS_MIN) { + printk(KERN_ERR "iscsi: invalid max cmds of %d. Must be a power of two that is at least %d.\n", + total_cmds, ISCSI_TOTAL_CMDS_MIN); + return -EINVAL; + } + + if (total_cmds > ISCSI_TOTAL_CMDS_MAX) { + printk(KERN_INFO "iscsi: invalid max cmds of %d. Must be a power of 2 less than or equal to %d. Using %d.\n", + requested_cmds_max, ISCSI_TOTAL_CMDS_MAX, + ISCSI_TOTAL_CMDS_MAX); + total_cmds = ISCSI_TOTAL_CMDS_MAX; + } + + if (!is_power_of_2(total_cmds)) { + total_cmds = rounddown_pow_of_two(total_cmds); + if (total_cmds < ISCSI_TOTAL_CMDS_MIN) { + printk(KERN_ERR "iscsi: invalid max cmds of %d. Must be a power of 2 greater than %d.\n", requested_cmds_max, ISCSI_TOTAL_CMDS_MIN); + return -EINVAL; + } + + printk(KERN_INFO "iscsi: invalid max cmds %d. Must be a power of 2. Rounding max cmds down to %d.\n", + requested_cmds_max, total_cmds); + } + + scsi_cmds = total_cmds - ISCSI_MGMT_CMDS_MAX; + if (shost->can_queue && scsi_cmds > shost->can_queue) { + total_cmds = shost->can_queue; + + printk(KERN_INFO "iscsi: requested max cmds %u is higher than driver limit. Using driver limit %u\n", + requested_cmds_max, shost->can_queue); + goto check; + } + + return scsi_cmds; +} +EXPORT_SYMBOL_GPL(iscsi_host_get_max_scsi_cmds); + /** * iscsi_host_add - add host to system * @shost: scsi host @@ -2681,8 +2788,6 @@ void iscsi_host_remove(struct Scsi_Host *shost) flush_signals(current); scsi_remove_host(shost); - if (ihost->workq) - destroy_workqueue(ihost->workq); } EXPORT_SYMBOL_GPL(iscsi_host_remove); @@ -2690,6 +2795,9 @@ void iscsi_host_free(struct Scsi_Host *shost) { struct iscsi_host *ihost = shost_priv(shost); + if (ihost->workq) + destroy_workqueue(ihost->workq); + kfree(ihost->netdev); kfree(ihost->hwaddress); kfree(ihost->initiatorname); @@ -2743,7 +2851,7 @@ iscsi_session_setup(struct iscsi_transport *iscsit, struct Scsi_Host *shost, struct iscsi_host *ihost = shost_priv(shost); struct iscsi_session *session; struct iscsi_cls_session *cls_session; - int cmd_i, scsi_cmds, total_cmds = cmds_max; + int cmd_i, scsi_cmds; unsigned long flags; spin_lock_irqsave(&ihost->lock, flags); @@ -2754,37 +2862,9 @@ iscsi_session_setup(struct iscsi_transport *iscsit, struct Scsi_Host *shost, ihost->num_sessions++; spin_unlock_irqrestore(&ihost->lock, flags); - if (!total_cmds) - total_cmds = ISCSI_DEF_XMIT_CMDS_MAX; - /* - * The iscsi layer needs some tasks for nop handling and tmfs, - * so the cmds_max must at least be greater than ISCSI_MGMT_CMDS_MAX - * + 1 command for scsi IO. - */ - if (total_cmds < ISCSI_TOTAL_CMDS_MIN) { - printk(KERN_ERR "iscsi: invalid can_queue of %d. can_queue " - "must be a power of two that is at least %d.\n", - total_cmds, ISCSI_TOTAL_CMDS_MIN); + scsi_cmds = iscsi_host_get_max_scsi_cmds(shost, cmds_max); + if (scsi_cmds < 0) goto dec_session_count; - } - - if (total_cmds > ISCSI_TOTAL_CMDS_MAX) { - printk(KERN_ERR "iscsi: invalid can_queue of %d. can_queue " - "must be a power of 2 less than or equal to %d.\n", - cmds_max, ISCSI_TOTAL_CMDS_MAX); - total_cmds = ISCSI_TOTAL_CMDS_MAX; - } - - if (!is_power_of_2(total_cmds)) { - printk(KERN_ERR "iscsi: invalid can_queue of %d. can_queue " - "must be a power of 2.\n", total_cmds); - total_cmds = rounddown_pow_of_two(total_cmds); - if (total_cmds < ISCSI_TOTAL_CMDS_MIN) - goto dec_session_count; - printk(KERN_INFO "iscsi: Rounding can_queue to %d.\n", - total_cmds); - } - scsi_cmds = total_cmds - ISCSI_MGMT_CMDS_MAX; cls_session = iscsi_alloc_session(shost, iscsit, sizeof(struct iscsi_session) + @@ -2800,7 +2880,7 @@ iscsi_session_setup(struct iscsi_transport *iscsit, struct Scsi_Host *shost, session->lu_reset_timeout = 15; session->abort_timeout = 10; session->scsi_cmds_max = scsi_cmds; - session->cmds_max = total_cmds; + session->cmds_max = scsi_cmds + ISCSI_MGMT_CMDS_MAX; session->queued_cmdsn = session->cmdsn = initial_cmdsn; session->exp_cmdsn = initial_cmdsn + 1; session->max_cmdsn = initial_cmdsn + 1; @@ -2919,7 +2999,6 @@ iscsi_conn_setup(struct iscsi_cls_session *cls_session, int dd_size, INIT_LIST_HEAD(&conn->mgmtqueue); INIT_LIST_HEAD(&conn->cmdqueue); INIT_LIST_HEAD(&conn->requeue); - spin_lock_init(&conn->taskqueuelock); INIT_WORK(&conn->xmitwork, iscsi_xmitworker); /* allocate login_task used for the login/text sequences */ @@ -3085,10 +3164,16 @@ fail_mgmt_tasks(struct iscsi_session *session, struct iscsi_conn *conn) ISCSI_DBG_SESSION(conn->session, "failing mgmt itt 0x%x state %d\n", task->itt, task->state); + + spin_lock_bh(&session->back_lock); + if (cleanup_queued_task(task)) { + spin_unlock_bh(&session->back_lock); + continue; + } + state = ISCSI_TASK_ABRT_SESS_RECOV; if (task->state == ISCSI_TASK_PENDING) state = ISCSI_TASK_COMPLETED; - spin_lock_bh(&session->back_lock); iscsi_complete_task(task, state); spin_unlock_bh(&session->back_lock); } @@ -3189,6 +3274,13 @@ int iscsi_conn_bind(struct iscsi_cls_session *cls_session, spin_unlock_bh(&session->frwd_lock); /* + * The target could have reduced it's window size between logins, so + * we have to reset max/exp cmdsn so we can see the new values. + */ + spin_lock_bh(&session->back_lock); + session->max_cmdsn = session->exp_cmdsn = session->cmdsn + 1; + spin_unlock_bh(&session->back_lock); + /* * Unblock xmitworker(), Login Phase will pass through. */ clear_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx); @@ -3338,125 +3430,125 @@ int iscsi_session_get_param(struct iscsi_cls_session *cls_session, switch(param) { case ISCSI_PARAM_FAST_ABORT: - len = sprintf(buf, "%d\n", session->fast_abort); + len = sysfs_emit(buf, "%d\n", session->fast_abort); break; case ISCSI_PARAM_ABORT_TMO: - len = sprintf(buf, "%d\n", session->abort_timeout); + len = sysfs_emit(buf, "%d\n", session->abort_timeout); break; case ISCSI_PARAM_LU_RESET_TMO: - len = sprintf(buf, "%d\n", session->lu_reset_timeout); + len = sysfs_emit(buf, "%d\n", session->lu_reset_timeout); break; case ISCSI_PARAM_TGT_RESET_TMO: - len = sprintf(buf, "%d\n", session->tgt_reset_timeout); + len = sysfs_emit(buf, "%d\n", session->tgt_reset_timeout); break; case ISCSI_PARAM_INITIAL_R2T_EN: - len = sprintf(buf, "%d\n", session->initial_r2t_en); + len = sysfs_emit(buf, "%d\n", session->initial_r2t_en); break; case ISCSI_PARAM_MAX_R2T: - len = sprintf(buf, "%hu\n", session->max_r2t); + len = sysfs_emit(buf, "%hu\n", session->max_r2t); break; case ISCSI_PARAM_IMM_DATA_EN: - len = sprintf(buf, "%d\n", session->imm_data_en); + len = sysfs_emit(buf, "%d\n", session->imm_data_en); break; case ISCSI_PARAM_FIRST_BURST: - len = sprintf(buf, "%u\n", session->first_burst); + len = sysfs_emit(buf, "%u\n", session->first_burst); break; case ISCSI_PARAM_MAX_BURST: - len = sprintf(buf, "%u\n", session->max_burst); + len = sysfs_emit(buf, "%u\n", session->max_burst); break; case ISCSI_PARAM_PDU_INORDER_EN: - len = sprintf(buf, "%d\n", session->pdu_inorder_en); + len = sysfs_emit(buf, "%d\n", session->pdu_inorder_en); break; case ISCSI_PARAM_DATASEQ_INORDER_EN: - len = sprintf(buf, "%d\n", session->dataseq_inorder_en); + len = sysfs_emit(buf, "%d\n", session->dataseq_inorder_en); break; case ISCSI_PARAM_DEF_TASKMGMT_TMO: - len = sprintf(buf, "%d\n", session->def_taskmgmt_tmo); + len = sysfs_emit(buf, "%d\n", session->def_taskmgmt_tmo); break; case ISCSI_PARAM_ERL: - len = sprintf(buf, "%d\n", session->erl); + len = sysfs_emit(buf, "%d\n", session->erl); break; case ISCSI_PARAM_TARGET_NAME: - len = sprintf(buf, "%s\n", session->targetname); + len = sysfs_emit(buf, "%s\n", session->targetname); break; case ISCSI_PARAM_TARGET_ALIAS: - len = sprintf(buf, "%s\n", session->targetalias); + len = sysfs_emit(buf, "%s\n", session->targetalias); break; case ISCSI_PARAM_TPGT: - len = sprintf(buf, "%d\n", session->tpgt); + len = sysfs_emit(buf, "%d\n", session->tpgt); break; case ISCSI_PARAM_USERNAME: - len = sprintf(buf, "%s\n", session->username); + len = sysfs_emit(buf, "%s\n", session->username); break; case ISCSI_PARAM_USERNAME_IN: - len = sprintf(buf, "%s\n", session->username_in); + len = sysfs_emit(buf, "%s\n", session->username_in); break; case ISCSI_PARAM_PASSWORD: - len = sprintf(buf, "%s\n", session->password); + len = sysfs_emit(buf, "%s\n", session->password); break; case ISCSI_PARAM_PASSWORD_IN: - len = sprintf(buf, "%s\n", session->password_in); + len = sysfs_emit(buf, "%s\n", session->password_in); break; case ISCSI_PARAM_IFACE_NAME: - len = sprintf(buf, "%s\n", session->ifacename); + len = sysfs_emit(buf, "%s\n", session->ifacename); break; case ISCSI_PARAM_INITIATOR_NAME: - len = sprintf(buf, "%s\n", session->initiatorname); + len = sysfs_emit(buf, "%s\n", session->initiatorname); break; case ISCSI_PARAM_BOOT_ROOT: - len = sprintf(buf, "%s\n", session->boot_root); + len = sysfs_emit(buf, "%s\n", session->boot_root); break; case ISCSI_PARAM_BOOT_NIC: - len = sprintf(buf, "%s\n", session->boot_nic); + len = sysfs_emit(buf, "%s\n", session->boot_nic); break; case ISCSI_PARAM_BOOT_TARGET: - len = sprintf(buf, "%s\n", session->boot_target); + len = sysfs_emit(buf, "%s\n", session->boot_target); break; case ISCSI_PARAM_AUTO_SND_TGT_DISABLE: - len = sprintf(buf, "%u\n", session->auto_snd_tgt_disable); + len = sysfs_emit(buf, "%u\n", session->auto_snd_tgt_disable); break; case ISCSI_PARAM_DISCOVERY_SESS: - len = sprintf(buf, "%u\n", session->discovery_sess); + len = sysfs_emit(buf, "%u\n", session->discovery_sess); break; case ISCSI_PARAM_PORTAL_TYPE: - len = sprintf(buf, "%s\n", session->portal_type); + len = sysfs_emit(buf, "%s\n", session->portal_type); break; case ISCSI_PARAM_CHAP_AUTH_EN: - len = sprintf(buf, "%u\n", session->chap_auth_en); + len = sysfs_emit(buf, "%u\n", session->chap_auth_en); break; case ISCSI_PARAM_DISCOVERY_LOGOUT_EN: - len = sprintf(buf, "%u\n", session->discovery_logout_en); + len = sysfs_emit(buf, "%u\n", session->discovery_logout_en); break; case ISCSI_PARAM_BIDI_CHAP_EN: - len = sprintf(buf, "%u\n", session->bidi_chap_en); + len = sysfs_emit(buf, "%u\n", session->bidi_chap_en); break; case ISCSI_PARAM_DISCOVERY_AUTH_OPTIONAL: - len = sprintf(buf, "%u\n", session->discovery_auth_optional); + len = sysfs_emit(buf, "%u\n", session->discovery_auth_optional); break; case ISCSI_PARAM_DEF_TIME2WAIT: - len = sprintf(buf, "%d\n", session->time2wait); + len = sysfs_emit(buf, "%d\n", session->time2wait); break; case ISCSI_PARAM_DEF_TIME2RETAIN: - len = sprintf(buf, "%d\n", session->time2retain); + len = sysfs_emit(buf, "%d\n", session->time2retain); break; case ISCSI_PARAM_TSID: - len = sprintf(buf, "%u\n", session->tsid); + len = sysfs_emit(buf, "%u\n", session->tsid); break; case ISCSI_PARAM_ISID: - len = sprintf(buf, "%02x%02x%02x%02x%02x%02x\n", + len = sysfs_emit(buf, "%02x%02x%02x%02x%02x%02x\n", session->isid[0], session->isid[1], session->isid[2], session->isid[3], session->isid[4], session->isid[5]); break; case ISCSI_PARAM_DISCOVERY_PARENT_IDX: - len = sprintf(buf, "%u\n", session->discovery_parent_idx); + len = sysfs_emit(buf, "%u\n", session->discovery_parent_idx); break; case ISCSI_PARAM_DISCOVERY_PARENT_TYPE: if (session->discovery_parent_type) - len = sprintf(buf, "%s\n", + len = sysfs_emit(buf, "%s\n", session->discovery_parent_type); else - len = sprintf(buf, "\n"); + len = sysfs_emit(buf, "\n"); break; default: return -ENOSYS; @@ -3488,16 +3580,16 @@ int iscsi_conn_get_addr_param(struct sockaddr_storage *addr, case ISCSI_PARAM_CONN_ADDRESS: case ISCSI_HOST_PARAM_IPADDRESS: if (sin) - len = sprintf(buf, "%pI4\n", &sin->sin_addr.s_addr); + len = sysfs_emit(buf, "%pI4\n", &sin->sin_addr.s_addr); else - len = sprintf(buf, "%pI6\n", &sin6->sin6_addr); + len = sysfs_emit(buf, "%pI6\n", &sin6->sin6_addr); break; case ISCSI_PARAM_CONN_PORT: case ISCSI_PARAM_LOCAL_PORT: if (sin) - len = sprintf(buf, "%hu\n", be16_to_cpu(sin->sin_port)); + len = sysfs_emit(buf, "%hu\n", be16_to_cpu(sin->sin_port)); else - len = sprintf(buf, "%hu\n", + len = sysfs_emit(buf, "%hu\n", be16_to_cpu(sin6->sin6_port)); break; default: @@ -3516,88 +3608,88 @@ int iscsi_conn_get_param(struct iscsi_cls_conn *cls_conn, switch(param) { case ISCSI_PARAM_PING_TMO: - len = sprintf(buf, "%u\n", conn->ping_timeout); + len = sysfs_emit(buf, "%u\n", conn->ping_timeout); break; case ISCSI_PARAM_RECV_TMO: - len = sprintf(buf, "%u\n", conn->recv_timeout); + len = sysfs_emit(buf, "%u\n", conn->recv_timeout); break; case ISCSI_PARAM_MAX_RECV_DLENGTH: - len = sprintf(buf, "%u\n", conn->max_recv_dlength); + len = sysfs_emit(buf, "%u\n", conn->max_recv_dlength); break; case ISCSI_PARAM_MAX_XMIT_DLENGTH: - len = sprintf(buf, "%u\n", conn->max_xmit_dlength); + len = sysfs_emit(buf, "%u\n", conn->max_xmit_dlength); break; case ISCSI_PARAM_HDRDGST_EN: - len = sprintf(buf, "%d\n", conn->hdrdgst_en); + len = sysfs_emit(buf, "%d\n", conn->hdrdgst_en); break; case ISCSI_PARAM_DATADGST_EN: - len = sprintf(buf, "%d\n", conn->datadgst_en); + len = sysfs_emit(buf, "%d\n", conn->datadgst_en); break; case ISCSI_PARAM_IFMARKER_EN: - len = sprintf(buf, "%d\n", conn->ifmarker_en); + len = sysfs_emit(buf, "%d\n", conn->ifmarker_en); break; case ISCSI_PARAM_OFMARKER_EN: - len = sprintf(buf, "%d\n", conn->ofmarker_en); + len = sysfs_emit(buf, "%d\n", conn->ofmarker_en); break; case ISCSI_PARAM_EXP_STATSN: - len = sprintf(buf, "%u\n", conn->exp_statsn); + len = sysfs_emit(buf, "%u\n", conn->exp_statsn); break; case ISCSI_PARAM_PERSISTENT_PORT: - len = sprintf(buf, "%d\n", conn->persistent_port); + len = sysfs_emit(buf, "%d\n", conn->persistent_port); break; case ISCSI_PARAM_PERSISTENT_ADDRESS: - len = sprintf(buf, "%s\n", conn->persistent_address); + len = sysfs_emit(buf, "%s\n", conn->persistent_address); break; case ISCSI_PARAM_STATSN: - len = sprintf(buf, "%u\n", conn->statsn); + len = sysfs_emit(buf, "%u\n", conn->statsn); break; case ISCSI_PARAM_MAX_SEGMENT_SIZE: - len = sprintf(buf, "%u\n", conn->max_segment_size); + len = sysfs_emit(buf, "%u\n", conn->max_segment_size); break; case ISCSI_PARAM_KEEPALIVE_TMO: - len = sprintf(buf, "%u\n", conn->keepalive_tmo); + len = sysfs_emit(buf, "%u\n", conn->keepalive_tmo); break; case ISCSI_PARAM_LOCAL_PORT: - len = sprintf(buf, "%u\n", conn->local_port); + len = sysfs_emit(buf, "%u\n", conn->local_port); break; case ISCSI_PARAM_TCP_TIMESTAMP_STAT: - len = sprintf(buf, "%u\n", conn->tcp_timestamp_stat); + len = sysfs_emit(buf, "%u\n", conn->tcp_timestamp_stat); break; case ISCSI_PARAM_TCP_NAGLE_DISABLE: - len = sprintf(buf, "%u\n", conn->tcp_nagle_disable); + len = sysfs_emit(buf, "%u\n", conn->tcp_nagle_disable); break; case ISCSI_PARAM_TCP_WSF_DISABLE: - len = sprintf(buf, "%u\n", conn->tcp_wsf_disable); + len = sysfs_emit(buf, "%u\n", conn->tcp_wsf_disable); break; case ISCSI_PARAM_TCP_TIMER_SCALE: - len = sprintf(buf, "%u\n", conn->tcp_timer_scale); + len = sysfs_emit(buf, "%u\n", conn->tcp_timer_scale); break; case ISCSI_PARAM_TCP_TIMESTAMP_EN: - len = sprintf(buf, "%u\n", conn->tcp_timestamp_en); + len = sysfs_emit(buf, "%u\n", conn->tcp_timestamp_en); break; case ISCSI_PARAM_IP_FRAGMENT_DISABLE: - len = sprintf(buf, "%u\n", conn->fragment_disable); + len = sysfs_emit(buf, "%u\n", conn->fragment_disable); break; case ISCSI_PARAM_IPV4_TOS: - len = sprintf(buf, "%u\n", conn->ipv4_tos); + len = sysfs_emit(buf, "%u\n", conn->ipv4_tos); break; case ISCSI_PARAM_IPV6_TC: - len = sprintf(buf, "%u\n", conn->ipv6_traffic_class); + len = sysfs_emit(buf, "%u\n", conn->ipv6_traffic_class); break; case ISCSI_PARAM_IPV6_FLOW_LABEL: - len = sprintf(buf, "%u\n", conn->ipv6_flow_label); + len = sysfs_emit(buf, "%u\n", conn->ipv6_flow_label); break; case ISCSI_PARAM_IS_FW_ASSIGNED_IPV6: - len = sprintf(buf, "%u\n", conn->is_fw_assigned_ipv6); + len = sysfs_emit(buf, "%u\n", conn->is_fw_assigned_ipv6); break; case ISCSI_PARAM_TCP_XMIT_WSF: - len = sprintf(buf, "%u\n", conn->tcp_xmit_wsf); + len = sysfs_emit(buf, "%u\n", conn->tcp_xmit_wsf); break; case ISCSI_PARAM_TCP_RECV_WSF: - len = sprintf(buf, "%u\n", conn->tcp_recv_wsf); + len = sysfs_emit(buf, "%u\n", conn->tcp_recv_wsf); break; case ISCSI_PARAM_LOCAL_IPADDR: - len = sprintf(buf, "%s\n", conn->local_ipaddr); + len = sysfs_emit(buf, "%s\n", conn->local_ipaddr); break; default: return -ENOSYS; @@ -3615,13 +3707,13 @@ int iscsi_host_get_param(struct Scsi_Host *shost, enum iscsi_host_param param, switch (param) { case ISCSI_HOST_PARAM_NETDEV_NAME: - len = sprintf(buf, "%s\n", ihost->netdev); + len = sysfs_emit(buf, "%s\n", ihost->netdev); break; case ISCSI_HOST_PARAM_HWADDRESS: - len = sprintf(buf, "%s\n", ihost->hwaddress); + len = sysfs_emit(buf, "%s\n", ihost->hwaddress); break; case ISCSI_HOST_PARAM_INITIATOR_NAME: - len = sprintf(buf, "%s\n", ihost->initiatorname); + len = sysfs_emit(buf, "%s\n", ihost->initiatorname); break; default: return -ENOSYS; diff --git a/drivers/scsi/libiscsi_tcp.c b/drivers/scsi/libiscsi_tcp.c index 83f14b2c8804..2e9ffe3d1a55 100644 --- a/drivers/scsi/libiscsi_tcp.c +++ b/drivers/scsi/libiscsi_tcp.c @@ -524,48 +524,79 @@ static int iscsi_tcp_data_in(struct iscsi_conn *conn, struct iscsi_task *task) /** * iscsi_tcp_r2t_rsp - iSCSI R2T Response processing * @conn: iscsi connection - * @task: scsi command task + * @hdr: PDU header */ -static int iscsi_tcp_r2t_rsp(struct iscsi_conn *conn, struct iscsi_task *task) +static int iscsi_tcp_r2t_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr) { struct iscsi_session *session = conn->session; - struct iscsi_tcp_task *tcp_task = task->dd_data; - struct iscsi_tcp_conn *tcp_conn = conn->dd_data; - struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr; + struct iscsi_tcp_task *tcp_task; + struct iscsi_tcp_conn *tcp_conn; + struct iscsi_r2t_rsp *rhdr; struct iscsi_r2t_info *r2t; - int r2tsn = be32_to_cpu(rhdr->r2tsn); + struct iscsi_task *task; u32 data_length; u32 data_offset; + int r2tsn; int rc; + spin_lock(&session->back_lock); + task = iscsi_itt_to_ctask(conn, hdr->itt); + if (!task) { + spin_unlock(&session->back_lock); + return ISCSI_ERR_BAD_ITT; + } else if (task->sc->sc_data_direction != DMA_TO_DEVICE) { + spin_unlock(&session->back_lock); + return ISCSI_ERR_PROTO; + } + /* + * A bad target might complete the cmd before we have handled R2Ts + * so get a ref to the task that will be dropped in the xmit path. + */ + if (task->state != ISCSI_TASK_RUNNING) { + spin_unlock(&session->back_lock); + /* Let the path that got the early rsp complete it */ + return 0; + } + task->last_xfer = jiffies; + __iscsi_get_task(task); + + tcp_conn = conn->dd_data; + rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr; + /* fill-in new R2T associated with the task */ + iscsi_update_cmdsn(session, (struct iscsi_nopin *)rhdr); + spin_unlock(&session->back_lock); + if (tcp_conn->in.datalen) { iscsi_conn_printk(KERN_ERR, conn, "invalid R2t with datalen %d\n", tcp_conn->in.datalen); - return ISCSI_ERR_DATALEN; + rc = ISCSI_ERR_DATALEN; + goto put_task; } + tcp_task = task->dd_data; + r2tsn = be32_to_cpu(rhdr->r2tsn); if (tcp_task->exp_datasn != r2tsn){ ISCSI_DBG_TCP(conn, "task->exp_datasn(%d) != rhdr->r2tsn(%d)\n", tcp_task->exp_datasn, r2tsn); - return ISCSI_ERR_R2TSN; + rc = ISCSI_ERR_R2TSN; + goto put_task; } - /* fill-in new R2T associated with the task */ - iscsi_update_cmdsn(session, (struct iscsi_nopin*)rhdr); - - if (!task->sc || session->state != ISCSI_STATE_LOGGED_IN) { + if (session->state != ISCSI_STATE_LOGGED_IN) { iscsi_conn_printk(KERN_INFO, conn, "dropping R2T itt %d in recovery.\n", task->itt); - return 0; + rc = 0; + goto put_task; } data_length = be32_to_cpu(rhdr->data_length); if (data_length == 0) { iscsi_conn_printk(KERN_ERR, conn, "invalid R2T with zero data len\n"); - return ISCSI_ERR_DATALEN; + rc = ISCSI_ERR_DATALEN; + goto put_task; } if (data_length > session->max_burst) @@ -579,7 +610,8 @@ static int iscsi_tcp_r2t_rsp(struct iscsi_conn *conn, struct iscsi_task *task) "invalid R2T with data len %u at offset %u " "and total length %d\n", data_length, data_offset, task->sc->sdb.length); - return ISCSI_ERR_DATALEN; + rc = ISCSI_ERR_DATALEN; + goto put_task; } spin_lock(&tcp_task->pool2queue); @@ -589,7 +621,8 @@ static int iscsi_tcp_r2t_rsp(struct iscsi_conn *conn, struct iscsi_task *task) "Target has sent more R2Ts than it " "negotiated for or driver has leaked.\n"); spin_unlock(&tcp_task->pool2queue); - return ISCSI_ERR_PROTO; + rc = ISCSI_ERR_PROTO; + goto put_task; } r2t->exp_statsn = rhdr->statsn; @@ -607,6 +640,10 @@ static int iscsi_tcp_r2t_rsp(struct iscsi_conn *conn, struct iscsi_task *task) iscsi_requeue_task(task); return 0; + +put_task: + iscsi_put_task(task); + return rc; } /* @@ -730,20 +767,11 @@ iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr) rc = iscsi_complete_pdu(conn, hdr, NULL, 0); break; case ISCSI_OP_R2T: - spin_lock(&conn->session->back_lock); - task = iscsi_itt_to_ctask(conn, hdr->itt); - spin_unlock(&conn->session->back_lock); - if (!task) - rc = ISCSI_ERR_BAD_ITT; - else if (ahslen) + if (ahslen) { rc = ISCSI_ERR_AHSLEN; - else if (task->sc->sc_data_direction == DMA_TO_DEVICE) { - task->last_xfer = jiffies; - spin_lock(&conn->session->frwd_lock); - rc = iscsi_tcp_r2t_rsp(conn, task); - spin_unlock(&conn->session->frwd_lock); - } else - rc = ISCSI_ERR_PROTO; + break; + } + rc = iscsi_tcp_r2t_rsp(conn, hdr); break; case ISCSI_OP_LOGIN_RSP: case ISCSI_OP_TEXT_RSP: diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index f5582c8e77c9..ac066f86bb14 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -3648,25 +3648,16 @@ _base_get_msix_index(struct MPT3SAS_ADAPTER *ioc, base_mod64(atomic64_add_return(1, &ioc->total_io_cnt), ioc->reply_queue_count) : 0; - return ioc->cpu_msix_table[raw_smp_processor_id()]; -} + if (scmd && ioc->shost->nr_hw_queues > 1) { + u32 tag = blk_mq_unique_tag(scmd->request); -/** - * _base_sdev_nr_inflight_request -get number of inflight requests - * of a request queue. - * @q: request_queue object - * - * returns number of inflight request of a request queue. - */ -inline unsigned long -_base_sdev_nr_inflight_request(struct request_queue *q) -{ - struct blk_mq_hw_ctx *hctx = q->queue_hw_ctx[0]; + return blk_mq_unique_tag_to_hwq(tag) + + ioc->high_iops_queues; + } - return atomic_read(&hctx->nr_active); + return ioc->cpu_msix_table[raw_smp_processor_id()]; } - /** * _base_get_high_iops_msix_index - get the msix index of * high iops queues @@ -3686,7 +3677,8 @@ _base_get_high_iops_msix_index(struct MPT3SAS_ADAPTER *ioc, * reply queues in terms of batch count 16 when outstanding * IOs on the target device is >=8. */ - if (_base_sdev_nr_inflight_request(scmd->device->request_queue) > + + if (atomic_read(&scmd->device->device_busy) > MPT3SAS_DEVICE_HIGH_IOPS_DEPTH) return base_mod64(( atomic64_add_return(1, &ioc->high_iops_outstanding) / @@ -3739,8 +3731,23 @@ mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx, struct scsi_cmnd *scmd) { struct scsiio_tracker *request = scsi_cmd_priv(scmd); - unsigned int tag = scmd->request->tag; u16 smid; + u32 tag, unique_tag; + + unique_tag = blk_mq_unique_tag(scmd->request); + tag = blk_mq_unique_tag_to_tag(unique_tag); + + /* + * Store hw queue number corresponding to the tag. + * This hw queue number is used later to determine + * the unique_tag using the logic below. This unique_tag + * is used to retrieve the scmd pointer corresponding + * to tag using scsi_host_find_tag() API. + * + * tag = smid - 1; + * unique_tag = ioc->io_queue_num[tag] << BLK_MQ_UNIQUE_TAG_BITS | tag; + */ + ioc->io_queue_num[tag] = blk_mq_unique_tag_to_hwq(unique_tag); smid = tag + 1; request->cb_idx = cb_idx; @@ -3831,6 +3838,7 @@ mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid) mpt3sas_base_clear_st(ioc, st); _base_recovery_check(ioc); + ioc->io_queue_num[smid - 1] = 0; return; } @@ -5362,6 +5370,9 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc) kfree(ioc->chain_lookup); ioc->chain_lookup = NULL; } + + kfree(ioc->io_queue_num); + ioc->io_queue_num = NULL; } /** @@ -5641,7 +5652,8 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) reply_post_free_sz = ioc->reply_post_queue_depth * sizeof(Mpi2DefaultReplyDescriptor_t); rdpq_sz = reply_post_free_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK; - if (_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable) + if ((_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable) + || (ioc->reply_queue_count < RDPQ_MAX_INDEX_IN_ONE_CHUNK)) rdpq_sz = reply_post_free_sz * ioc->reply_queue_count; ret = base_alloc_rdpq_dma_pool(ioc, rdpq_sz); if (ret == -EAGAIN) { @@ -5772,6 +5784,11 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) ioc_info(ioc, "internal(0x%p): depth(%d), start smid(%d)\n", ioc->internal, ioc->internal_depth, ioc->internal_smid)); + + ioc->io_queue_num = kcalloc(ioc->scsiio_depth, + sizeof(u16), GFP_KERNEL); + if (!ioc->io_queue_num) + goto out; /* * The number of NVMe page sized blocks needed is: * (((sg_tablesize * 8) - 1) / (page_size - 8)) + 1 @@ -8174,8 +8191,11 @@ mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, ioc_state = mpt3sas_base_get_iocstate(ioc, 0); if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT || (ioc_state & MPI2_IOC_STATE_MASK) == - MPI2_IOC_STATE_COREDUMP) + MPI2_IOC_STATE_COREDUMP) { is_fault = 1; + ioc->htb_rel.trigger_info_dwords[1] = + (ioc_state & MPI2_DOORBELL_DATA_MASK); + } } _base_pre_reset_handler(ioc); mpt3sas_wait_for_commands_to_complete(ioc); diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h index 2def7a340616..315aee6ef86f 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.h +++ b/drivers/scsi/mpt3sas/mpt3sas_base.h @@ -77,8 +77,8 @@ #define MPT3SAS_DRIVER_NAME "mpt3sas" #define MPT3SAS_AUTHOR "Avago Technologies <MPT-FusionLinux.pdl@avagotech.com>" #define MPT3SAS_DESCRIPTION "LSI MPT Fusion SAS 3.0 Device Driver" -#define MPT3SAS_DRIVER_VERSION "36.100.00.00" -#define MPT3SAS_MAJOR_VERSION 36 +#define MPT3SAS_DRIVER_VERSION "37.100.00.00" +#define MPT3SAS_MAJOR_VERSION 37 #define MPT3SAS_MINOR_VERSION 100 #define MPT3SAS_BUILD_VERSION 0 #define MPT3SAS_RELEASE_VERSION 00 @@ -1073,6 +1073,50 @@ struct hba_port { #define MULTIPATH_DISABLED_PORT_ID 0xFF +/** + * struct htb_rel_query - diagnostic buffer release reason + * @unique_id - unique id associated with this buffer. + * @buffer_rel_condition - Release condition ioctl/sysfs/reset + * @reserved + * @trigger_type - Master/Event/scsi/MPI + * @trigger_info_dwords - Data Correspondig to trigger type + */ +struct htb_rel_query { + u16 buffer_rel_condition; + u16 reserved; + u32 trigger_type; + u32 trigger_info_dwords[2]; +}; + +/* Buffer_rel_condition bit fields */ + +/* Bit 0 - Diag Buffer not Released */ +#define MPT3_DIAG_BUFFER_NOT_RELEASED (0x00) +/* Bit 0 - Diag Buffer Released */ +#define MPT3_DIAG_BUFFER_RELEASED (0x01) + +/* + * Bit 1 - Diag Buffer Released by IOCTL, + * This bit is valid only if Bit 0 is one + */ +#define MPT3_DIAG_BUFFER_REL_IOCTL (0x02 | MPT3_DIAG_BUFFER_RELEASED) + +/* + * Bit 2 - Diag Buffer Released by Trigger, + * This bit is valid only if Bit 0 is one + */ +#define MPT3_DIAG_BUFFER_REL_TRIGGER (0x04 | MPT3_DIAG_BUFFER_RELEASED) + +/* + * Bit 3 - Diag Buffer Released by SysFs, + * This bit is valid only if Bit 0 is one + */ +#define MPT3_DIAG_BUFFER_REL_SYSFS (0x08 | MPT3_DIAG_BUFFER_RELEASED) + +/* DIAG RESET Master trigger flags */ +#define MPT_DIAG_RESET_ISSUED_BY_DRIVER 0x00000000 +#define MPT_DIAG_RESET_ISSUED_BY_USER 0x00000001 + typedef void (*MPT3SAS_FLUSH_RUNNING_CMDS)(struct MPT3SAS_ADAPTER *ioc); /** * struct MPT3SAS_ADAPTER - per adapter struct @@ -1439,6 +1483,7 @@ struct MPT3SAS_ADAPTER { spinlock_t scsi_lookup_lock; int pending_io_count; wait_queue_head_t reset_wq; + u16 *io_queue_num; /* PCIe SGL */ struct dma_pool *pcie_sgl_dma_pool; @@ -1529,6 +1574,8 @@ struct MPT3SAS_ADAPTER { u32 diagnostic_flags[MPI2_DIAG_BUF_TYPE_COUNT]; u32 ring_buffer_offset; u32 ring_buffer_sz; + struct htb_rel_query htb_rel; + u8 reset_from_user; u8 is_warpdrive; u8 is_mcpu_endpoint; u8 hide_ir_msg; @@ -1565,6 +1612,7 @@ struct mpt3sas_debugfs_buffer { }; #define MPT_DRV_SUPPORT_BITMAP_MEMMOVE 0x00000001 +#define MPT_DRV_SUPPORT_BITMAP_ADDNLQUERY 0x00000002 typedef u8 (*MPT_CALLBACK)(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply); diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.c b/drivers/scsi/mpt3sas/mpt3sas_ctl.c index c8a0ce18f2c5..44f9a05db94e 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_ctl.c +++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.c @@ -479,6 +479,8 @@ void mpt3sas_ctl_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc) ioc_info(ioc, "%s: Releasing the trace buffer due to adapter reset.", __func__); + ioc->htb_rel.buffer_rel_condition = + MPT3_DIAG_BUFFER_REL_TRIGGER; mpt3sas_send_diag_release(ioc, i, &issue_reset); } } @@ -1334,6 +1336,7 @@ _ctl_do_reset(struct MPT3SAS_ADAPTER *ioc, void __user *arg) dctlprintk(ioc, ioc_info(ioc, "%s: enter\n", __func__)); + ioc->reset_from_user = 1; retval = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); ioc_info(ioc, "Ioctl: host reset: %s\n", ((!retval) ? "SUCCESS" : "FAILED")); @@ -1687,6 +1690,9 @@ _ctl_diag_register_2(struct MPT3SAS_ADAPTER *ioc, request_data = ioc->diag_buffer[buffer_type]; request_data_sz = diag_register->requested_buffer_size; ioc->unique_id[buffer_type] = diag_register->unique_id; + /* Reset ioc variables used for additional query commands */ + ioc->reset_from_user = 0; + memset(&ioc->htb_rel, 0, sizeof(struct htb_rel_query)); ioc->diag_buffer_status[buffer_type] &= MPT3_DIAG_BUFFER_IS_DRIVER_ALLOCATED; memcpy(ioc->product_specific[buffer_type], @@ -2469,7 +2475,61 @@ _ctl_diag_read_buffer(struct MPT3SAS_ADAPTER *ioc, void __user *arg) return rc; } +/** + * _ctl_addnl_diag_query - query relevant info associated with diag buffers + * @ioc: per adapter object + * @arg: user space buffer containing ioctl content + * + * The application will send only unique_id. Driver will + * inspect unique_id first, if valid, fill the details related to cause + * for diag buffer release. + */ +static long +_ctl_addnl_diag_query(struct MPT3SAS_ADAPTER *ioc, void __user *arg) +{ + struct mpt3_addnl_diag_query karg; + u32 buffer_type = 0; + if (copy_from_user(&karg, arg, sizeof(karg))) { + pr_err("%s: failure at %s:%d/%s()!\n", + ioc->name, __FILE__, __LINE__, __func__); + return -EFAULT; + } + dctlprintk(ioc, ioc_info(ioc, "%s\n", __func__)); + if (karg.unique_id == 0) { + ioc_err(ioc, "%s: unique_id is(0x%08x)\n", + __func__, karg.unique_id); + return -EPERM; + } + buffer_type = _ctl_diag_get_bufftype(ioc, karg.unique_id); + if (buffer_type == MPT3_DIAG_UID_NOT_FOUND) { + ioc_err(ioc, "%s: buffer with unique_id(0x%08x) not found\n", + __func__, karg.unique_id); + return -EPERM; + } + memset(&karg.buffer_rel_condition, 0, sizeof(struct htb_rel_query)); + if ((ioc->diag_buffer_status[buffer_type] & + MPT3_DIAG_BUFFER_IS_REGISTERED) == 0) { + ioc_info(ioc, "%s: buffer_type(0x%02x) is not registered\n", + __func__, buffer_type); + goto out; + } + if ((ioc->diag_buffer_status[buffer_type] & + MPT3_DIAG_BUFFER_IS_RELEASED) == 0) { + ioc_err(ioc, "%s: buffer_type(0x%02x) is not released\n", + __func__, buffer_type); + return -EPERM; + } + memcpy(&karg.buffer_rel_condition, &ioc->htb_rel, + sizeof(struct htb_rel_query)); +out: + if (copy_to_user(arg, &karg, sizeof(struct mpt3_addnl_diag_query))) { + ioc_err(ioc, "%s: unable to write mpt3_addnl_diag_query data @ %p\n", + __func__, arg); + return -EFAULT; + } + return 0; +} #ifdef CONFIG_COMPAT /** @@ -2533,7 +2593,7 @@ _ctl_ioctl_main(struct file *file, unsigned int cmd, void __user *arg, struct MPT3SAS_ADAPTER *ioc; struct mpt3_ioctl_header ioctl_header; enum block_state state; - long ret = -EINVAL; + long ret = -ENOIOCTLCMD; /* get IOCTL header */ if (copy_from_user(&ioctl_header, (char __user *)arg, @@ -2643,6 +2703,10 @@ _ctl_ioctl_main(struct file *file, unsigned int cmd, void __user *arg, if (_IOC_SIZE(cmd) == sizeof(struct mpt3_diag_read_buffer)) ret = _ctl_diag_read_buffer(ioc, arg); break; + case MPT3ADDNLDIAGQUERY: + if (_IOC_SIZE(cmd) == sizeof(struct mpt3_addnl_diag_query)) + ret = _ctl_addnl_diag_query(ioc, arg); + break; default: dctlprintk(ioc, ioc_info(ioc, "unsupported ioctl opcode(0x%08x)\n", @@ -3425,6 +3489,7 @@ host_trace_buffer_enable_store(struct device *cdev, MPT3_DIAG_BUFFER_IS_RELEASED)) goto out; ioc_info(ioc, "releasing host trace buffer\n"); + ioc->htb_rel.buffer_rel_condition = MPT3_DIAG_BUFFER_REL_SYSFS; mpt3sas_send_diag_release(ioc, MPI2_DIAG_BUF_TYPE_TRACE, &issue_reset); } diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.h b/drivers/scsi/mpt3sas/mpt3sas_ctl.h index 0f7aa4ddade0..d2ccdafb8df2 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_ctl.h +++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.h @@ -94,6 +94,8 @@ struct mpt3_diag_query) #define MPT3DIAGREADBUFFER _IOWR(MPT3_MAGIC_NUMBER, 30, \ struct mpt3_diag_read_buffer) +#define MPT3ADDNLDIAGQUERY _IOWR(MPT3_MAGIC_NUMBER, 32, \ + struct mpt3_addnl_diag_query) /* Trace Buffer default UniqueId */ #define MPT2DIAGBUFFUNIQUEID (0x07075900) @@ -430,4 +432,24 @@ struct mpt3_diag_read_buffer { uint32_t diagnostic_data[1]; }; +/** + * struct mpt3_addnl_diag_query - diagnostic buffer release reason + * @hdr - generic header + * @unique_id - unique id associated with this buffer. + * @buffer_rel_condition - Release condition ioctl/sysfs/reset + * @reserved1 + * @trigger_type - Master/Event/scsi/MPI + * @trigger_info_dwords - Data Correspondig to trigger type + * @reserved2 + */ +struct mpt3_addnl_diag_query { + struct mpt3_ioctl_header hdr; + uint32_t unique_id; + uint16_t buffer_rel_condition; + uint16_t reserved1; + uint32_t trigger_type; + uint32_t trigger_info_dwords[2]; + uint32_t reserved2[2]; +}; + #endif /* MPT3SAS_CTL_H_INCLUDED */ diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c index c8b09a81834d..ffca03064797 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c +++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c @@ -54,6 +54,7 @@ #include <linux/interrupt.h> #include <linux/aer.h> #include <linux/raid_class.h> +#include <linux/blk-mq-pci.h> #include <asm/unaligned.h> #include "mpt3sas_base.h" @@ -168,6 +169,11 @@ MODULE_PARM_DESC(multipath_on_hba, "\t SAS 2.0 & SAS 3.0 HBA - This will be disabled,\n\t\t" "\t SAS 3.5 HBA - This will be enabled)"); +static int host_tagset_enable = 1; +module_param(host_tagset_enable, int, 0444); +MODULE_PARM_DESC(host_tagset_enable, + "Shared host tagset enable/disable Default: enable(1)"); + /* raid transport support */ static struct raid_template *mpt3sas_raid_template; static struct raid_template *mpt2sas_raid_template; @@ -1743,10 +1749,12 @@ mpt3sas_scsih_scsi_lookup_get(struct MPT3SAS_ADAPTER *ioc, u16 smid) struct scsi_cmnd *scmd = NULL; struct scsiio_tracker *st; Mpi25SCSIIORequest_t *mpi_request; + u16 tag = smid - 1; if (smid > 0 && smid <= ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT) { - u32 unique_tag = smid - 1; + u32 unique_tag = + ioc->io_queue_num[tag] << BLK_MQ_UNIQUE_TAG_BITS | tag; mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); @@ -11599,6 +11607,22 @@ scsih_scan_finished(struct Scsi_Host *shost, unsigned long time) return 1; } +/** + * scsih_map_queues - map reply queues with request queues + * @shost: SCSI host pointer + */ +static int scsih_map_queues(struct Scsi_Host *shost) +{ + struct MPT3SAS_ADAPTER *ioc = + (struct MPT3SAS_ADAPTER *)shost->hostdata; + + if (ioc->shost->nr_hw_queues == 1) + return 0; + + return blk_mq_pci_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT], + ioc->pdev, ioc->high_iops_queues); +} + /* shost template for SAS 2.0 HBA devices */ static struct scsi_host_template mpt2sas_driver_template = { .module = THIS_MODULE, @@ -11666,6 +11690,7 @@ static struct scsi_host_template mpt3sas_driver_template = { .sdev_attrs = mpt3sas_dev_attrs, .track_queue_depth = 1, .cmd_size = sizeof(struct scsiio_tracker), + .map_queues = scsih_map_queues, }; /* raid transport support for SAS 3.0 HBA devices */ @@ -11922,6 +11947,8 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id) * Enable MEMORY MOVE support flag. */ ioc->drv_support_bitmap |= MPT_DRV_SUPPORT_BITMAP_MEMMOVE; + /* Enable ADDITIONAL QUERY support flag. */ + ioc->drv_support_bitmap |= MPT_DRV_SUPPORT_BITMAP_ADDNLQUERY; ioc->enable_sdev_max_qd = enable_sdev_max_qd; @@ -12028,6 +12055,21 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id) } else ioc->hide_drives = 0; + shost->host_tagset = 0; + shost->nr_hw_queues = 1; + + if (ioc->is_gen35_ioc && ioc->reply_queue_count > 1 && + host_tagset_enable && ioc->smp_affinity_enable) { + + shost->host_tagset = 1; + shost->nr_hw_queues = + ioc->reply_queue_count - ioc->high_iops_queues; + + dev_info(&ioc->pdev->dev, + "Max SCSIIO MPT commands: %d shared with nr_hw_queues = %d\n", + shost->can_queue, shost->nr_hw_queues); + } + rv = scsi_add_host(shost, &pdev->dev); if (rv) { ioc_err(ioc, "failure at %s:%d/%s()!\n", diff --git a/drivers/scsi/mpt3sas/mpt3sas_trigger_diag.c b/drivers/scsi/mpt3sas/mpt3sas_trigger_diag.c index 8ec9bab20ec4..d9b7d0ee25b0 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_trigger_diag.c +++ b/drivers/scsi/mpt3sas/mpt3sas_trigger_diag.c @@ -132,6 +132,35 @@ mpt3sas_process_trigger_data(struct MPT3SAS_ADAPTER *ioc, &issue_reset); } + ioc->htb_rel.buffer_rel_condition = MPT3_DIAG_BUFFER_REL_TRIGGER; + if (event_data) { + ioc->htb_rel.trigger_type = event_data->trigger_type; + switch (event_data->trigger_type) { + case MPT3SAS_TRIGGER_SCSI: + memcpy(&ioc->htb_rel.trigger_info_dwords, + &event_data->u.scsi, + sizeof(struct SL_WH_SCSI_TRIGGER_T)); + break; + case MPT3SAS_TRIGGER_MPI: + memcpy(&ioc->htb_rel.trigger_info_dwords, + &event_data->u.mpi, + sizeof(struct SL_WH_MPI_TRIGGER_T)); + break; + case MPT3SAS_TRIGGER_MASTER: + ioc->htb_rel.trigger_info_dwords[0] = + event_data->u.master.MasterData; + break; + case MPT3SAS_TRIGGER_EVENT: + memcpy(&ioc->htb_rel.trigger_info_dwords, + &event_data->u.event, + sizeof(struct SL_WH_EVENT_TRIGGER_T)); + break; + default: + ioc_err(ioc, "%d - Is not a valid Trigger type\n", + event_data->trigger_type); + break; + } + } _mpt3sas_raise_sigio(ioc, event_data); dTriggerDiagPrintk(ioc, ioc_info(ioc, "%s: exit\n", @@ -201,9 +230,14 @@ mpt3sas_trigger_master(struct MPT3SAS_ADAPTER *ioc, u32 trigger_bitmask) event_data.u.master.MasterData = trigger_bitmask; if (trigger_bitmask & MASTER_TRIGGER_FW_FAULT || - trigger_bitmask & MASTER_TRIGGER_ADAPTER_RESET) + trigger_bitmask & MASTER_TRIGGER_ADAPTER_RESET) { + ioc->htb_rel.trigger_type = MPT3SAS_TRIGGER_MASTER; + ioc->htb_rel.trigger_info_dwords[0] = trigger_bitmask; + if (ioc->reset_from_user) + ioc->htb_rel.trigger_info_dwords[1] = + MPT_DIAG_RESET_ISSUED_BY_USER; _mpt3sas_raise_sigio(ioc, &event_data); - else + } else mpt3sas_send_trigger_data_event(ioc, &event_data); out: diff --git a/drivers/scsi/pmcraid.h b/drivers/scsi/pmcraid.h index 15c962108075..6d36debde18e 100644 --- a/drivers/scsi/pmcraid.h +++ b/drivers/scsi/pmcraid.h @@ -244,7 +244,7 @@ struct pmcraid_ioarcb { __u8 hrrq_id; __u8 cdb[PMCRAID_MAX_CDB_LEN]; struct pmcraid_ioarcb_add_data add_data; -} __attribute__((packed, aligned(PMCRAID_IOARCB_ALIGNMENT))); +}; /* well known resource handle values */ #define PMCRAID_IOA_RES_HANDLE 0xffffffff @@ -1040,8 +1040,8 @@ struct pmcraid_passthrough_ioctl_buffer { struct pmcraid_ioctl_header ioctl_header; struct pmcraid_ioarcb ioarcb; struct pmcraid_ioasa ioasa; - u8 request_buffer[1]; -} __attribute__ ((packed)); + u8 request_buffer[]; +} __attribute__ ((packed, aligned(PMCRAID_IOARCB_ALIGNMENT))); /* * keys to differentiate between driver handled IOCTLs and passthrough diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c index 0d09480b66cd..c48daf52725d 100644 --- a/drivers/scsi/qla2xxx/qla_target.c +++ b/drivers/scsi/qla2xxx/qla_target.c @@ -981,8 +981,7 @@ void qlt_free_session_done(struct work_struct *work) int rc; if (!own || - (own && - (own->iocb.u.isp24.status_subcode == ELS_PLOGI))) { + (own->iocb.u.isp24.status_subcode == ELS_PLOGI)) { rc = qla2x00_post_async_logout_work(vha, sess, NULL); if (rc != QLA_SUCCESS) diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c index a4b014e1cd8c..7bd9a4a04ad5 100644 --- a/drivers/scsi/qla4xxx/ql4_os.c +++ b/drivers/scsi/qla4xxx/ql4_os.c @@ -841,7 +841,7 @@ static int __qla4xxx_is_chap_active(struct device *dev, void *data) sess = cls_session->dd_data; ddb_entry = sess->dd_data; - if (iscsi_session_chkready(cls_session)) + if (iscsi_is_session_online(cls_session)) goto exit_is_chap_active; if (ddb_entry->chap_tbl_idx == *chap_tbl_idx) diff --git a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c index 2e68c0a87698..91074fd97f64 100644 --- a/drivers/scsi/scsi_transport_iscsi.c +++ b/drivers/scsi/scsi_transport_iscsi.c @@ -132,7 +132,11 @@ show_transport_handle(struct device *dev, struct device_attribute *attr, char *buf) { struct iscsi_internal *priv = dev_to_iscsi_internal(dev); - return sprintf(buf, "%llu\n", (unsigned long long)iscsi_handle(priv->iscsi_transport)); + + if (!capable(CAP_SYS_ADMIN)) + return -EACCES; + return sysfs_emit(buf, "%llu\n", + (unsigned long long)iscsi_handle(priv->iscsi_transport)); } static DEVICE_ATTR(handle, S_IRUGO, show_transport_handle, NULL); @@ -142,7 +146,7 @@ show_transport_##name(struct device *dev, \ struct device_attribute *attr,char *buf) \ { \ struct iscsi_internal *priv = dev_to_iscsi_internal(dev); \ - return sprintf(buf, format"\n", priv->iscsi_transport->name); \ + return sysfs_emit(buf, format"\n", priv->iscsi_transport->name);\ } \ static DEVICE_ATTR(name, S_IRUGO, show_transport_##name, NULL); @@ -183,7 +187,7 @@ static ssize_t show_ep_handle(struct device *dev, struct device_attribute *attr, char *buf) { struct iscsi_endpoint *ep = iscsi_dev_to_endpoint(dev); - return sprintf(buf, "%llu\n", (unsigned long long) ep->id); + return sysfs_emit(buf, "%llu\n", (unsigned long long) ep->id); } static ISCSI_ATTR(ep, handle, S_IRUGO, show_ep_handle, NULL); @@ -1701,10 +1705,8 @@ static const char *iscsi_session_state_name(int state) int iscsi_session_chkready(struct iscsi_cls_session *session) { - unsigned long flags; int err; - spin_lock_irqsave(&session->lock, flags); switch (session->state) { case ISCSI_SESSION_LOGGED_IN: err = 0; @@ -1719,7 +1721,6 @@ int iscsi_session_chkready(struct iscsi_cls_session *session) err = DID_NO_CONNECT << 16; break; } - spin_unlock_irqrestore(&session->lock, flags); return err; } EXPORT_SYMBOL_GPL(iscsi_session_chkready); @@ -2883,6 +2884,9 @@ iscsi_set_param(struct iscsi_transport *transport, struct iscsi_uevent *ev) struct iscsi_cls_session *session; int err = 0, value = 0; + if (ev->u.set_param.len > PAGE_SIZE) + return -EINVAL; + session = iscsi_session_lookup(ev->u.set_param.sid); conn = iscsi_conn_lookup(ev->u.set_param.sid, ev->u.set_param.cid); if (!conn || !session) @@ -3030,6 +3034,9 @@ iscsi_set_host_param(struct iscsi_transport *transport, if (!transport->set_host_param) return -ENOSYS; + if (ev->u.set_host_param.len > PAGE_SIZE) + return -EINVAL; + shost = scsi_host_lookup(ev->u.set_host_param.host_no); if (!shost) { printk(KERN_ERR "set_host_param could not find host no %u\n", @@ -3617,6 +3624,7 @@ iscsi_if_recv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, uint32_t *group) { int err = 0; u32 portid; + u32 pdu_len; struct iscsi_uevent *ev = nlmsg_data(nlh); struct iscsi_transport *transport = NULL; struct iscsi_internal *priv; @@ -3624,6 +3632,9 @@ iscsi_if_recv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, uint32_t *group) struct iscsi_cls_conn *conn; struct iscsi_endpoint *ep = NULL; + if (!netlink_capable(skb, CAP_SYS_ADMIN)) + return -EPERM; + if (nlh->nlmsg_type == ISCSI_UEVENT_PATH_UPDATE) *group = ISCSI_NL_GRP_UIP; else @@ -3756,6 +3767,14 @@ iscsi_if_recv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, uint32_t *group) err = -EINVAL; break; case ISCSI_UEVENT_SEND_PDU: + pdu_len = nlh->nlmsg_len - sizeof(*nlh) - sizeof(*ev); + + if ((ev->u.send_pdu.hdr_size > pdu_len) || + (ev->u.send_pdu.data_size > (pdu_len - ev->u.send_pdu.hdr_size))) { + err = -EINVAL; + break; + } + conn = iscsi_conn_lookup(ev->u.send_pdu.sid, ev->u.send_pdu.cid); if (conn) { mutex_lock(&conn_mutex); @@ -3960,7 +3979,7 @@ static ssize_t show_conn_state(struct device *dev, conn->state < ARRAY_SIZE(connection_state_names)) state = connection_state_names[conn->state]; - return sprintf(buf, "%s\n", state); + return sysfs_emit(buf, "%s\n", state); } static ISCSI_CLASS_ATTR(conn, state, S_IRUGO, show_conn_state, NULL); @@ -4188,7 +4207,7 @@ show_priv_session_state(struct device *dev, struct device_attribute *attr, char *buf) { struct iscsi_cls_session *session = iscsi_dev_to_session(dev->parent); - return sprintf(buf, "%s\n", iscsi_session_state_name(session->state)); + return sysfs_emit(buf, "%s\n", iscsi_session_state_name(session->state)); } static ISCSI_CLASS_ATTR(priv_sess, state, S_IRUGO, show_priv_session_state, NULL); @@ -4197,7 +4216,7 @@ show_priv_session_creator(struct device *dev, struct device_attribute *attr, char *buf) { struct iscsi_cls_session *session = iscsi_dev_to_session(dev->parent); - return sprintf(buf, "%d\n", session->creator); + return sysfs_emit(buf, "%d\n", session->creator); } static ISCSI_CLASS_ATTR(priv_sess, creator, S_IRUGO, show_priv_session_creator, NULL); @@ -4206,7 +4225,7 @@ show_priv_session_target_id(struct device *dev, struct device_attribute *attr, char *buf) { struct iscsi_cls_session *session = iscsi_dev_to_session(dev->parent); - return sprintf(buf, "%d\n", session->target_id); + return sysfs_emit(buf, "%d\n", session->target_id); } static ISCSI_CLASS_ATTR(priv_sess, target_id, S_IRUGO, show_priv_session_target_id, NULL); @@ -4219,8 +4238,8 @@ show_priv_session_##field(struct device *dev, \ struct iscsi_cls_session *session = \ iscsi_dev_to_session(dev->parent); \ if (session->field == -1) \ - return sprintf(buf, "off\n"); \ - return sprintf(buf, format"\n", session->field); \ + return sysfs_emit(buf, "off\n"); \ + return sysfs_emit(buf, format"\n", session->field); \ } #define iscsi_priv_session_attr_store(field) \ diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index a3d2d4bc4a3d..ed0b1bb99f08 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -707,9 +707,9 @@ static int sd_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, put_unaligned_be16(spsp, &cdb[2]); put_unaligned_be32(len, &cdb[6]); - ret = scsi_execute_req(sdev, cdb, - send ? DMA_TO_DEVICE : DMA_FROM_DEVICE, - buffer, len, NULL, SD_TIMEOUT, sdkp->max_retries, NULL); + ret = scsi_execute(sdev, cdb, send ? DMA_TO_DEVICE : DMA_FROM_DEVICE, + buffer, len, NULL, NULL, SD_TIMEOUT, sdkp->max_retries, 0, + RQF_PM, NULL); return ret <= 0 ? ret : -EIO; } #endif /* CONFIG_BLK_SED_OPAL */ @@ -3379,10 +3379,12 @@ static int sd_probe(struct device *dev) sdp->type != TYPE_RBC) goto out; -#ifndef CONFIG_BLK_DEV_ZONED - if (sdp->type == TYPE_ZBC) + if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED) && sdp->type == TYPE_ZBC) { + sdev_printk(KERN_WARNING, sdp, + "Unsupported ZBC host-managed device.\n"); goto out; -#endif + } + SCSI_LOG_HLQUEUE(3, sdev_printk(KERN_INFO, sdp, "sd_probe\n")); diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index 03adb39293c2..ee558675eab4 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -704,6 +704,7 @@ int sd_zbc_revalidate_zones(struct scsi_disk *sdkp) unsigned int nr_zones = sdkp->rev_nr_zones; u32 max_append; int ret = 0; + unsigned int flags; /* * For all zoned disks, initialize zone append emulation data if not @@ -736,16 +737,19 @@ int sd_zbc_revalidate_zones(struct scsi_disk *sdkp) disk->queue->nr_zones == nr_zones) goto unlock; + flags = memalloc_noio_save(); sdkp->zone_blocks = zone_blocks; sdkp->nr_zones = nr_zones; - sdkp->rev_wp_offset = kvcalloc(nr_zones, sizeof(u32), GFP_NOIO); + sdkp->rev_wp_offset = kvcalloc(nr_zones, sizeof(u32), GFP_KERNEL); if (!sdkp->rev_wp_offset) { ret = -ENOMEM; + memalloc_noio_restore(flags); goto unlock; } ret = blk_revalidate_disk_zones(disk, sd_zbc_revalidate_zones_cb); + memalloc_noio_restore(flags); kvfree(sdkp->rev_wp_offset); sdkp->rev_wp_offset = NULL; diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index 721f55db181f..77161750c9fb 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -451,6 +451,8 @@ static void ufshcd_print_evt(struct ufs_hba *hba, u32 id, if (!found) dev_err(hba->dev, "No record of %s\n", err_name); + else + dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt); } static void ufshcd_print_evt_hist(struct ufs_hba *hba) @@ -1866,7 +1868,7 @@ static ssize_t ufshcd_clkgate_delay_show(struct device *dev, { struct ufs_hba *hba = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms); + return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms); } static ssize_t ufshcd_clkgate_delay_store(struct device *dev, @@ -1889,7 +1891,7 @@ static ssize_t ufshcd_clkgate_enable_show(struct device *dev, { struct ufs_hba *hba = dev_get_drvdata(dev); - return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled); + return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled); } static ssize_t ufshcd_clkgate_enable_store(struct device *dev, diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index ee61f821f75d..18e56c1c1b30 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -570,7 +570,7 @@ enum ufshcd_quirks { /* * This quirk allows only sg entries aligned with page size. */ - UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE = 1 << 13, + UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE = 1 << 14, }; enum ufshcd_caps { diff --git a/drivers/sfi/Kconfig b/drivers/sfi/Kconfig deleted file mode 100644 index 3d0b64db214e..000000000000 --- a/drivers/sfi/Kconfig +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# SFI Configuration -# - -menuconfig SFI - bool "SFI (Simple Firmware Interface) Support" - help - The Simple Firmware Interface (SFI) provides a lightweight method - for platform firmware to pass information to the operating system - via static tables in memory. Kernel SFI support is required to - boot on SFI-only platforms. Currently, all SFI-only platforms are - based on the 2nd generation Intel Atom processor platform, - code-named Moorestown. - - For more information, see http://simplefirmware.org - - Say 'Y' here to enable the kernel to boot on SFI-only platforms. diff --git a/drivers/sfi/Makefile b/drivers/sfi/Makefile deleted file mode 100644 index ca9436b12386..000000000000 --- a/drivers/sfi/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-y += sfi_acpi.o -obj-y += sfi_core.o - diff --git a/drivers/sfi/sfi_acpi.c b/drivers/sfi/sfi_acpi.c deleted file mode 100644 index d277b36eb389..000000000000 --- a/drivers/sfi/sfi_acpi.c +++ /dev/null @@ -1,214 +0,0 @@ -/* sfi_acpi.c Simple Firmware Interface - ACPI extensions */ - -/* - - This file is provided under a dual BSD/GPLv2 license. When using or - redistributing this file, you may do so under either license. - - GPL LICENSE SUMMARY - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of version 2 of the GNU General Public License as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - The full GNU General Public License is included in this distribution - in the file called LICENSE.GPL. - - BSD LICENSE - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#define KMSG_COMPONENT "SFI" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt - -#include <linux/kernel.h> -#include <linux/sfi_acpi.h> -#include "sfi_core.h" - -/* - * SFI can access ACPI-defined tables via an optional ACPI XSDT. - * - * This allows re-use, and avoids re-definition, of standard tables. - * For example, the "MCFG" table is defined by PCI, reserved by ACPI, - * and is expected to be present many SFI-only systems. - */ - -static struct acpi_table_xsdt *xsdt_va __read_mostly; - -#define XSDT_GET_NUM_ENTRIES(ptable, entry_type) \ - ((ptable->header.length - sizeof(struct acpi_table_header)) / \ - (sizeof(entry_type))) - -static inline struct sfi_table_header *acpi_to_sfi_th( - struct acpi_table_header *th) -{ - return (struct sfi_table_header *)th; -} - -static inline struct acpi_table_header *sfi_to_acpi_th( - struct sfi_table_header *th) -{ - return (struct acpi_table_header *)th; -} - -/* - * sfi_acpi_parse_xsdt() - * - * Parse the ACPI XSDT for later access by sfi_acpi_table_parse(). - */ -static int __init sfi_acpi_parse_xsdt(struct sfi_table_header *th) -{ - struct sfi_table_key key = SFI_ANY_KEY; - int tbl_cnt, i; - void *ret; - - xsdt_va = (struct acpi_table_xsdt *)th; - tbl_cnt = XSDT_GET_NUM_ENTRIES(xsdt_va, u64); - for (i = 0; i < tbl_cnt; i++) { - ret = sfi_check_table(xsdt_va->table_offset_entry[i], &key); - if (IS_ERR(ret)) { - disable_sfi(); - return -1; - } - } - - return 0; -} - -int __init sfi_acpi_init(void) -{ - struct sfi_table_key xsdt_key = { .sig = SFI_SIG_XSDT }; - - sfi_table_parse(SFI_SIG_XSDT, NULL, NULL, sfi_acpi_parse_xsdt); - - /* Only call the get_table to keep the table mapped */ - xsdt_va = (struct acpi_table_xsdt *)sfi_get_table(&xsdt_key); - return 0; -} - -static struct acpi_table_header *sfi_acpi_get_table(struct sfi_table_key *key) -{ - u32 tbl_cnt, i; - void *ret; - - tbl_cnt = XSDT_GET_NUM_ENTRIES(xsdt_va, u64); - for (i = 0; i < tbl_cnt; i++) { - ret = sfi_check_table(xsdt_va->table_offset_entry[i], key); - if (!IS_ERR(ret) && ret) - return sfi_to_acpi_th(ret); - } - - return NULL; -} - -static void sfi_acpi_put_table(struct acpi_table_header *table) -{ - sfi_put_table(acpi_to_sfi_th(table)); -} - -/* - * sfi_acpi_table_parse() - * - * Find specified table in XSDT, run handler on it and return its return value - */ -int sfi_acpi_table_parse(char *signature, char *oem_id, char *oem_table_id, - int(*handler)(struct acpi_table_header *)) -{ - struct acpi_table_header *table = NULL; - struct sfi_table_key key; - int ret = 0; - - if (sfi_disabled) - return -1; - - key.sig = signature; - key.oem_id = oem_id; - key.oem_table_id = oem_table_id; - - table = sfi_acpi_get_table(&key); - if (!table) - return -EINVAL; - - ret = handler(table); - sfi_acpi_put_table(table); - return ret; -} - -static ssize_t sfi_acpi_table_show(struct file *filp, struct kobject *kobj, - struct bin_attribute *bin_attr, char *buf, - loff_t offset, size_t count) -{ - struct sfi_table_attr *tbl_attr = - container_of(bin_attr, struct sfi_table_attr, attr); - struct acpi_table_header *th = NULL; - struct sfi_table_key key; - ssize_t cnt; - - key.sig = tbl_attr->name; - key.oem_id = NULL; - key.oem_table_id = NULL; - - th = sfi_acpi_get_table(&key); - if (!th) - return 0; - - cnt = memory_read_from_buffer(buf, count, &offset, - th, th->length); - sfi_acpi_put_table(th); - - return cnt; -} - - -void __init sfi_acpi_sysfs_init(void) -{ - u32 tbl_cnt, i; - struct sfi_table_attr *tbl_attr; - - tbl_cnt = XSDT_GET_NUM_ENTRIES(xsdt_va, u64); - for (i = 0; i < tbl_cnt; i++) { - tbl_attr = - sfi_sysfs_install_table(xsdt_va->table_offset_entry[i]); - tbl_attr->attr.read = sfi_acpi_table_show; - } - - return; -} diff --git a/drivers/sfi/sfi_core.c b/drivers/sfi/sfi_core.c deleted file mode 100644 index a5136901dd8a..000000000000 --- a/drivers/sfi/sfi_core.c +++ /dev/null @@ -1,522 +0,0 @@ -/* sfi_core.c Simple Firmware Interface - core internals */ - -/* - - This file is provided under a dual BSD/GPLv2 license. When using or - redistributing this file, you may do so under either license. - - GPL LICENSE SUMMARY - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of version 2 of the GNU General Public License as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - The full GNU General Public License is included in this distribution - in the file called LICENSE.GPL. - - BSD LICENSE - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#define KMSG_COMPONENT "SFI" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt - -#include <linux/memblock.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/errno.h> -#include <linux/types.h> -#include <linux/acpi.h> -#include <linux/init.h> -#include <linux/sfi.h> -#include <linux/slab.h> -#include <linux/io.h> - -#include "sfi_core.h" - -#define ON_SAME_PAGE(addr1, addr2) \ - (((unsigned long)(addr1) & PAGE_MASK) == \ - ((unsigned long)(addr2) & PAGE_MASK)) -#define TABLE_ON_PAGE(page, table, size) (ON_SAME_PAGE(page, table) && \ - ON_SAME_PAGE(page, table + size)) - -int sfi_disabled __read_mostly; -EXPORT_SYMBOL(sfi_disabled); - -static u64 syst_pa __read_mostly; -static struct sfi_table_simple *syst_va __read_mostly; - -/* - * FW creates and saves the SFI tables in memory. When these tables get - * used, they may need to be mapped to virtual address space, and the mapping - * can happen before or after the memremap() is ready, so a flag is needed - * to indicating this - */ -static u32 sfi_use_memremap __read_mostly; - -/* - * sfi_un/map_memory calls early_memremap/memunmap which is a __init function - * and introduces section mismatch. So use __ref to make it calm. - */ -static void __iomem * __ref sfi_map_memory(u64 phys, u32 size) -{ - if (!phys || !size) - return NULL; - - if (sfi_use_memremap) - return memremap(phys, size, MEMREMAP_WB); - else - return early_memremap(phys, size); -} - -static void __ref sfi_unmap_memory(void __iomem *virt, u32 size) -{ - if (!virt || !size) - return; - - if (sfi_use_memremap) - memunmap(virt); - else - early_memunmap(virt, size); -} - -static void sfi_print_table_header(unsigned long long pa, - struct sfi_table_header *header) -{ - pr_info("%4.4s %llX, %04X (v%d %6.6s %8.8s)\n", - header->sig, pa, - header->len, header->rev, header->oem_id, - header->oem_table_id); -} - -/* - * sfi_verify_table() - * Sanity check table lengh, calculate checksum - */ -static int sfi_verify_table(struct sfi_table_header *table) -{ - - u8 checksum = 0; - u8 *puchar = (u8 *)table; - u32 length = table->len; - - /* Sanity check table length against arbitrary 1MB limit */ - if (length > 0x100000) { - pr_err("Invalid table length 0x%x\n", length); - return -1; - } - - while (length--) - checksum += *puchar++; - - if (checksum) { - pr_err("Checksum %2.2X should be %2.2X\n", - table->csum, table->csum - checksum); - return -1; - } - return 0; -} - -/* - * sfi_map_table() - * - * Return address of mapped table - * Check for common case that we can re-use mapping to SYST, - * which requires syst_pa, syst_va to be initialized. - */ -static struct sfi_table_header *sfi_map_table(u64 pa) -{ - struct sfi_table_header *th; - u32 length; - - if (!TABLE_ON_PAGE(syst_pa, pa, sizeof(struct sfi_table_header))) - th = sfi_map_memory(pa, sizeof(struct sfi_table_header)); - else - th = (void *)syst_va + (pa - syst_pa); - - /* If table fits on same page as its header, we are done */ - if (TABLE_ON_PAGE(th, th, th->len)) - return th; - - /* Entire table does not fit on same page as SYST */ - length = th->len; - if (!TABLE_ON_PAGE(syst_pa, pa, sizeof(struct sfi_table_header))) - sfi_unmap_memory(th, sizeof(struct sfi_table_header)); - - return sfi_map_memory(pa, length); -} - -/* - * sfi_unmap_table() - * - * Undoes effect of sfi_map_table() by unmapping table - * if it did not completely fit on same page as SYST. - */ -static void sfi_unmap_table(struct sfi_table_header *th) -{ - if (!TABLE_ON_PAGE(syst_va, th, th->len)) - sfi_unmap_memory(th, TABLE_ON_PAGE(th, th, th->len) ? - sizeof(*th) : th->len); -} - -static int sfi_table_check_key(struct sfi_table_header *th, - struct sfi_table_key *key) -{ - - if (strncmp(th->sig, key->sig, SFI_SIGNATURE_SIZE) - || (key->oem_id && strncmp(th->oem_id, - key->oem_id, SFI_OEM_ID_SIZE)) - || (key->oem_table_id && strncmp(th->oem_table_id, - key->oem_table_id, SFI_OEM_TABLE_ID_SIZE))) - return -1; - - return 0; -} - -/* - * This function will be used in 2 cases: - * 1. used to enumerate and verify the tables addressed by SYST/XSDT, - * thus no signature will be given (in kernel boot phase) - * 2. used to parse one specific table, signature must exist, and - * the mapped virt address will be returned, and the virt space - * will be released by call sfi_put_table() later - * - * This two cases are from two different functions with two different - * sections and causes section mismatch warning. So use __ref to tell - * modpost not to make any noise. - * - * Return value: - * NULL: when can't find a table matching the key - * ERR_PTR(error): error value - * virt table address: when a matched table is found - */ -struct sfi_table_header * - __ref sfi_check_table(u64 pa, struct sfi_table_key *key) -{ - struct sfi_table_header *th; - void *ret = NULL; - - th = sfi_map_table(pa); - if (!th) - return ERR_PTR(-ENOMEM); - - if (!key->sig) { - sfi_print_table_header(pa, th); - if (sfi_verify_table(th)) - ret = ERR_PTR(-EINVAL); - } else { - if (!sfi_table_check_key(th, key)) - return th; /* Success */ - } - - sfi_unmap_table(th); - return ret; -} - -/* - * sfi_get_table() - * - * Search SYST for the specified table with the signature in - * the key, and return the mapped table - */ -struct sfi_table_header *sfi_get_table(struct sfi_table_key *key) -{ - struct sfi_table_header *th; - u32 tbl_cnt, i; - - tbl_cnt = SFI_GET_NUM_ENTRIES(syst_va, u64); - for (i = 0; i < tbl_cnt; i++) { - th = sfi_check_table(syst_va->pentry[i], key); - if (!IS_ERR(th) && th) - return th; - } - - return NULL; -} - -void sfi_put_table(struct sfi_table_header *th) -{ - sfi_unmap_table(th); -} - -/* Find table with signature, run handler on it */ -int sfi_table_parse(char *signature, char *oem_id, char *oem_table_id, - sfi_table_handler handler) -{ - struct sfi_table_header *table = NULL; - struct sfi_table_key key; - int ret = -EINVAL; - - if (sfi_disabled || !handler || !signature) - goto exit; - - key.sig = signature; - key.oem_id = oem_id; - key.oem_table_id = oem_table_id; - - table = sfi_get_table(&key); - if (!table) - goto exit; - - ret = handler(table); - sfi_put_table(table); -exit: - return ret; -} -EXPORT_SYMBOL_GPL(sfi_table_parse); - -/* - * sfi_parse_syst() - * Checksum all the tables in SYST and print their headers - * - * success: set syst_va, return 0 - */ -static int __init sfi_parse_syst(void) -{ - struct sfi_table_key key = SFI_ANY_KEY; - int tbl_cnt, i; - void *ret; - - syst_va = sfi_map_memory(syst_pa, sizeof(struct sfi_table_simple)); - if (!syst_va) - return -ENOMEM; - - tbl_cnt = SFI_GET_NUM_ENTRIES(syst_va, u64); - for (i = 0; i < tbl_cnt; i++) { - ret = sfi_check_table(syst_va->pentry[i], &key); - if (IS_ERR(ret)) - return PTR_ERR(ret); - } - - return 0; -} - -/* - * The OS finds the System Table by searching 16-byte boundaries between - * physical address 0x000E0000 and 0x000FFFFF. The OS shall search this region - * starting at the low address and shall stop searching when the 1st valid SFI - * System Table is found. - * - * success: set syst_pa, return 0 - * fail: return -1 - */ -static __init int sfi_find_syst(void) -{ - unsigned long offset, len; - void *start; - - len = SFI_SYST_SEARCH_END - SFI_SYST_SEARCH_BEGIN; - start = sfi_map_memory(SFI_SYST_SEARCH_BEGIN, len); - if (!start) - return -1; - - for (offset = 0; offset < len; offset += 16) { - struct sfi_table_header *syst_hdr; - - syst_hdr = start + offset; - if (strncmp(syst_hdr->sig, SFI_SIG_SYST, - SFI_SIGNATURE_SIZE)) - continue; - - if (syst_hdr->len > PAGE_SIZE) - continue; - - sfi_print_table_header(SFI_SYST_SEARCH_BEGIN + offset, - syst_hdr); - - if (sfi_verify_table(syst_hdr)) - continue; - - /* - * Enforce SFI spec mandate that SYST reside within a page. - */ - if (!ON_SAME_PAGE(syst_pa, syst_pa + syst_hdr->len)) { - pr_info("SYST 0x%llx + 0x%x crosses page\n", - syst_pa, syst_hdr->len); - continue; - } - - /* Success */ - syst_pa = SFI_SYST_SEARCH_BEGIN + offset; - sfi_unmap_memory(start, len); - return 0; - } - - sfi_unmap_memory(start, len); - return -1; -} - -static struct kobject *sfi_kobj; -static struct kobject *tables_kobj; - -static ssize_t sfi_table_show(struct file *filp, struct kobject *kobj, - struct bin_attribute *bin_attr, char *buf, - loff_t offset, size_t count) -{ - struct sfi_table_attr *tbl_attr = - container_of(bin_attr, struct sfi_table_attr, attr); - struct sfi_table_header *th = NULL; - struct sfi_table_key key; - ssize_t cnt; - - key.sig = tbl_attr->name; - key.oem_id = NULL; - key.oem_table_id = NULL; - - if (strncmp(SFI_SIG_SYST, tbl_attr->name, SFI_SIGNATURE_SIZE)) { - th = sfi_get_table(&key); - if (!th) - return 0; - - cnt = memory_read_from_buffer(buf, count, &offset, - th, th->len); - sfi_put_table(th); - } else - cnt = memory_read_from_buffer(buf, count, &offset, - syst_va, syst_va->header.len); - - return cnt; -} - -struct sfi_table_attr __init *sfi_sysfs_install_table(u64 pa) -{ - struct sfi_table_attr *tbl_attr; - struct sfi_table_header *th; - int ret; - - tbl_attr = kzalloc(sizeof(struct sfi_table_attr), GFP_KERNEL); - if (!tbl_attr) - return NULL; - - th = sfi_map_table(pa); - if (!th || !th->sig[0]) { - kfree(tbl_attr); - return NULL; - } - - sysfs_attr_init(&tbl_attr->attr.attr); - memcpy(tbl_attr->name, th->sig, SFI_SIGNATURE_SIZE); - - tbl_attr->attr.size = 0; - tbl_attr->attr.read = sfi_table_show; - tbl_attr->attr.attr.name = tbl_attr->name; - tbl_attr->attr.attr.mode = 0400; - - ret = sysfs_create_bin_file(tables_kobj, - &tbl_attr->attr); - if (ret) { - kfree(tbl_attr); - tbl_attr = NULL; - } - - sfi_unmap_table(th); - return tbl_attr; -} - -static int __init sfi_sysfs_init(void) -{ - int tbl_cnt, i; - - if (sfi_disabled) - return 0; - - sfi_kobj = kobject_create_and_add("sfi", firmware_kobj); - if (!sfi_kobj) - return 0; - - tables_kobj = kobject_create_and_add("tables", sfi_kobj); - if (!tables_kobj) { - kobject_put(sfi_kobj); - return 0; - } - - sfi_sysfs_install_table(syst_pa); - - tbl_cnt = SFI_GET_NUM_ENTRIES(syst_va, u64); - - for (i = 0; i < tbl_cnt; i++) - sfi_sysfs_install_table(syst_va->pentry[i]); - - sfi_acpi_sysfs_init(); - kobject_uevent(sfi_kobj, KOBJ_ADD); - kobject_uevent(tables_kobj, KOBJ_ADD); - pr_info("SFI sysfs interfaces init success\n"); - return 0; -} - -void __init sfi_init(void) -{ - if (!acpi_disabled) - disable_sfi(); - - if (sfi_disabled) - return; - - pr_info("Simple Firmware Interface v0.81 http://simplefirmware.org\n"); - - if (sfi_find_syst() || sfi_parse_syst() || sfi_platform_init()) - disable_sfi(); - - return; -} - -void __init sfi_init_late(void) -{ - int length; - - if (sfi_disabled) - return; - - length = syst_va->header.len; - sfi_unmap_memory(syst_va, sizeof(struct sfi_table_simple)); - - /* Use memremap now after it is ready */ - sfi_use_memremap = 1; - syst_va = sfi_map_memory(syst_pa, length); - - sfi_acpi_init(); -} - -/* - * The reason we put it here because we need wait till the /sys/firmware - * is setup, then our interface can be registered in /sys/firmware/sfi - */ -core_initcall(sfi_sysfs_init); diff --git a/drivers/sfi/sfi_core.h b/drivers/sfi/sfi_core.h deleted file mode 100644 index 1d5cfe854cf7..000000000000 --- a/drivers/sfi/sfi_core.h +++ /dev/null @@ -1,81 +0,0 @@ -/* sfi_core.h Simple Firmware Interface, internal header */ - -/* - - This file is provided under a dual BSD/GPLv2 license. When using or - redistributing this file, you may do so under either license. - - GPL LICENSE SUMMARY - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of version 2 of the GNU General Public License as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - The full GNU General Public License is included in this distribution - in the file called LICENSE.GPL. - - BSD LICENSE - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#include <linux/sysfs.h> - -struct sfi_table_key{ - char *sig; - char *oem_id; - char *oem_table_id; -}; - -/* sysfs interface */ -struct sfi_table_attr { - struct bin_attribute attr; - char name[8]; -}; - -#define SFI_ANY_KEY { .sig = NULL, .oem_id = NULL, .oem_table_id = NULL } - -extern int __init sfi_acpi_init(void); -extern struct sfi_table_header *sfi_check_table(u64 paddr, - struct sfi_table_key *key); -struct sfi_table_header *sfi_get_table(struct sfi_table_key *key); -extern void sfi_put_table(struct sfi_table_header *table); -extern struct sfi_table_attr __init *sfi_sysfs_install_table(u64 pa); -extern void __init sfi_acpi_sysfs_init(void); diff --git a/drivers/soc/Kconfig b/drivers/soc/Kconfig index f357c6c659d2..e8a30c4c5aec 100644 --- a/drivers/soc/Kconfig +++ b/drivers/soc/Kconfig @@ -6,6 +6,7 @@ source "drivers/soc/amlogic/Kconfig" source "drivers/soc/aspeed/Kconfig" source "drivers/soc/atmel/Kconfig" source "drivers/soc/bcm/Kconfig" +source "drivers/soc/canaan/Kconfig" source "drivers/soc/fsl/Kconfig" source "drivers/soc/imx/Kconfig" source "drivers/soc/ixp4xx/Kconfig" @@ -22,6 +23,5 @@ source "drivers/soc/ti/Kconfig" source "drivers/soc/ux500/Kconfig" source "drivers/soc/versatile/Kconfig" source "drivers/soc/xilinx/Kconfig" -source "drivers/soc/kendryte/Kconfig" endmenu diff --git a/drivers/soc/Makefile b/drivers/soc/Makefile index 9bceb12b291d..f678e4d9e585 100644 --- a/drivers/soc/Makefile +++ b/drivers/soc/Makefile @@ -7,6 +7,7 @@ obj-$(CONFIG_ARCH_ACTIONS) += actions/ obj-y += aspeed/ obj-$(CONFIG_ARCH_AT91) += atmel/ obj-y += bcm/ +obj-$(CONFIG_SOC_CANAAN) += canaan/ obj-$(CONFIG_ARCH_DOVE) += dove/ obj-$(CONFIG_MACH_DOVE) += dove/ obj-y += fsl/ @@ -28,4 +29,3 @@ obj-y += ti/ obj-$(CONFIG_ARCH_U8500) += ux500/ obj-$(CONFIG_PLAT_VERSATILE) += versatile/ obj-y += xilinx/ -obj-$(CONFIG_SOC_KENDRYTE) += kendryte/ diff --git a/drivers/soc/canaan/Kconfig b/drivers/soc/canaan/Kconfig new file mode 100644 index 000000000000..8179b69518b4 --- /dev/null +++ b/drivers/soc/canaan/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 + +config SOC_K210_SYSCTL + bool "Canaan Kendryte K210 SoC system controller" + depends on RISCV && SOC_CANAAN && OF + default SOC_CANAAN + select PM + select SIMPLE_PM_BUS + select SYSCON + select MFD_SYSCON + help + Canaan Kendryte K210 SoC system controller driver. diff --git a/drivers/soc/canaan/Makefile b/drivers/soc/canaan/Makefile new file mode 100644 index 000000000000..570280ad7967 --- /dev/null +++ b/drivers/soc/canaan/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_SOC_K210_SYSCTL) += k210-sysctl.o diff --git a/drivers/soc/canaan/k210-sysctl.c b/drivers/soc/canaan/k210-sysctl.c new file mode 100644 index 000000000000..27a346c406bc --- /dev/null +++ b/drivers/soc/canaan/k210-sysctl.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2019 Christoph Hellwig. + * Copyright (c) 2019 Western Digital Corporation or its affiliates. + */ +#include <linux/io.h> +#include <linux/platform_device.h> +#include <linux/of_platform.h> +#include <linux/clk.h> +#include <asm/soc.h> + +#include <soc/canaan/k210-sysctl.h> + +static int k210_sysctl_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct clk *pclk; + int ret; + + dev_info(dev, "K210 system controller\n"); + + /* Get power bus clock */ + pclk = devm_clk_get(dev, NULL); + if (IS_ERR(pclk)) + return dev_err_probe(dev, PTR_ERR(pclk), + "Get bus clock failed\n"); + + ret = clk_prepare_enable(pclk); + if (ret) { + dev_err(dev, "Enable bus clock failed\n"); + return ret; + } + + /* Populate children */ + ret = devm_of_platform_populate(dev); + if (ret) + dev_err(dev, "Populate platform failed %d\n", ret); + + return ret; +} + +static const struct of_device_id k210_sysctl_of_match[] = { + { .compatible = "canaan,k210-sysctl", }, + { /* sentinel */ }, +}; + +static struct platform_driver k210_sysctl_driver = { + .driver = { + .name = "k210-sysctl", + .of_match_table = k210_sysctl_of_match, + }, + .probe = k210_sysctl_probe, +}; +builtin_platform_driver(k210_sysctl_driver); + +/* + * System controller registers base address and size. + */ +#define K210_SYSCTL_BASE_ADDR 0x50440000ULL +#define K210_SYSCTL_BASE_SIZE 0x1000 + +/* + * This needs to be called very early during initialization, given that + * PLL1 needs to be enabled to be able to use all SRAM. + */ +static void __init k210_soc_early_init(const void *fdt) +{ + void __iomem *sysctl_base; + + sysctl_base = ioremap(K210_SYSCTL_BASE_ADDR, K210_SYSCTL_BASE_SIZE); + if (!sysctl_base) + panic("k210-sysctl: ioremap failed"); + + k210_clk_early_init(sysctl_base); + + iounmap(sysctl_base); +} +SOC_EARLY_INIT_DECLARE(k210_soc, "canaan,kendryte-k210", k210_soc_early_init); diff --git a/drivers/soc/kendryte/Kconfig b/drivers/soc/kendryte/Kconfig deleted file mode 100644 index 49785b1b0217..000000000000 --- a/drivers/soc/kendryte/Kconfig +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -if SOC_KENDRYTE - -config K210_SYSCTL - bool "Kendryte K210 system controller" - default y - depends on RISCV - help - Enables controlling the K210 various clocks and to enable - general purpose use of the extra 2MB of SRAM normally - reserved for the AI engine. - -endif diff --git a/drivers/soc/kendryte/Makefile b/drivers/soc/kendryte/Makefile deleted file mode 100644 index 002d9ce95c0d..000000000000 --- a/drivers/soc/kendryte/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -obj-$(CONFIG_K210_SYSCTL) += k210-sysctl.o diff --git a/drivers/soc/kendryte/k210-sysctl.c b/drivers/soc/kendryte/k210-sysctl.c deleted file mode 100644 index 707019223dd8..000000000000 --- a/drivers/soc/kendryte/k210-sysctl.c +++ /dev/null @@ -1,260 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (c) 2019 Christoph Hellwig. - * Copyright (c) 2019 Western Digital Corporation or its affiliates. - */ -#include <linux/types.h> -#include <linux/io.h> -#include <linux/of.h> -#include <linux/platform_device.h> -#include <linux/clk-provider.h> -#include <linux/clkdev.h> -#include <linux/bitfield.h> -#include <asm/soc.h> - -#define K210_SYSCTL_CLK0_FREQ 26000000UL - -/* Registers base address */ -#define K210_SYSCTL_SYSCTL_BASE_ADDR 0x50440000ULL - -/* Registers */ -#define K210_SYSCTL_PLL0 0x08 -#define K210_SYSCTL_PLL1 0x0c -/* clkr: 4bits, clkf1: 6bits, clkod: 4bits, bwadj: 4bits */ -#define PLL_RESET (1 << 20) -#define PLL_PWR (1 << 21) -#define PLL_INTFB (1 << 22) -#define PLL_BYPASS (1 << 23) -#define PLL_TEST (1 << 24) -#define PLL_OUT_EN (1 << 25) -#define PLL_TEST_EN (1 << 26) -#define K210_SYSCTL_PLL_LOCK 0x18 -#define PLL0_LOCK1 (1 << 0) -#define PLL0_LOCK2 (1 << 1) -#define PLL0_SLIP_CLEAR (1 << 2) -#define PLL0_TEST_CLK_OUT (1 << 3) -#define PLL1_LOCK1 (1 << 8) -#define PLL1_LOCK2 (1 << 9) -#define PLL1_SLIP_CLEAR (1 << 10) -#define PLL1_TEST_CLK_OUT (1 << 11) -#define PLL2_LOCK1 (1 << 16) -#define PLL2_LOCK2 (1 << 16) -#define PLL2_SLIP_CLEAR (1 << 18) -#define PLL2_TEST_CLK_OUT (1 << 19) -#define K210_SYSCTL_CLKSEL0 0x20 -#define CLKSEL_ACLK (1 << 0) -#define K210_SYSCTL_CLKEN_CENT 0x28 -#define CLKEN_CPU (1 << 0) -#define CLKEN_SRAM0 (1 << 1) -#define CLKEN_SRAM1 (1 << 2) -#define CLKEN_APB0 (1 << 3) -#define CLKEN_APB1 (1 << 4) -#define CLKEN_APB2 (1 << 5) -#define K210_SYSCTL_CLKEN_PERI 0x2c -#define CLKEN_ROM (1 << 0) -#define CLKEN_DMA (1 << 1) -#define CLKEN_AI (1 << 2) -#define CLKEN_DVP (1 << 3) -#define CLKEN_FFT (1 << 4) -#define CLKEN_GPIO (1 << 5) -#define CLKEN_SPI0 (1 << 6) -#define CLKEN_SPI1 (1 << 7) -#define CLKEN_SPI2 (1 << 8) -#define CLKEN_SPI3 (1 << 9) -#define CLKEN_I2S0 (1 << 10) -#define CLKEN_I2S1 (1 << 11) -#define CLKEN_I2S2 (1 << 12) -#define CLKEN_I2C0 (1 << 13) -#define CLKEN_I2C1 (1 << 14) -#define CLKEN_I2C2 (1 << 15) -#define CLKEN_UART1 (1 << 16) -#define CLKEN_UART2 (1 << 17) -#define CLKEN_UART3 (1 << 18) -#define CLKEN_AES (1 << 19) -#define CLKEN_FPIO (1 << 20) -#define CLKEN_TIMER0 (1 << 21) -#define CLKEN_TIMER1 (1 << 22) -#define CLKEN_TIMER2 (1 << 23) -#define CLKEN_WDT0 (1 << 24) -#define CLKEN_WDT1 (1 << 25) -#define CLKEN_SHA (1 << 26) -#define CLKEN_OTP (1 << 27) -#define CLKEN_RTC (1 << 29) - -struct k210_sysctl { - void __iomem *regs; - struct clk_hw hw; -}; - -static void k210_set_bits(u32 val, void __iomem *reg) -{ - writel(readl(reg) | val, reg); -} - -static void k210_clear_bits(u32 val, void __iomem *reg) -{ - writel(readl(reg) & ~val, reg); -} - -static void k210_pll1_enable(void __iomem *regs) -{ - u32 val; - - val = readl(regs + K210_SYSCTL_PLL1); - val &= ~GENMASK(19, 0); /* clkr1 = 0 */ - val |= FIELD_PREP(GENMASK(9, 4), 0x3B); /* clkf1 = 59 */ - val |= FIELD_PREP(GENMASK(13, 10), 0x3); /* clkod1 = 3 */ - val |= FIELD_PREP(GENMASK(19, 14), 0x3B); /* bwadj1 = 59 */ - writel(val, regs + K210_SYSCTL_PLL1); - - k210_clear_bits(PLL_BYPASS, regs + K210_SYSCTL_PLL1); - k210_set_bits(PLL_PWR, regs + K210_SYSCTL_PLL1); - - /* - * Reset the pll. The magic NOPs come from the Kendryte reference SDK. - */ - k210_clear_bits(PLL_RESET, regs + K210_SYSCTL_PLL1); - k210_set_bits(PLL_RESET, regs + K210_SYSCTL_PLL1); - nop(); - nop(); - k210_clear_bits(PLL_RESET, regs + K210_SYSCTL_PLL1); - - for (;;) { - val = readl(regs + K210_SYSCTL_PLL_LOCK); - if (val & PLL1_LOCK2) - break; - writel(val | PLL1_SLIP_CLEAR, regs + K210_SYSCTL_PLL_LOCK); - } - - k210_set_bits(PLL_OUT_EN, regs + K210_SYSCTL_PLL1); -} - -static unsigned long k210_sysctl_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct k210_sysctl *s = container_of(hw, struct k210_sysctl, hw); - u32 clksel0, pll0; - u64 pll0_freq, clkr0, clkf0, clkod0; - - /* - * If the clock selector is not set, use the base frequency. - * Otherwise, use PLL0 frequency with a frequency divisor. - */ - clksel0 = readl(s->regs + K210_SYSCTL_CLKSEL0); - if (!(clksel0 & CLKSEL_ACLK)) - return K210_SYSCTL_CLK0_FREQ; - - /* - * Get PLL0 frequency: - * freq = base frequency * clkf0 / (clkr0 * clkod0) - */ - pll0 = readl(s->regs + K210_SYSCTL_PLL0); - clkr0 = 1 + FIELD_GET(GENMASK(3, 0), pll0); - clkf0 = 1 + FIELD_GET(GENMASK(9, 4), pll0); - clkod0 = 1 + FIELD_GET(GENMASK(13, 10), pll0); - pll0_freq = clkf0 * K210_SYSCTL_CLK0_FREQ / (clkr0 * clkod0); - - /* Get the frequency divisor from the clock selector */ - return pll0_freq / (2ULL << FIELD_GET(0x00000006, clksel0)); -} - -static const struct clk_ops k210_sysctl_clk_ops = { - .recalc_rate = k210_sysctl_clk_recalc_rate, -}; - -static const struct clk_init_data k210_clk_init_data = { - .name = "k210-sysctl-pll1", - .ops = &k210_sysctl_clk_ops, -}; - -static int k210_sysctl_probe(struct platform_device *pdev) -{ - struct k210_sysctl *s; - int error; - - pr_info("Kendryte K210 SoC sysctl\n"); - - s = devm_kzalloc(&pdev->dev, sizeof(*s), GFP_KERNEL); - if (!s) - return -ENOMEM; - - s->regs = devm_ioremap_resource(&pdev->dev, - platform_get_resource(pdev, IORESOURCE_MEM, 0)); - if (IS_ERR(s->regs)) - return PTR_ERR(s->regs); - - s->hw.init = &k210_clk_init_data; - error = devm_clk_hw_register(&pdev->dev, &s->hw); - if (error) { - dev_err(&pdev->dev, "failed to register clk"); - return error; - } - - error = devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_simple_get, - &s->hw); - if (error) { - dev_err(&pdev->dev, "adding clk provider failed\n"); - return error; - } - - return 0; -} - -static const struct of_device_id k210_sysctl_of_match[] = { - { .compatible = "kendryte,k210-sysctl", }, - {} -}; - -static struct platform_driver k210_sysctl_driver = { - .driver = { - .name = "k210-sysctl", - .of_match_table = k210_sysctl_of_match, - }, - .probe = k210_sysctl_probe, -}; - -static int __init k210_sysctl_init(void) -{ - return platform_driver_register(&k210_sysctl_driver); -} -core_initcall(k210_sysctl_init); - -/* - * This needs to be called very early during initialization, given that - * PLL1 needs to be enabled to be able to use all SRAM. - */ -static void __init k210_soc_early_init(const void *fdt) -{ - void __iomem *regs; - - regs = ioremap(K210_SYSCTL_SYSCTL_BASE_ADDR, 0x1000); - if (!regs) - panic("K210 sysctl ioremap"); - - /* Enable PLL1 to make the KPU SRAM useable */ - k210_pll1_enable(regs); - - k210_set_bits(PLL_OUT_EN, regs + K210_SYSCTL_PLL0); - - k210_set_bits(CLKEN_CPU | CLKEN_SRAM0 | CLKEN_SRAM1, - regs + K210_SYSCTL_CLKEN_CENT); - k210_set_bits(CLKEN_ROM | CLKEN_TIMER0 | CLKEN_RTC, - regs + K210_SYSCTL_CLKEN_PERI); - - k210_set_bits(CLKSEL_ACLK, regs + K210_SYSCTL_CLKSEL0); - - iounmap(regs); -} -SOC_EARLY_INIT_DECLARE(generic_k210, "kendryte,k210", k210_soc_early_init); - -#ifdef CONFIG_SOC_KENDRYTE_K210_DTB_BUILTIN -/* - * Generic entry for the default k210.dtb embedded DTB for boards with: - * - Vendor ID: 0x4B5 - * - Arch ID: 0xE59889E6A5A04149 (= "Canaan AI" in UTF-8 encoded Chinese) - * - Impl ID: 0x4D41495832303030 (= "MAIX2000") - * These values are reported by the SiPEED MAXDUINO, SiPEED MAIX GO and - * SiPEED Dan dock boards. - */ -SOC_BUILTIN_DTB_DECLARE(k210, 0x4B5, 0xE59889E6A5A04149, 0x4D41495832303030); -#endif diff --git a/drivers/soc/litex/Kconfig b/drivers/soc/litex/Kconfig index 7a7c38282e11..e7011d665b15 100644 --- a/drivers/soc/litex/Kconfig +++ b/drivers/soc/litex/Kconfig @@ -12,9 +12,21 @@ config LITEX_SOC_CONTROLLER select LITEX help This option enables the SoC Controller Driver which verifies - LiteX CSR access and provides common litex_get_reg/litex_set_reg + LiteX CSR access and provides common litex_[read|write]* accessors. All drivers that use functions from litex.h must depend on LITEX. +config LITEX_SUBREG_SIZE + int "Size of a LiteX CSR subregister, in bytes" + depends on LITEX + range 1 4 + default 4 + help + LiteX MMIO registers (referred to as Configuration and Status + registers, or CSRs) are spread across adjacent 8- or 32-bit + subregisters, located at 32-bit aligned MMIO addresses. Use + this to select the appropriate size (1 or 4 bytes) matching + your particular LiteX build. + endmenu diff --git a/drivers/soc/litex/litex_soc_ctrl.c b/drivers/soc/litex/litex_soc_ctrl.c index 9b0766384570..6268bfa7f0d6 100644 --- a/drivers/soc/litex/litex_soc_ctrl.c +++ b/drivers/soc/litex/litex_soc_ctrl.c @@ -15,79 +15,11 @@ #include <linux/module.h> #include <linux/errno.h> #include <linux/io.h> +#include <linux/reboot.h> -/* - * LiteX SoC Generator, depending on the configuration, can split a single - * logical CSR (Control&Status Register) into a series of consecutive physical - * registers. - * - * For example, in the configuration with 8-bit CSR Bus, 32-bit aligned (the - * default one for 32-bit CPUs) a 32-bit logical CSR will be generated as four - * 32-bit physical registers, each one containing one byte of meaningful data. - * - * For details see: https://github.com/enjoy-digital/litex/wiki/CSR-Bus - * - * The purpose of `litex_set_reg`/`litex_get_reg` is to implement the logic - * of writing to/reading from the LiteX CSR in a single place that can be - * then reused by all LiteX drivers. - */ - -/** - * litex_set_reg() - Writes the value to the LiteX CSR (Control&Status Register) - * @reg: Address of the CSR - * @reg_size: The width of the CSR expressed in the number of bytes - * @val: Value to be written to the CSR - * - * In the currently supported LiteX configuration (8-bit CSR Bus, 32-bit aligned), - * a 32-bit LiteX CSR is generated as 4 consecutive 32-bit physical registers, - * each one containing one byte of meaningful data. - * - * This function splits a single possibly multi-byte write into a series of - * single-byte writes with a proper offset. - */ -void litex_set_reg(void __iomem *reg, unsigned long reg_size, - unsigned long val) -{ - unsigned long shifted_data, shift, i; - - for (i = 0; i < reg_size; ++i) { - shift = ((reg_size - i - 1) * LITEX_SUBREG_SIZE_BIT); - shifted_data = val >> shift; - - WRITE_LITEX_SUBREGISTER(shifted_data, reg, i); - } -} -EXPORT_SYMBOL_GPL(litex_set_reg); - -/** - * litex_get_reg() - Reads the value of the LiteX CSR (Control&Status Register) - * @reg: Address of the CSR - * @reg_size: The width of the CSR expressed in the number of bytes - * - * Return: Value read from the CSR - * - * In the currently supported LiteX configuration (8-bit CSR Bus, 32-bit aligned), - * a 32-bit LiteX CSR is generated as 4 consecutive 32-bit physical registers, - * each one containing one byte of meaningful data. - * - * This function generates a series of single-byte reads with a proper offset - * and joins their results into a single multi-byte value. - */ -unsigned long litex_get_reg(void __iomem *reg, unsigned long reg_size) -{ - unsigned long shifted_data, shift, i; - unsigned long result = 0; - - for (i = 0; i < reg_size; ++i) { - shifted_data = READ_LITEX_SUBREGISTER(reg, i); - - shift = ((reg_size - i - 1) * LITEX_SUBREG_SIZE_BIT); - result |= (shifted_data << shift); - } - - return result; -} -EXPORT_SYMBOL_GPL(litex_get_reg); +/* reset register located at the base address */ +#define RESET_REG_OFF 0x00 +#define RESET_REG_VALUE 0x00000001 #define SCRATCH_REG_OFF 0x04 #define SCRATCH_REG_VALUE 0x12345678 @@ -131,15 +63,27 @@ static int litex_check_csr_access(void __iomem *reg_addr) /* restore original value of the SCRATCH register */ litex_write32(reg_addr + SCRATCH_REG_OFF, SCRATCH_REG_VALUE); - pr_info("LiteX SoC Controller driver initialized"); + pr_info("LiteX SoC Controller driver initialized: subreg:%d, align:%d", + LITEX_SUBREG_SIZE, LITEX_SUBREG_ALIGN); return 0; } struct litex_soc_ctrl_device { void __iomem *base; + struct notifier_block reset_nb; }; +static int litex_reset_handler(struct notifier_block *this, unsigned long mode, + void *cmd) +{ + struct litex_soc_ctrl_device *soc_ctrl_dev = + container_of(this, struct litex_soc_ctrl_device, reset_nb); + + litex_write32(soc_ctrl_dev->base + RESET_REG_OFF, RESET_REG_VALUE); + return NOTIFY_DONE; +} + #ifdef CONFIG_OF static const struct of_device_id litex_soc_ctrl_of_match[] = { {.compatible = "litex,soc-controller"}, @@ -151,6 +95,7 @@ MODULE_DEVICE_TABLE(of, litex_soc_ctrl_of_match); static int litex_soc_ctrl_probe(struct platform_device *pdev) { struct litex_soc_ctrl_device *soc_ctrl_dev; + int error; soc_ctrl_dev = devm_kzalloc(&pdev->dev, sizeof(*soc_ctrl_dev), GFP_KERNEL); if (!soc_ctrl_dev) @@ -160,7 +105,29 @@ static int litex_soc_ctrl_probe(struct platform_device *pdev) if (IS_ERR(soc_ctrl_dev->base)) return PTR_ERR(soc_ctrl_dev->base); - return litex_check_csr_access(soc_ctrl_dev->base); + error = litex_check_csr_access(soc_ctrl_dev->base); + if (error) + return error; + + platform_set_drvdata(pdev, soc_ctrl_dev); + + soc_ctrl_dev->reset_nb.notifier_call = litex_reset_handler; + soc_ctrl_dev->reset_nb.priority = 128; + error = register_restart_handler(&soc_ctrl_dev->reset_nb); + if (error) { + dev_warn(&pdev->dev, "cannot register restart handler: %d\n", + error); + } + + return 0; +} + +static int litex_soc_ctrl_remove(struct platform_device *pdev) +{ + struct litex_soc_ctrl_device *soc_ctrl_dev = platform_get_drvdata(pdev); + + unregister_restart_handler(&soc_ctrl_dev->reset_nb); + return 0; } static struct platform_driver litex_soc_ctrl_driver = { @@ -169,6 +136,7 @@ static struct platform_driver litex_soc_ctrl_driver = { .of_match_table = of_match_ptr(litex_soc_ctrl_of_match) }, .probe = litex_soc_ctrl_probe, + .remove = litex_soc_ctrl_remove, }; module_platform_driver(litex_soc_ctrl_driver); diff --git a/drivers/soc/sifive/sifive_l2_cache.c b/drivers/soc/sifive/sifive_l2_cache.c index 44d7e1951da3..59640a1d0b28 100644 --- a/drivers/soc/sifive/sifive_l2_cache.c +++ b/drivers/soc/sifive/sifive_l2_cache.c @@ -17,6 +17,10 @@ #define SIFIVE_L2_DIRECCFIX_HIGH 0x104 #define SIFIVE_L2_DIRECCFIX_COUNT 0x108 +#define SIFIVE_L2_DIRECCFAIL_LOW 0x120 +#define SIFIVE_L2_DIRECCFAIL_HIGH 0x124 +#define SIFIVE_L2_DIRECCFAIL_COUNT 0x128 + #define SIFIVE_L2_DATECCFIX_LOW 0x140 #define SIFIVE_L2_DATECCFIX_HIGH 0x144 #define SIFIVE_L2_DATECCFIX_COUNT 0x148 @@ -29,7 +33,7 @@ #define SIFIVE_L2_WAYENABLE 0x08 #define SIFIVE_L2_ECCINJECTERR 0x40 -#define SIFIVE_L2_MAX_ECCINTR 3 +#define SIFIVE_L2_MAX_ECCINTR 4 static void __iomem *l2_base; static int g_irq[SIFIVE_L2_MAX_ECCINTR]; @@ -39,6 +43,7 @@ enum { DIR_CORR = 0, DATA_CORR, DATA_UNCORR, + DIR_UNCORR, }; #ifdef CONFIG_DEBUG_FS @@ -93,6 +98,7 @@ static void l2_config_read(void) static const struct of_device_id sifive_l2_ids[] = { { .compatible = "sifive,fu540-c000-ccache" }, + { .compatible = "sifive,fu740-c000-ccache" }, { /* end of table */ }, }; @@ -155,6 +161,15 @@ static irqreturn_t l2_int_handler(int irq, void *device) atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_CE, "DirECCFix"); } + if (irq == g_irq[DIR_UNCORR]) { + add_h = readl(l2_base + SIFIVE_L2_DIRECCFAIL_HIGH); + add_l = readl(l2_base + SIFIVE_L2_DIRECCFAIL_LOW); + /* Reading this register clears the DirFail interrupt sig */ + readl(l2_base + SIFIVE_L2_DIRECCFAIL_COUNT); + atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_UE, + "DirECCFail"); + panic("L2CACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l); + } if (irq == g_irq[DATA_CORR]) { add_h = readl(l2_base + SIFIVE_L2_DATECCFIX_HIGH); add_l = readl(l2_base + SIFIVE_L2_DATECCFIX_LOW); @@ -181,7 +196,7 @@ static int __init sifive_l2_init(void) { struct device_node *np; struct resource res; - int i, rc; + int i, rc, intr_num; np = of_find_matching_node(NULL, sifive_l2_ids); if (!np) @@ -194,7 +209,13 @@ static int __init sifive_l2_init(void) if (!l2_base) return -ENOMEM; - for (i = 0; i < SIFIVE_L2_MAX_ECCINTR; i++) { + intr_num = of_property_count_u32_elems(np, "interrupts"); + if (!intr_num) { + pr_err("L2CACHE: no interrupts property\n"); + return -ENODEV; + } + + for (i = 0; i < intr_num; i++) { g_irq[i] = irq_of_parse_and_map(np, i); rc = request_irq(g_irq[i], l2_int_handler, 0, "l2_ecc", NULL); if (rc) { diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index d1e8c3a54976..46885429928a 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -267,8 +267,10 @@ static int sdw_transfer_unlocked(struct sdw_bus *bus, struct sdw_msg *msg) ret = do_transfer(bus, msg); if (ret != 0 && ret != -ENODATA) - dev_err(bus->dev, "trf on Slave %d failed:%d\n", - msg->dev_num, ret); + dev_err(bus->dev, "trf on Slave %d failed:%d %s addr %x count %d\n", + msg->dev_num, ret, + (msg->flags & SDW_MSG_FLAG_WRITE) ? "write" : "read", + msg->addr, msg->len); if (msg->page) sdw_reset_page(bus, msg->dev_num); @@ -405,10 +407,11 @@ sdw_nwrite_no_pm(struct sdw_slave *slave, u32 addr, size_t count, u8 *val) return sdw_transfer(slave->bus, &msg); } -static int sdw_write_no_pm(struct sdw_slave *slave, u32 addr, u8 value) +int sdw_write_no_pm(struct sdw_slave *slave, u32 addr, u8 value) { return sdw_nwrite_no_pm(slave, addr, 1, &value); } +EXPORT_SYMBOL(sdw_write_no_pm); static int sdw_bread_no_pm(struct sdw_bus *bus, u16 dev_num, u32 addr) @@ -476,8 +479,7 @@ int sdw_bwrite_no_pm_unlocked(struct sdw_bus *bus, u16 dev_num, u32 addr, u8 val } EXPORT_SYMBOL(sdw_bwrite_no_pm_unlocked); -static int -sdw_read_no_pm(struct sdw_slave *slave, u32 addr) +int sdw_read_no_pm(struct sdw_slave *slave, u32 addr) { u8 buf; int ret; @@ -488,6 +490,19 @@ sdw_read_no_pm(struct sdw_slave *slave, u32 addr) else return buf; } +EXPORT_SYMBOL(sdw_read_no_pm); + +static int sdw_update_no_pm(struct sdw_slave *slave, u32 addr, u8 mask, u8 val) +{ + int tmp; + + tmp = sdw_read_no_pm(slave, addr); + if (tmp < 0) + return tmp; + + tmp = (tmp & ~mask) | val; + return sdw_write_no_pm(slave, addr, tmp); +} /** * sdw_nread() - Read "n" contiguous SDW Slave registers @@ -500,16 +515,16 @@ int sdw_nread(struct sdw_slave *slave, u32 addr, size_t count, u8 *val) { int ret; - ret = pm_runtime_get_sync(slave->bus->dev); + ret = pm_runtime_get_sync(&slave->dev); if (ret < 0 && ret != -EACCES) { - pm_runtime_put_noidle(slave->bus->dev); + pm_runtime_put_noidle(&slave->dev); return ret; } ret = sdw_nread_no_pm(slave, addr, count, val); - pm_runtime_mark_last_busy(slave->bus->dev); - pm_runtime_put(slave->bus->dev); + pm_runtime_mark_last_busy(&slave->dev); + pm_runtime_put(&slave->dev); return ret; } @@ -526,16 +541,16 @@ int sdw_nwrite(struct sdw_slave *slave, u32 addr, size_t count, u8 *val) { int ret; - ret = pm_runtime_get_sync(slave->bus->dev); + ret = pm_runtime_get_sync(&slave->dev); if (ret < 0 && ret != -EACCES) { - pm_runtime_put_noidle(slave->bus->dev); + pm_runtime_put_noidle(&slave->dev); return ret; } ret = sdw_nwrite_no_pm(slave, addr, count, val); - pm_runtime_mark_last_busy(slave->bus->dev); - pm_runtime_put(slave->bus->dev); + pm_runtime_mark_last_busy(&slave->dev); + pm_runtime_put(&slave->dev); return ret; } @@ -623,6 +638,7 @@ err: static int sdw_assign_device_num(struct sdw_slave *slave) { + struct sdw_bus *bus = slave->bus; int ret, dev_num; bool new_device = false; @@ -633,7 +649,7 @@ static int sdw_assign_device_num(struct sdw_slave *slave) dev_num = sdw_get_device_num(slave); mutex_unlock(&slave->bus->bus_lock); if (dev_num < 0) { - dev_err(slave->bus->dev, "Get dev_num failed: %d\n", + dev_err(bus->dev, "Get dev_num failed: %d\n", dev_num); return dev_num; } @@ -646,7 +662,7 @@ static int sdw_assign_device_num(struct sdw_slave *slave) } if (!new_device) - dev_dbg(slave->bus->dev, + dev_dbg(bus->dev, "Slave already registered, reusing dev_num:%d\n", slave->dev_num); @@ -656,7 +672,7 @@ static int sdw_assign_device_num(struct sdw_slave *slave) ret = sdw_write_no_pm(slave, SDW_SCP_DEVNUMBER, dev_num); if (ret < 0) { - dev_err(&slave->dev, "Program device_num %d failed: %d\n", + dev_err(bus->dev, "Program device_num %d failed: %d\n", dev_num, ret); return ret; } @@ -679,9 +695,8 @@ void sdw_extract_slave_id(struct sdw_bus *bus, id->class_id = SDW_CLASS_ID(addr); dev_dbg(bus->dev, - "SDW Slave class_id %x, part_id %x, mfg_id %x, unique_id %x, version %x\n", - id->class_id, id->part_id, id->mfg_id, - id->unique_id, id->sdw_version); + "SDW Slave class_id 0x%02x, mfg_id 0x%04x, part_id 0x%04x, unique_id 0x%x, version 0x%x\n", + id->class_id, id->mfg_id, id->part_id, id->unique_id, id->sdw_version); } static int sdw_program_device_num(struct sdw_bus *bus) @@ -735,7 +750,7 @@ static int sdw_program_device_num(struct sdw_bus *bus) */ ret = sdw_assign_device_num(slave); if (ret) { - dev_err(slave->bus->dev, + dev_err(bus->dev, "Assign dev_num failed:%d\n", ret); return ret; @@ -775,15 +790,17 @@ static int sdw_program_device_num(struct sdw_bus *bus) static void sdw_modify_slave_status(struct sdw_slave *slave, enum sdw_slave_status status) { - mutex_lock(&slave->bus->bus_lock); + struct sdw_bus *bus = slave->bus; - dev_vdbg(&slave->dev, + mutex_lock(&bus->bus_lock); + + dev_vdbg(bus->dev, "%s: changing status slave %d status %d new status %d\n", __func__, slave->dev_num, slave->status, status); if (status == SDW_SLAVE_UNATTACHED) { dev_dbg(&slave->dev, - "%s: initializing completion for Slave %d\n", + "%s: initializing enumeration and init completion for Slave %d\n", __func__, slave->dev_num); init_completion(&slave->enumeration_complete); @@ -792,13 +809,13 @@ static void sdw_modify_slave_status(struct sdw_slave *slave, } else if ((status == SDW_SLAVE_ATTACHED) && (slave->status == SDW_SLAVE_UNATTACHED)) { dev_dbg(&slave->dev, - "%s: signaling completion for Slave %d\n", + "%s: signaling enumeration completion for Slave %d\n", __func__, slave->dev_num); complete(&slave->enumeration_complete); } slave->status = status; - mutex_unlock(&slave->bus->bus_lock); + mutex_unlock(&bus->bus_lock); } static enum sdw_clk_stop_mode sdw_get_clk_stop_mode(struct sdw_slave *slave) @@ -950,17 +967,17 @@ int sdw_bus_prep_clk_stop(struct sdw_bus *bus) simple_clk_stop = false; } - if (is_slave && !simple_clk_stop) { + /* Skip remaining clock stop preparation if no Slave is attached */ + if (!is_slave) + return ret; + + if (!simple_clk_stop) { ret = sdw_bus_wait_for_clk_prep_deprep(bus, SDW_BROADCAST_DEV_NUM); if (ret < 0) return ret; } - /* Don't need to inform slaves if there is no slave attached */ - if (!is_slave) - return ret; - /* Inform slaves that prep is done */ list_for_each_entry(slave, &bus->slaves, node) { if (!slave->dev_num) @@ -1074,16 +1091,13 @@ int sdw_bus_exit_clk_stop(struct sdw_bus *bus) "clk stop deprep failed:%d", ret); } - if (is_slave && !simple_clk_stop) - sdw_bus_wait_for_clk_prep_deprep(bus, SDW_BROADCAST_DEV_NUM); - - /* - * Don't need to call slave callback function if there is no slave - * attached - */ + /* Skip remaining clock stop de-preparation if no Slave is attached */ if (!is_slave) return 0; + if (!simple_clk_stop) + sdw_bus_wait_for_clk_prep_deprep(bus, SDW_BROADCAST_DEV_NUM); + list_for_each_entry(slave, &bus->slaves, node) { if (!slave->dev_num) continue; @@ -1127,7 +1141,7 @@ int sdw_configure_dpn_intr(struct sdw_slave *slave, ret = sdw_update(slave, addr, (mask | SDW_DPN_INT_PORT_READY), val); if (ret < 0) - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DPN_INTMASK write failed:%d\n", val); return ret; @@ -1210,7 +1224,7 @@ static int sdw_slave_set_frequency(struct sdw_slave *slave) } scale_index++; - ret = sdw_write(slave, SDW_SCP_BUS_CLOCK_BASE, base); + ret = sdw_write_no_pm(slave, SDW_SCP_BUS_CLOCK_BASE, base); if (ret < 0) { dev_err(&slave->dev, "SDW_SCP_BUS_CLOCK_BASE write failed:%d\n", ret); @@ -1218,13 +1232,13 @@ static int sdw_slave_set_frequency(struct sdw_slave *slave) } /* initialize scale for both banks */ - ret = sdw_write(slave, SDW_SCP_BUSCLOCK_SCALE_B0, scale_index); + ret = sdw_write_no_pm(slave, SDW_SCP_BUSCLOCK_SCALE_B0, scale_index); if (ret < 0) { dev_err(&slave->dev, "SDW_SCP_BUSCLOCK_SCALE_B0 write failed:%d\n", ret); return ret; } - ret = sdw_write(slave, SDW_SCP_BUSCLOCK_SCALE_B1, scale_index); + ret = sdw_write_no_pm(slave, SDW_SCP_BUSCLOCK_SCALE_B1, scale_index); if (ret < 0) dev_err(&slave->dev, "SDW_SCP_BUSCLOCK_SCALE_B1 write failed:%d\n", ret); @@ -1256,9 +1270,9 @@ static int sdw_initialize_slave(struct sdw_slave *slave) val = slave->prop.scp_int1_mask; /* Enable SCP interrupts */ - ret = sdw_update(slave, SDW_SCP_INTMASK1, val, val); + ret = sdw_update_no_pm(slave, SDW_SCP_INTMASK1, val, val); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_SCP_INTMASK1 write failed:%d\n", ret); return ret; } @@ -1271,9 +1285,9 @@ static int sdw_initialize_slave(struct sdw_slave *slave) val = prop->dp0_prop->imp_def_interrupts; val |= SDW_DP0_INT_PORT_READY | SDW_DP0_INT_BRA_FAILURE; - ret = sdw_update(slave, SDW_DP0_INTMASK, val, val); + ret = sdw_update_no_pm(slave, SDW_DP0_INTMASK, val, val); if (ret < 0) - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DP0_INTMASK read failed:%d\n", ret); return ret; } @@ -1283,9 +1297,9 @@ static int sdw_handle_dp0_interrupt(struct sdw_slave *slave, u8 *slave_status) u8 clear, impl_int_mask; int status, status2, ret, count = 0; - status = sdw_read(slave, SDW_DP0_INT); + status = sdw_read_no_pm(slave, SDW_DP0_INT); if (status < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DP0_INT read failed:%d\n", status); return status; } @@ -1322,17 +1336,17 @@ static int sdw_handle_dp0_interrupt(struct sdw_slave *slave, u8 *slave_status) } /* clear the interrupts but don't touch reserved and SDCA_CASCADE fields */ - ret = sdw_write(slave, SDW_DP0_INT, clear); + ret = sdw_write_no_pm(slave, SDW_DP0_INT, clear); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DP0_INT write failed:%d\n", ret); return ret; } /* Read DP0 interrupt again */ - status2 = sdw_read(slave, SDW_DP0_INT); + status2 = sdw_read_no_pm(slave, SDW_DP0_INT); if (status2 < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DP0_INT read failed:%d\n", status2); return status2; } @@ -1345,7 +1359,7 @@ static int sdw_handle_dp0_interrupt(struct sdw_slave *slave, u8 *slave_status) } while ((status & SDW_DP0_INTERRUPTS) && (count < SDW_READ_INTR_CLEAR_RETRY)); if (count == SDW_READ_INTR_CLEAR_RETRY) - dev_warn(slave->bus->dev, "Reached MAX_RETRY on DP0 read\n"); + dev_warn(&slave->dev, "Reached MAX_RETRY on DP0 read\n"); return ret; } @@ -1361,9 +1375,9 @@ static int sdw_handle_port_interrupt(struct sdw_slave *slave, return sdw_handle_dp0_interrupt(slave, slave_status); addr = SDW_DPN_INT(port); - status = sdw_read(slave, addr); + status = sdw_read_no_pm(slave, addr); if (status < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DPN_INT read failed:%d\n", status); return status; @@ -1395,17 +1409,17 @@ static int sdw_handle_port_interrupt(struct sdw_slave *slave, } /* clear the interrupt but don't touch reserved fields */ - ret = sdw_write(slave, addr, clear); + ret = sdw_write_no_pm(slave, addr, clear); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DPN_INT write failed:%d\n", ret); return ret; } /* Read DPN interrupt again */ - status2 = sdw_read(slave, addr); + status2 = sdw_read_no_pm(slave, addr); if (status2 < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DPN_INT read failed:%d\n", status2); return status2; } @@ -1418,7 +1432,7 @@ static int sdw_handle_port_interrupt(struct sdw_slave *slave, } while ((status & SDW_DPN_INTERRUPTS) && (count < SDW_READ_INTR_CLEAR_RETRY)); if (count == SDW_READ_INTR_CLEAR_RETRY) - dev_warn(slave->bus->dev, "Reached MAX_RETRY on port read"); + dev_warn(&slave->dev, "Reached MAX_RETRY on port read"); return ret; } @@ -1440,30 +1454,30 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) ret = pm_runtime_get_sync(&slave->dev); if (ret < 0 && ret != -EACCES) { dev_err(&slave->dev, "Failed to resume device: %d\n", ret); - pm_runtime_put_noidle(slave->bus->dev); + pm_runtime_put_noidle(&slave->dev); return ret; } /* Read Intstat 1, Intstat 2 and Intstat 3 registers */ - ret = sdw_read(slave, SDW_SCP_INT1); + ret = sdw_read_no_pm(slave, SDW_SCP_INT1); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_SCP_INT1 read failed:%d\n", ret); goto io_err; } buf = ret; - ret = sdw_nread(slave, SDW_SCP_INTSTAT2, 2, buf2); + ret = sdw_nread_no_pm(slave, SDW_SCP_INTSTAT2, 2, buf2); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_SCP_INT2/3 read failed:%d\n", ret); goto io_err; } if (slave->prop.is_sdca) { - ret = sdw_read(slave, SDW_DP0_INT); + ret = sdw_read_no_pm(slave, SDW_DP0_INT); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DP0_INT read failed:%d\n", ret); goto io_err; } @@ -1558,9 +1572,9 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) } /* Ack interrupt */ - ret = sdw_write(slave, SDW_SCP_INT1, clear); + ret = sdw_write_no_pm(slave, SDW_SCP_INT1, clear); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_SCP_INT1 write failed:%d\n", ret); goto io_err; } @@ -1572,25 +1586,25 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) * Read status again to ensure no new interrupts arrived * while servicing interrupts. */ - ret = sdw_read(slave, SDW_SCP_INT1); + ret = sdw_read_no_pm(slave, SDW_SCP_INT1); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_SCP_INT1 read failed:%d\n", ret); goto io_err; } _buf = ret; - ret = sdw_nread(slave, SDW_SCP_INTSTAT2, 2, _buf2); + ret = sdw_nread_no_pm(slave, SDW_SCP_INTSTAT2, 2, _buf2); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_SCP_INT2/3 read failed:%d\n", ret); goto io_err; } if (slave->prop.is_sdca) { - ret = sdw_read(slave, SDW_DP0_INT); + ret = sdw_read_no_pm(slave, SDW_DP0_INT); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DP0_INT read failed:%d\n", ret); goto io_err; } @@ -1616,7 +1630,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) } while (stat != 0 && count < SDW_READ_INTR_CLEAR_RETRY); if (count == SDW_READ_INTR_CLEAR_RETRY) - dev_warn(slave->bus->dev, "Reached MAX_RETRY on alert read\n"); + dev_warn(&slave->dev, "Reached MAX_RETRY on alert read\n"); io_err: pm_runtime_mark_last_busy(&slave->dev); @@ -1722,7 +1736,7 @@ int sdw_handle_slave_status(struct sdw_bus *bus, case SDW_SLAVE_ALERT: ret = sdw_handle_slave_alerts(slave); if (ret) - dev_err(bus->dev, + dev_err(&slave->dev, "Slave %d alert handling failed: %d\n", i, ret); break; @@ -1741,24 +1755,29 @@ int sdw_handle_slave_status(struct sdw_bus *bus, ret = sdw_initialize_slave(slave); if (ret) - dev_err(bus->dev, + dev_err(&slave->dev, "Slave %d initialization failed: %d\n", i, ret); break; default: - dev_err(bus->dev, "Invalid slave %d status:%d\n", + dev_err(&slave->dev, "Invalid slave %d status:%d\n", i, status[i]); break; } ret = sdw_update_slave_status(slave, status[i]); if (ret) - dev_err(slave->bus->dev, + dev_err(&slave->dev, "Update Slave status failed:%d\n", ret); - if (attached_initializing) + if (attached_initializing) { + dev_dbg(&slave->dev, + "%s: signaling initialization completion for Slave %d\n", + __func__, slave->dev_num); + complete(&slave->initialization_complete); + } } return ret; diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c index 9fa55164354a..d05442e646a3 100644 --- a/drivers/soundwire/cadence_master.c +++ b/drivers/soundwire/cadence_master.c @@ -188,7 +188,7 @@ MODULE_PARM_DESC(cdns_mcp_int_mask, "Cadence MCP IntMask"); #define CDNS_PDI_CONFIG_PORT GENMASK(4, 0) /* Driver defaults */ -#define CDNS_TX_TIMEOUT 2000 +#define CDNS_TX_TIMEOUT 500 #define CDNS_SCP_RX_FIFOLEVEL 0x2 @@ -483,11 +483,11 @@ cdns_fill_msg_resp(struct sdw_cdns *cdns, for (i = 0; i < count; i++) { if (!(cdns->response_buf[i] & CDNS_MCP_RESP_ACK)) { no_ack = 1; - dev_dbg_ratelimited(cdns->dev, "Msg Ack not received\n"); - if (cdns->response_buf[i] & CDNS_MCP_RESP_NACK) { - nack = 1; - dev_err_ratelimited(cdns->dev, "Msg NACK received\n"); - } + dev_vdbg(cdns->dev, "Msg Ack not received, cmd %d\n", i); + } + if (cdns->response_buf[i] & CDNS_MCP_RESP_NACK) { + nack = 1; + dev_err_ratelimited(cdns->dev, "Msg NACK received, cmd %d\n", i); } } @@ -734,21 +734,18 @@ static void cdns_read_response(struct sdw_cdns *cdns) } static int cdns_update_slave_status(struct sdw_cdns *cdns, - u32 slave0, u32 slave1) + u64 slave_intstat) { enum sdw_slave_status status[SDW_MAX_DEVICES + 1]; bool is_slave = false; - u64 slave; u32 mask; int i, set_status; - /* combine the two status */ - slave = ((u64)slave1 << 32) | slave0; memset(status, 0, sizeof(status)); for (i = 0; i <= SDW_MAX_DEVICES; i++) { - mask = (slave >> (i * CDNS_MCP_SLAVE_STATUS_NUM)) & - CDNS_MCP_SLAVE_STATUS_BITS; + mask = (slave_intstat >> (i * CDNS_MCP_SLAVE_STATUS_NUM)) & + CDNS_MCP_SLAVE_STATUS_BITS; if (!mask) continue; @@ -918,13 +915,17 @@ static void cdns_update_slave_status_work(struct work_struct *work) struct sdw_cdns *cdns = container_of(work, struct sdw_cdns, work); u32 slave0, slave1; - - dev_dbg_ratelimited(cdns->dev, "Slave status change\n"); + u64 slave_intstat; slave0 = cdns_readl(cdns, CDNS_MCP_SLAVE_INTSTAT0); slave1 = cdns_readl(cdns, CDNS_MCP_SLAVE_INTSTAT1); - cdns_update_slave_status(cdns, slave0, slave1); + /* combine the two status */ + slave_intstat = ((u64)slave1 << 32) | slave0; + + dev_dbg_ratelimited(cdns->dev, "Slave status change: 0x%llx\n", slave_intstat); + + cdns_update_slave_status(cdns, slave_intstat); cdns_writel(cdns, CDNS_MCP_SLAVE_INTSTAT0, slave0); cdns_writel(cdns, CDNS_MCP_SLAVE_INTSTAT1, slave1); diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c index 66adb258a425..a2d5cdaa9998 100644 --- a/drivers/soundwire/intel.c +++ b/drivers/soundwire/intel.c @@ -967,7 +967,7 @@ static int intel_hw_params(struct snd_pcm_substream *substream, } /* Port configuration */ - pconfig = kcalloc(1, sizeof(*pconfig), GFP_KERNEL); + pconfig = kzalloc(sizeof(*pconfig), GFP_KERNEL); if (!pconfig) { ret = -ENOMEM; goto error; @@ -1673,10 +1673,12 @@ static int __maybe_unused intel_suspend_runtime(struct device *dev) } else if (clock_stop_quirks & SDW_INTEL_CLK_STOP_BUS_RESET || !clock_stop_quirks) { + bool wake_enable = true; + ret = sdw_cdns_clock_stop(cdns, true); if (ret < 0) { dev_err(dev, "cannot enable clock stop on suspend\n"); - return ret; + wake_enable = false; } ret = sdw_cdns_enable_interrupt(cdns, false); @@ -1691,7 +1693,7 @@ static int __maybe_unused intel_suspend_runtime(struct device *dev) return ret; } - intel_shim_wake(sdw, true); + intel_shim_wake(sdw, wake_enable); } else { dev_err(dev, "%s clock_stop_quirks %x unsupported\n", __func__, clock_stop_quirks); diff --git a/drivers/soundwire/intel.h b/drivers/soundwire/intel.h index 76820d0b9deb..06bac8ba14e9 100644 --- a/drivers/soundwire/intel.h +++ b/drivers/soundwire/intel.h @@ -48,8 +48,6 @@ struct sdw_intel { #endif }; -#define SDW_INTEL_QUIRK_MASK_BUS_DISABLE BIT(1) - int intel_master_startup(struct platform_device *pdev); int intel_master_process_wakeen_event(struct platform_device *pdev); diff --git a/drivers/soundwire/intel_init.c b/drivers/soundwire/intel_init.c index cabdadb09a1b..05b726cdfebc 100644 --- a/drivers/soundwire/intel_init.c +++ b/drivers/soundwire/intel_init.c @@ -18,42 +18,12 @@ #include "cadence_master.h" #include "intel.h" -#define SDW_LINK_TYPE 4 /* from Intel ACPI documentation */ -#define SDW_MAX_LINKS 4 #define SDW_SHIM_LCAP 0x0 #define SDW_SHIM_BASE 0x2C000 #define SDW_ALH_BASE 0x2C800 #define SDW_LINK_BASE 0x30000 #define SDW_LINK_SIZE 0x10000 -static int ctrl_link_mask; -module_param_named(sdw_link_mask, ctrl_link_mask, int, 0444); -MODULE_PARM_DESC(sdw_link_mask, "Intel link mask (one bit per link)"); - -static bool is_link_enabled(struct fwnode_handle *fw_node, int i) -{ - struct fwnode_handle *link; - char name[32]; - u32 quirk_mask = 0; - - /* Find master handle */ - snprintf(name, sizeof(name), - "mipi-sdw-link-%d-subproperties", i); - - link = fwnode_get_named_child_node(fw_node, name); - if (!link) - return false; - - fwnode_property_read_u32(link, - "intel-quirk-mask", - &quirk_mask); - - if (quirk_mask & SDW_INTEL_QUIRK_MASK_BUS_DISABLE) - return false; - - return true; -} - static int sdw_intel_cleanup(struct sdw_intel_ctx *ctx) { struct sdw_intel_link_res *link = ctx->links; @@ -81,74 +51,6 @@ static int sdw_intel_cleanup(struct sdw_intel_ctx *ctx) return 0; } -static int -sdw_intel_scan_controller(struct sdw_intel_acpi_info *info) -{ - struct acpi_device *adev; - int ret, i; - u8 count; - - if (acpi_bus_get_device(info->handle, &adev)) - return -EINVAL; - - /* Found controller, find links supported */ - count = 0; - ret = fwnode_property_read_u8_array(acpi_fwnode_handle(adev), - "mipi-sdw-master-count", &count, 1); - - /* - * In theory we could check the number of links supported in - * hardware, but in that step we cannot assume SoundWire IP is - * powered. - * - * In addition, if the BIOS doesn't even provide this - * 'master-count' property then all the inits based on link - * masks will fail as well. - * - * We will check the hardware capabilities in the startup() step - */ - - if (ret) { - dev_err(&adev->dev, - "Failed to read mipi-sdw-master-count: %d\n", ret); - return -EINVAL; - } - - /* Check count is within bounds */ - if (count > SDW_MAX_LINKS) { - dev_err(&adev->dev, "Link count %d exceeds max %d\n", - count, SDW_MAX_LINKS); - return -EINVAL; - } - - if (!count) { - dev_warn(&adev->dev, "No SoundWire links detected\n"); - return -EINVAL; - } - dev_dbg(&adev->dev, "ACPI reports %d SDW Link devices\n", count); - - info->count = count; - info->link_mask = 0; - - for (i = 0; i < count; i++) { - if (ctrl_link_mask && !(ctrl_link_mask & BIT(i))) { - dev_dbg(&adev->dev, - "Link %d masked, will not be enabled\n", i); - continue; - } - - if (!is_link_enabled(acpi_fwnode_handle(adev), i)) { - dev_dbg(&adev->dev, - "Link %d not selected in firmware\n", i); - continue; - } - - info->link_mask |= BIT(i); - } - - return 0; -} - #define HDA_DSP_REG_ADSPIC2 (0x10) #define HDA_DSP_REG_ADSPIS2 (0x14) #define HDA_DSP_REG_ADSPIC2_SNDW BIT(5) @@ -357,65 +259,6 @@ sdw_intel_startup_controller(struct sdw_intel_ctx *ctx) return 0; } -static acpi_status sdw_intel_acpi_cb(acpi_handle handle, u32 level, - void *cdata, void **return_value) -{ - struct sdw_intel_acpi_info *info = cdata; - struct acpi_device *adev; - acpi_status status; - u64 adr; - - status = acpi_evaluate_integer(handle, METHOD_NAME__ADR, NULL, &adr); - if (ACPI_FAILURE(status)) - return AE_OK; /* keep going */ - - if (acpi_bus_get_device(handle, &adev)) { - pr_err("%s: Couldn't find ACPI handle\n", __func__); - return AE_NOT_FOUND; - } - - info->handle = handle; - - /* - * On some Intel platforms, multiple children of the HDAS - * device can be found, but only one of them is the SoundWire - * controller. The SNDW device is always exposed with - * Name(_ADR, 0x40000000), with bits 31..28 representing the - * SoundWire link so filter accordingly - */ - if (FIELD_GET(GENMASK(31, 28), adr) != SDW_LINK_TYPE) - return AE_OK; /* keep going */ - - /* device found, stop namespace walk */ - return AE_CTRL_TERMINATE; -} - -/** - * sdw_intel_acpi_scan() - SoundWire Intel init routine - * @parent_handle: ACPI parent handle - * @info: description of what firmware/DSDT tables expose - * - * This scans the namespace and queries firmware to figure out which - * links to enable. A follow-up use of sdw_intel_probe() and - * sdw_intel_startup() is required for creation of devices and bus - * startup - */ -int sdw_intel_acpi_scan(acpi_handle *parent_handle, - struct sdw_intel_acpi_info *info) -{ - acpi_status status; - - status = acpi_walk_namespace(ACPI_TYPE_DEVICE, - parent_handle, 1, - sdw_intel_acpi_cb, - NULL, info, NULL); - if (ACPI_FAILURE(status)) - return -ENODEV; - - return sdw_intel_scan_controller(info); -} -EXPORT_SYMBOL_NS(sdw_intel_acpi_scan, SOUNDWIRE_INTEL_INIT); - /** * sdw_intel_probe() - SoundWire Intel probe routine * @res: resource data diff --git a/drivers/soundwire/slave.c b/drivers/soundwire/slave.c index a08f4081c1c4..180f38bd003b 100644 --- a/drivers/soundwire/slave.c +++ b/drivers/soundwire/slave.c @@ -163,15 +163,13 @@ int sdw_acpi_find_slaves(struct sdw_bus *bus) if (id.unique_id != id2.unique_id) { dev_dbg(bus->dev, - "Valid unique IDs %x %x for Slave mfg %x part %d\n", - id.unique_id, id2.unique_id, - id.mfg_id, id.part_id); + "Valid unique IDs 0x%x 0x%x for Slave mfg_id 0x%04x, part_id 0x%04x\n", + id.unique_id, id2.unique_id, id.mfg_id, id.part_id); ignore_unique_id = false; } else { dev_err(bus->dev, - "Invalid unique IDs %x %x for Slave mfg %x part %d\n", - id.unique_id, id2.unique_id, - id.mfg_id, id.part_id); + "Invalid unique IDs 0x%x 0x%x for Slave mfg_id 0x%04x, part_id 0x%04x\n", + id.unique_id, id2.unique_id, id.mfg_id, id.part_id); return -ENODEV; } } diff --git a/drivers/soundwire/sysfs_slave.c b/drivers/soundwire/sysfs_slave.c index b48b6617a396..3210359cd944 100644 --- a/drivers/soundwire/sysfs_slave.c +++ b/drivers/soundwire/sysfs_slave.c @@ -130,7 +130,7 @@ static struct attribute *slave_dev_attrs[] = { * we don't use ATTRIBUTES_GROUP here since we want to add a subdirectory * for device-level properties */ -static struct attribute_group sdw_slave_dev_attr_group = { +static const struct attribute_group sdw_slave_dev_attr_group = { .attrs = slave_dev_attrs, .name = "dev-properties", }; diff --git a/drivers/spmi/spmi-pmic-arb.c b/drivers/spmi/spmi-pmic-arb.c index de844b412110..bbbd311eda03 100644 --- a/drivers/spmi/spmi-pmic-arb.c +++ b/drivers/spmi/spmi-pmic-arb.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2012-2015, 2017, The Linux Foundation. All rights reserved. + * Copyright (c) 2012-2015, 2017, 2021, The Linux Foundation. All rights reserved. */ #include <linux/bitmap.h> #include <linux/delay.h> @@ -505,8 +505,7 @@ static void cleanup_irq(struct spmi_pmic_arb *pmic_arb, u16 apid, int id) static void periph_interrupt(struct spmi_pmic_arb *pmic_arb, u16 apid) { unsigned int irq; - u32 status; - int id; + u32 status, id; u8 sid = (pmic_arb->apid_data[apid].ppid >> 8) & 0xF; u8 per = pmic_arb->apid_data[apid].ppid & 0xFF; diff --git a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h index 5a5121d958ed..8c65733e0255 100644 --- a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h +++ b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h @@ -22,7 +22,6 @@ #include <asm/processor.h> #include <linux/i2c.h> -#include <linux/sfi.h> #include <media/v4l2-subdev.h> #include "atomisp.h" diff --git a/drivers/staging/vme/devices/vme_user.c b/drivers/staging/vme/devices/vme_user.c index fd0ea4dbcb91..e3fa38bd7f12 100644 --- a/drivers/staging/vme/devices/vme_user.c +++ b/drivers/staging/vme/devices/vme_user.c @@ -175,7 +175,7 @@ static ssize_t buffer_from_user(unsigned int minor, const char __user *buf, static ssize_t vme_user_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) { - unsigned int minor = MINOR(file_inode(file)->i_rdev); + unsigned int minor = iminor(file_inode(file)); ssize_t retval; size_t image_size; @@ -218,7 +218,7 @@ static ssize_t vme_user_read(struct file *file, char __user *buf, size_t count, static ssize_t vme_user_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - unsigned int minor = MINOR(file_inode(file)->i_rdev); + unsigned int minor = iminor(file_inode(file)); ssize_t retval; size_t image_size; @@ -260,7 +260,7 @@ static ssize_t vme_user_write(struct file *file, const char __user *buf, static loff_t vme_user_llseek(struct file *file, loff_t off, int whence) { - unsigned int minor = MINOR(file_inode(file)->i_rdev); + unsigned int minor = iminor(file_inode(file)); size_t image_size; loff_t res; @@ -294,7 +294,7 @@ static int vme_user_ioctl(struct inode *inode, struct file *file, struct vme_slave slave; struct vme_irq_id irq_req; unsigned long copied; - unsigned int minor = MINOR(inode->i_rdev); + unsigned int minor = iminor(inode); int retval; dma_addr_t pci_addr; void __user *argp = (void __user *)arg; @@ -412,7 +412,7 @@ vme_user_unlocked_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { int ret; struct inode *inode = file_inode(file); - unsigned int minor = MINOR(inode->i_rdev); + unsigned int minor = iminor(inode); mutex_lock(&image[minor].mutex); ret = vme_user_ioctl(inode, file, cmd, arg); @@ -481,7 +481,7 @@ static int vme_user_master_mmap(unsigned int minor, struct vm_area_struct *vma) static int vme_user_mmap(struct file *file, struct vm_area_struct *vma) { - unsigned int minor = MINOR(file_inode(file)->i_rdev); + unsigned int minor = iminor(file_inode(file)); if (type[minor] == MASTER_MINOR) return vme_user_master_mmap(minor, vma); @@ -689,7 +689,7 @@ err_dev: return err; } -static int vme_user_remove(struct vme_dev *dev) +static void vme_user_remove(struct vme_dev *dev) { int i; @@ -717,8 +717,6 @@ static int vme_user_remove(struct vme_dev *dev) /* Unregister the major and minor device numbers */ unregister_chrdev_region(MKDEV(VME_MAJOR, 0), VME_DEVS); - - return 0; } static struct vme_driver vme_user_driver = { diff --git a/drivers/target/sbp/sbp_target.c b/drivers/target/sbp/sbp_target.c index e4a9b9fe3dfb..2a6165febd3b 100644 --- a/drivers/target/sbp/sbp_target.c +++ b/drivers/target/sbp/sbp_target.c @@ -1006,7 +1006,7 @@ static void tgt_agent_fetch_work(struct work_struct *work) agent->state = AGENT_STATE_SUSPENDED; spin_unlock_bh(&agent->lock); - }; + } } static struct sbp_target_agent *sbp_target_agent_register( diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c index 8ed93fd205c7..ee3d52061281 100644 --- a/drivers/target/target_core_iblock.c +++ b/drivers/target/target_core_iblock.c @@ -315,10 +315,8 @@ iblock_get_bio(struct se_cmd *cmd, sector_t lba, u32 sg_num, int op, * Only allocate as many vector entries as the bio code allows us to, * we'll loop later on until we have handled the whole request. */ - if (sg_num > BIO_MAX_PAGES) - sg_num = BIO_MAX_PAGES; - - bio = bio_alloc_bioset(GFP_NOIO, sg_num, &ib_dev->ibd_bio_set); + bio = bio_alloc_bioset(GFP_NOIO, bio_max_segs(sg_num), + &ib_dev->ibd_bio_set); if (!bio) { pr_err("Unable to allocate memory for bio\n"); return NULL; @@ -638,8 +636,7 @@ iblock_alloc_bip(struct se_cmd *cmd, struct bio *bio, return -ENODEV; } - bip = bio_integrity_alloc(bio, GFP_NOIO, - min_t(unsigned int, cmd->t_prot_nents, BIO_MAX_PAGES)); + bip = bio_integrity_alloc(bio, GFP_NOIO, bio_max_segs(cmd->t_prot_nents)); if (IS_ERR(bip)) { pr_err("Unable to allocate bio_integrity_payload\n"); return PTR_ERR(bip); diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c index 14db5e568f22..d4cc43afe05b 100644 --- a/drivers/target/target_core_pr.c +++ b/drivers/target/target_core_pr.c @@ -3739,6 +3739,7 @@ core_scsi3_pri_read_keys(struct se_cmd *cmd) spin_unlock(&dev->t10_pr.registration_lock); put_unaligned_be32(add_len, &buf[4]); + target_set_cmd_data_length(cmd, 8 + add_len); transport_kunmap_data_sg(cmd); @@ -3757,7 +3758,7 @@ core_scsi3_pri_read_reservation(struct se_cmd *cmd) struct t10_pr_registration *pr_reg; unsigned char *buf; u64 pr_res_key; - u32 add_len = 16; /* Hardcoded to 16 when a reservation is held. */ + u32 add_len = 0; if (cmd->data_length < 8) { pr_err("PRIN SA READ_RESERVATIONS SCSI Data Length: %u" @@ -3775,8 +3776,9 @@ core_scsi3_pri_read_reservation(struct se_cmd *cmd) pr_reg = dev->dev_pr_res_holder; if (pr_reg) { /* - * Set the hardcoded Additional Length + * Set the Additional Length to 16 when a reservation is held */ + add_len = 16; put_unaligned_be32(add_len, &buf[4]); if (cmd->data_length < 22) @@ -3812,6 +3814,8 @@ core_scsi3_pri_read_reservation(struct se_cmd *cmd) (pr_reg->pr_res_type & 0x0f); } + target_set_cmd_data_length(cmd, 8 + add_len); + err: spin_unlock(&dev->dev_reservation_lock); transport_kunmap_data_sg(cmd); @@ -3830,7 +3834,7 @@ core_scsi3_pri_report_capabilities(struct se_cmd *cmd) struct se_device *dev = cmd->se_dev; struct t10_reservation *pr_tmpl = &dev->t10_pr; unsigned char *buf; - u16 add_len = 8; /* Hardcoded to 8. */ + u16 len = 8; /* Hardcoded to 8. */ if (cmd->data_length < 6) { pr_err("PRIN SA REPORT_CAPABILITIES SCSI Data Length:" @@ -3842,7 +3846,7 @@ core_scsi3_pri_report_capabilities(struct se_cmd *cmd) if (!buf) return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; - put_unaligned_be16(add_len, &buf[0]); + put_unaligned_be16(len, &buf[0]); buf[2] |= 0x10; /* CRH: Compatible Reservation Hanlding bit. */ buf[2] |= 0x08; /* SIP_C: Specify Initiator Ports Capable bit */ buf[2] |= 0x04; /* ATP_C: All Target Ports Capable bit */ @@ -3871,6 +3875,8 @@ core_scsi3_pri_report_capabilities(struct se_cmd *cmd) buf[4] |= 0x02; /* PR_TYPE_WRITE_EXCLUSIVE */ buf[5] |= 0x01; /* PR_TYPE_EXCLUSIVE_ACCESS_ALLREG */ + target_set_cmd_data_length(cmd, len); + transport_kunmap_data_sg(cmd); return 0; @@ -4031,6 +4037,7 @@ core_scsi3_pri_read_full_status(struct se_cmd *cmd) * Set ADDITIONAL_LENGTH */ put_unaligned_be32(add_len, &buf[4]); + target_set_cmd_data_length(cmd, 8 + add_len); transport_kunmap_data_sg(cmd); diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c index 33770e5808ce..3cbc074992bc 100644 --- a/drivers/target/target_core_pscsi.c +++ b/drivers/target/target_core_pscsi.c @@ -881,7 +881,7 @@ pscsi_map_sg(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents, if (!bio) { new_bio: - nr_vecs = min_t(int, BIO_MAX_PAGES, nr_pages); + nr_vecs = bio_max_segs(nr_pages); nr_pages -= nr_vecs; /* * Calls bio_kmalloc() and sets bio->bi_end_io() diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 93ea17cbad79..5ecb9f18a53d 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -879,11 +879,9 @@ void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) } EXPORT_SYMBOL(target_complete_cmd); -void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length) +void target_set_cmd_data_length(struct se_cmd *cmd, int length) { - if ((scsi_status == SAM_STAT_GOOD || - cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) && - length < cmd->data_length) { + if (length < cmd->data_length) { if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) { cmd->residual_count += cmd->data_length - length; } else { @@ -893,6 +891,15 @@ void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int len cmd->data_length = length; } +} +EXPORT_SYMBOL(target_set_cmd_data_length); + +void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length) +{ + if (scsi_status == SAM_STAT_GOOD || + cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) { + target_set_cmd_data_length(cmd, length); + } target_complete_cmd(cmd, scsi_status); } diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c index a5991df23581..bf73cd5f4b04 100644 --- a/drivers/target/target_core_user.c +++ b/drivers/target/target_core_user.c @@ -1566,6 +1566,88 @@ static struct se_device *tcmu_alloc_device(struct se_hba *hba, const char *name) return &udev->se_dev; } +static void tcmu_dev_call_rcu(struct rcu_head *p) +{ + struct se_device *dev = container_of(p, struct se_device, rcu_head); + struct tcmu_dev *udev = TCMU_DEV(dev); + + kfree(udev->uio_info.name); + kfree(udev->name); + kfree(udev); +} + +static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd *cmd) +{ + if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) { + kmem_cache_free(tcmu_cmd_cache, cmd); + return 0; + } + return -EINVAL; +} + +static void tcmu_blocks_release(struct radix_tree_root *blocks, + int start, int end) +{ + int i; + struct page *page; + + for (i = start; i < end; i++) { + page = radix_tree_delete(blocks, i); + if (page) { + __free_page(page); + atomic_dec(&global_db_count); + } + } +} + +static void tcmu_remove_all_queued_tmr(struct tcmu_dev *udev) +{ + struct tcmu_tmr *tmr, *tmp; + + list_for_each_entry_safe(tmr, tmp, &udev->tmr_queue, queue_entry) { + list_del_init(&tmr->queue_entry); + kfree(tmr); + } +} + +static void tcmu_dev_kref_release(struct kref *kref) +{ + struct tcmu_dev *udev = container_of(kref, struct tcmu_dev, kref); + struct se_device *dev = &udev->se_dev; + struct tcmu_cmd *cmd; + bool all_expired = true; + int i; + + vfree(udev->mb_addr); + udev->mb_addr = NULL; + + spin_lock_bh(&timed_out_udevs_lock); + if (!list_empty(&udev->timedout_entry)) + list_del(&udev->timedout_entry); + spin_unlock_bh(&timed_out_udevs_lock); + + /* Upper layer should drain all requests before calling this */ + mutex_lock(&udev->cmdr_lock); + idr_for_each_entry(&udev->commands, cmd, i) { + if (tcmu_check_and_free_pending_cmd(cmd) != 0) + all_expired = false; + } + /* There can be left over TMR cmds. Remove them. */ + tcmu_remove_all_queued_tmr(udev); + if (!list_empty(&udev->qfull_queue)) + all_expired = false; + idr_destroy(&udev->commands); + WARN_ON(!all_expired); + + tcmu_blocks_release(&udev->data_blocks, 0, udev->dbi_max + 1); + bitmap_free(udev->data_bitmap); + mutex_unlock(&udev->cmdr_lock); + + pr_debug("dev_kref_release\n"); + + call_rcu(&dev->rcu_head, tcmu_dev_call_rcu); +} + static void run_qfull_queue(struct tcmu_dev *udev, bool fail) { struct tcmu_cmd *tcmu_cmd, *tmp_cmd; @@ -1678,6 +1760,25 @@ static struct page *tcmu_try_get_block_page(struct tcmu_dev *udev, uint32_t dbi) return page; } +static void tcmu_vma_open(struct vm_area_struct *vma) +{ + struct tcmu_dev *udev = vma->vm_private_data; + + pr_debug("vma_open\n"); + + kref_get(&udev->kref); +} + +static void tcmu_vma_close(struct vm_area_struct *vma) +{ + struct tcmu_dev *udev = vma->vm_private_data; + + pr_debug("vma_close\n"); + + /* release ref from tcmu_vma_open */ + kref_put(&udev->kref, tcmu_dev_kref_release); +} + static vm_fault_t tcmu_vma_fault(struct vm_fault *vmf) { struct tcmu_dev *udev = vmf->vma->vm_private_data; @@ -1716,6 +1817,8 @@ static vm_fault_t tcmu_vma_fault(struct vm_fault *vmf) } static const struct vm_operations_struct tcmu_vm_ops = { + .open = tcmu_vma_open, + .close = tcmu_vma_close, .fault = tcmu_vma_fault, }; @@ -1732,6 +1835,8 @@ static int tcmu_mmap(struct uio_info *info, struct vm_area_struct *vma) if (vma_pages(vma) != (udev->ring_size >> PAGE_SHIFT)) return -EINVAL; + tcmu_vma_open(vma); + return 0; } @@ -1744,93 +1849,12 @@ static int tcmu_open(struct uio_info *info, struct inode *inode) return -EBUSY; udev->inode = inode; - kref_get(&udev->kref); pr_debug("open\n"); return 0; } -static void tcmu_dev_call_rcu(struct rcu_head *p) -{ - struct se_device *dev = container_of(p, struct se_device, rcu_head); - struct tcmu_dev *udev = TCMU_DEV(dev); - - kfree(udev->uio_info.name); - kfree(udev->name); - kfree(udev); -} - -static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd *cmd) -{ - if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) { - kmem_cache_free(tcmu_cmd_cache, cmd); - return 0; - } - return -EINVAL; -} - -static void tcmu_blocks_release(struct radix_tree_root *blocks, - int start, int end) -{ - int i; - struct page *page; - - for (i = start; i < end; i++) { - page = radix_tree_delete(blocks, i); - if (page) { - __free_page(page); - atomic_dec(&global_db_count); - } - } -} - -static void tcmu_remove_all_queued_tmr(struct tcmu_dev *udev) -{ - struct tcmu_tmr *tmr, *tmp; - - list_for_each_entry_safe(tmr, tmp, &udev->tmr_queue, queue_entry) { - list_del_init(&tmr->queue_entry); - kfree(tmr); - } -} - -static void tcmu_dev_kref_release(struct kref *kref) -{ - struct tcmu_dev *udev = container_of(kref, struct tcmu_dev, kref); - struct se_device *dev = &udev->se_dev; - struct tcmu_cmd *cmd; - bool all_expired = true; - int i; - - vfree(udev->mb_addr); - udev->mb_addr = NULL; - - spin_lock_bh(&timed_out_udevs_lock); - if (!list_empty(&udev->timedout_entry)) - list_del(&udev->timedout_entry); - spin_unlock_bh(&timed_out_udevs_lock); - - /* Upper layer should drain all requests before calling this */ - mutex_lock(&udev->cmdr_lock); - idr_for_each_entry(&udev->commands, cmd, i) { - if (tcmu_check_and_free_pending_cmd(cmd) != 0) - all_expired = false; - } - /* There can be left over TMR cmds. Remove them. */ - tcmu_remove_all_queued_tmr(udev); - if (!list_empty(&udev->qfull_queue)) - all_expired = false; - idr_destroy(&udev->commands); - WARN_ON(!all_expired); - - tcmu_blocks_release(&udev->data_blocks, 0, udev->dbi_max + 1); - bitmap_free(udev->data_bitmap); - mutex_unlock(&udev->cmdr_lock); - - call_rcu(&dev->rcu_head, tcmu_dev_call_rcu); -} - static int tcmu_release(struct uio_info *info, struct inode *inode) { struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info); @@ -1838,8 +1862,7 @@ static int tcmu_release(struct uio_info *info, struct inode *inode) clear_bit(TCMU_DEV_BIT_OPEN, &udev->flags); pr_debug("close\n"); - /* release ref from open */ - kref_put(&udev->kref, tcmu_dev_kref_release); + return 0; } diff --git a/drivers/tty/Makefile b/drivers/tty/Makefile index b3ccae932660..730de6bf048b 100644 --- a/drivers/tty/Makefile +++ b/drivers/tty/Makefile @@ -9,8 +9,6 @@ obj-$(CONFIG_AUDIT) += tty_audit.o obj-$(CONFIG_MAGIC_SYSRQ) += sysrq.o obj-$(CONFIG_N_HDLC) += n_hdlc.o obj-$(CONFIG_N_GSM) += n_gsm.o -obj-$(CONFIG_TRACE_ROUTER) += n_tracerouter.o -obj-$(CONFIG_TRACE_SINK) += n_tracesink.o obj-$(CONFIG_R3964) += n_r3964.o obj-y += vt/ diff --git a/drivers/tty/n_tracerouter.c b/drivers/tty/n_tracerouter.c deleted file mode 100644 index 3490ed51b1a3..000000000000 --- a/drivers/tty/n_tracerouter.c +++ /dev/null @@ -1,235 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * n_tracerouter.c - Trace data router through tty space - * - * Copyright (C) Intel 2011 - * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * This trace router uses the Linux line discipline framework to route - * trace data coming from a HW Modem to a PTI (Parallel Trace Module) port. - * The solution is not specific to a HW modem and this line disciple can - * be used to route any stream of data in kernel space. - * This is part of a solution for the P1149.7, compact JTAG, standard. - */ - -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/types.h> -#include <linux/ioctl.h> -#include <linux/tty.h> -#include <linux/tty_ldisc.h> -#include <linux/errno.h> -#include <linux/string.h> -#include <linux/mutex.h> -#include <linux/slab.h> -#include <linux/bug.h> -#include "n_tracesink.h" - -/* - * Other ldisc drivers use 65536 which basically means, - * 'I can always accept 64k' and flow control is off. - * This number is deemed appropriate for this driver. - */ -#define RECEIVE_ROOM 65536 -#define DRIVERNAME "n_tracerouter" - -/* - * struct to hold private configuration data for this ldisc. - * opencalled is used to hold if this ldisc has been opened. - * kref_tty holds the tty reference the ldisc sits on top of. - */ -struct tracerouter_data { - u8 opencalled; - struct tty_struct *kref_tty; -}; -static struct tracerouter_data *tr_data; - -/* lock for when tty reference is being used */ -static DEFINE_MUTEX(routelock); - -/** - * n_tracerouter_open() - Called when a tty is opened by a SW entity. - * @tty: terminal device to the ldisc. - * - * Return: - * 0 for success. - * - * Caveats: This should only be opened one time per SW entity. - */ -static int n_tracerouter_open(struct tty_struct *tty) -{ - int retval = -EEXIST; - - mutex_lock(&routelock); - if (tr_data->opencalled == 0) { - - tr_data->kref_tty = tty_kref_get(tty); - if (tr_data->kref_tty == NULL) { - retval = -EFAULT; - } else { - tr_data->opencalled = 1; - tty->disc_data = tr_data; - tty->receive_room = RECEIVE_ROOM; - tty_driver_flush_buffer(tty); - retval = 0; - } - } - mutex_unlock(&routelock); - return retval; -} - -/** - * n_tracerouter_close() - close connection - * @tty: terminal device to the ldisc. - * - * Called when a software entity wants to close a connection. - */ -static void n_tracerouter_close(struct tty_struct *tty) -{ - struct tracerouter_data *tptr = tty->disc_data; - - mutex_lock(&routelock); - WARN_ON(tptr->kref_tty != tr_data->kref_tty); - tty_driver_flush_buffer(tty); - tty_kref_put(tr_data->kref_tty); - tr_data->kref_tty = NULL; - tr_data->opencalled = 0; - tty->disc_data = NULL; - mutex_unlock(&routelock); -} - -/** - * n_tracerouter_read() - read request from user space - * @tty: terminal device passed into the ldisc. - * @file: pointer to open file object. - * @buf: pointer to the data buffer that gets eventually returned. - * @nr: number of bytes of the data buffer that is returned. - * - * function that allows read() functionality in userspace. By default if this - * is not implemented it returns -EIO. This module is functioning like a - * router via n_tracerouter_receivebuf(), and there is no real requirement - * to implement this function. However, an error return value other than - * -EIO should be used just to show that there was an intent not to have - * this function implemented. Return value based on read() man pages. - * - * Return: - * -EINVAL - */ -static ssize_t n_tracerouter_read(struct tty_struct *tty, struct file *file, - unsigned char *buf, size_t nr, - void **cookie, unsigned long offset) -{ - return -EINVAL; -} - -/** - * n_tracerouter_write() - Function that allows write() in userspace. - * @tty: terminal device passed into the ldisc. - * @file: pointer to open file object. - * @buf: pointer to the data buffer that gets eventually returned. - * @nr: number of bytes of the data buffer that is returned. - * - * By default if this is not implemented, it returns -EIO. - * This should not be implemented, ever, because - * 1. this driver is functioning like a router via - * n_tracerouter_receivebuf() - * 2. No writes to HW will ever go through this line discpline driver. - * However, an error return value other than -EIO should be used - * just to show that there was an intent not to have this function - * implemented. Return value based on write() man pages. - * - * Return: - * -EINVAL - */ -static ssize_t n_tracerouter_write(struct tty_struct *tty, struct file *file, - const unsigned char *buf, size_t nr) { - return -EINVAL; -} - -/** - * n_tracerouter_receivebuf() - Routing function for driver. - * @tty: terminal device passed into the ldisc. It's assumed - * tty will never be NULL. - * @cp: buffer, block of characters to be eventually read by - * someone, somewhere (user read() call or some kernel function). - * @fp: flag buffer. - * @count: number of characters (aka, bytes) in cp. - * - * This function takes the input buffer, cp, and passes it to - * an external API function for processing. - */ -static void n_tracerouter_receivebuf(struct tty_struct *tty, - const unsigned char *cp, - char *fp, int count) -{ - mutex_lock(&routelock); - n_tracesink_datadrain((u8 *) cp, count); - mutex_unlock(&routelock); -} - -/* - * Flush buffer is not impelemented as the ldisc has no internal buffering - * so the tty_driver_flush_buffer() is sufficient for this driver's needs. - */ - -static struct tty_ldisc_ops tty_ptirouter_ldisc = { - .owner = THIS_MODULE, - .magic = TTY_LDISC_MAGIC, - .name = DRIVERNAME, - .open = n_tracerouter_open, - .close = n_tracerouter_close, - .read = n_tracerouter_read, - .write = n_tracerouter_write, - .receive_buf = n_tracerouter_receivebuf -}; - -/** - * n_tracerouter_init - module initialisation - * - * Registers this module as a line discipline driver. - * - * Return: - * 0 for success, any other value error. - */ -static int __init n_tracerouter_init(void) -{ - int retval; - - tr_data = kzalloc(sizeof(struct tracerouter_data), GFP_KERNEL); - if (tr_data == NULL) - return -ENOMEM; - - - /* Note N_TRACEROUTER is defined in linux/tty.h */ - retval = tty_register_ldisc(N_TRACEROUTER, &tty_ptirouter_ldisc); - if (retval < 0) { - pr_err("%s: Registration failed: %d\n", __func__, retval); - kfree(tr_data); - } - return retval; -} - -/** - * n_tracerouter_exit - module unload - * - * Removes this module as a line discipline driver. - */ -static void __exit n_tracerouter_exit(void) -{ - int retval = tty_unregister_ldisc(N_TRACEROUTER); - - if (retval < 0) - pr_err("%s: Unregistration failed: %d\n", __func__, retval); - else - kfree(tr_data); -} - -module_init(n_tracerouter_init); -module_exit(n_tracerouter_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Jay Freyensee"); -MODULE_ALIAS_LDISC(N_TRACEROUTER); -MODULE_DESCRIPTION("Trace router ldisc driver"); diff --git a/drivers/tty/n_tracesink.c b/drivers/tty/n_tracesink.c deleted file mode 100644 index 1d9931041fd8..000000000000 --- a/drivers/tty/n_tracesink.c +++ /dev/null @@ -1,230 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * n_tracesink.c - Trace data router and sink path through tty space. - * - * Copyright (C) Intel 2011 - * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * The trace sink uses the Linux line discipline framework to receive - * trace data coming from the PTI source line discipline driver - * to a user-desired tty port, like USB. - * This is to provide a way to extract modem trace data on - * devices that do not have a PTI HW module, or just need modem - * trace data to come out of a different HW output port. - * This is part of a solution for the P1149.7, compact JTAG, standard. - */ - -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/types.h> -#include <linux/ioctl.h> -#include <linux/tty.h> -#include <linux/tty_ldisc.h> -#include <linux/errno.h> -#include <linux/string.h> -#include <linux/bug.h> -#include "n_tracesink.h" - -/* - * Other ldisc drivers use 65536 which basically means, - * 'I can always accept 64k' and flow control is off. - * This number is deemed appropriate for this driver. - */ -#define RECEIVE_ROOM 65536 -#define DRIVERNAME "n_tracesink" - -/* - * there is a quirk with this ldisc is he can write data - * to a tty from anyone calling his kernel API, which - * meets customer requirements in the drivers/misc/pti.c - * project. So he needs to know when he can and cannot write when - * the API is called. In theory, the API can be called - * after an init() but before a successful open() which - * would crash the system if tty is not checked. - */ -static struct tty_struct *this_tty; -static DEFINE_MUTEX(writelock); - -/** - * n_tracesink_open() - Called when a tty is opened by a SW entity. - * @tty: terminal device to the ldisc. - * - * Return: - * 0 for success, - * -EFAULT = couldn't get a tty kref n_tracesink will sit - * on top of - * -EEXIST = open() called successfully once and it cannot - * be called again. - * - * Caveats: open() should only be successful the first time a - * SW entity calls it. - */ -static int n_tracesink_open(struct tty_struct *tty) -{ - int retval = -EEXIST; - - mutex_lock(&writelock); - if (this_tty == NULL) { - this_tty = tty_kref_get(tty); - if (this_tty == NULL) { - retval = -EFAULT; - } else { - tty->disc_data = this_tty; - tty_driver_flush_buffer(tty); - retval = 0; - } - } - mutex_unlock(&writelock); - - return retval; -} - -/** - * n_tracesink_close() - close connection - * @tty: terminal device to the ldisc. - * - * Called when a software entity wants to close a connection. - */ -static void n_tracesink_close(struct tty_struct *tty) -{ - mutex_lock(&writelock); - tty_driver_flush_buffer(tty); - tty_kref_put(this_tty); - this_tty = NULL; - tty->disc_data = NULL; - mutex_unlock(&writelock); -} - -/** - * n_tracesink_read() - read request from user space - * @tty: terminal device passed into the ldisc. - * @file: pointer to open file object. - * @buf: pointer to the data buffer that gets eventually returned. - * @nr: number of bytes of the data buffer that is returned. - * - * function that allows read() functionality in userspace. By default if this - * is not implemented it returns -EIO. This module is functioning like a - * router via n_tracesink_receivebuf(), and there is no real requirement - * to implement this function. However, an error return value other than - * -EIO should be used just to show that there was an intent not to have - * this function implemented. Return value based on read() man pages. - * - * Return: - * -EINVAL - */ -static ssize_t n_tracesink_read(struct tty_struct *tty, struct file *file, - unsigned char *buf, size_t nr, - void **cookie, unsigned long offset) -{ - return -EINVAL; -} - -/** - * n_tracesink_write() - Function that allows write() in userspace. - * @tty: terminal device passed into the ldisc. - * @file: pointer to open file object. - * @buf: pointer to the data buffer that gets eventually returned. - * @nr: number of bytes of the data buffer that is returned. - * - * By default if this is not implemented, it returns -EIO. - * This should not be implemented, ever, because - * 1. this driver is functioning like a router via - * n_tracesink_receivebuf() - * 2. No writes to HW will ever go through this line discpline driver. - * However, an error return value other than -EIO should be used - * just to show that there was an intent not to have this function - * implemented. Return value based on write() man pages. - * - * Return: - * -EINVAL - */ -static ssize_t n_tracesink_write(struct tty_struct *tty, struct file *file, - const unsigned char *buf, size_t nr) { - return -EINVAL; -} - -/** - * n_tracesink_datadrain() - Kernel API function used to route - * trace debugging data to user-defined - * port like USB. - * - * @buf: Trace debuging data buffer to write to tty target - * port. Null value will return with no write occurring. - * @count: Size of buf. Value of 0 or a negative number will - * return with no write occuring. - * - * Caveat: If this line discipline does not set the tty it sits - * on top of via an open() call, this API function will not - * call the tty's write() call because it will have no pointer - * to call the write(). - */ -void n_tracesink_datadrain(u8 *buf, int count) -{ - mutex_lock(&writelock); - - if ((buf != NULL) && (count > 0) && (this_tty != NULL)) - this_tty->ops->write(this_tty, buf, count); - - mutex_unlock(&writelock); -} -EXPORT_SYMBOL_GPL(n_tracesink_datadrain); - -/* - * Flush buffer is not impelemented as the ldisc has no internal buffering - * so the tty_driver_flush_buffer() is sufficient for this driver's needs. - */ - -/* - * tty_ldisc function operations for this driver. - */ -static struct tty_ldisc_ops tty_n_tracesink = { - .owner = THIS_MODULE, - .magic = TTY_LDISC_MAGIC, - .name = DRIVERNAME, - .open = n_tracesink_open, - .close = n_tracesink_close, - .read = n_tracesink_read, - .write = n_tracesink_write -}; - -/** - * n_tracesink_init- module initialisation - * - * Registers this module as a line discipline driver. - * - * Return: - * 0 for success, any other value error. - */ -static int __init n_tracesink_init(void) -{ - /* Note N_TRACESINK is defined in linux/tty.h */ - int retval = tty_register_ldisc(N_TRACESINK, &tty_n_tracesink); - - if (retval < 0) - pr_err("%s: Registration failed: %d\n", __func__, retval); - - return retval; -} - -/** - * n_tracesink_exit - module unload - * - * Removes this module as a line discipline driver. - */ -static void __exit n_tracesink_exit(void) -{ - int retval = tty_unregister_ldisc(N_TRACESINK); - - if (retval < 0) - pr_err("%s: Unregistration failed: %d\n", __func__, retval); -} - -module_init(n_tracesink_init); -module_exit(n_tracesink_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Jay Freyensee"); -MODULE_ALIAS_LDISC(N_TRACESINK); -MODULE_DESCRIPTION("Trace sink ldisc driver"); diff --git a/drivers/tty/n_tracesink.h b/drivers/tty/n_tracesink.h deleted file mode 100644 index 7031d515a700..000000000000 --- a/drivers/tty/n_tracesink.h +++ /dev/null @@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * n_tracesink.h - Kernel driver API to route trace data in kernel space. - * - * Copyright (C) Intel 2011 - * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * The PTI (Parallel Trace Interface) driver directs trace data routed from - * various parts in the system out through the Intel Penwell PTI port and - * out of the mobile device for analysis with a debugging tool - * (Lauterbach, Fido). This is part of a solution for the MIPI P1149.7, - * compact JTAG, standard. - * - * This header file is used by n_tracerouter to be able to send the - * data of it's tty port to the tty port this module sits. This - * mechanism can also be used independent of the PTI module. - * - */ - -#ifndef N_TRACESINK_H_ -#define N_TRACESINK_H_ - -void n_tracesink_datadrain(u8 *buf, int count); - -#endif diff --git a/drivers/uio/uio_pci_generic.c b/drivers/uio/uio_pci_generic.c index b8e44d16279f..c7d681fef198 100644 --- a/drivers/uio/uio_pci_generic.c +++ b/drivers/uio/uio_pci_generic.c @@ -92,7 +92,7 @@ static int probe(struct pci_dev *pdev, gdev->info.version = DRIVER_VERSION; gdev->info.release = release; gdev->pdev = pdev; - if (pdev->irq) { + if (pdev->irq && (pdev->irq != IRQ_NOTCONNECTED)) { gdev->info.irq = pdev->irq; gdev->info.irq_flags = IRQF_SHARED; gdev->info.handler = irqhandler; diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index ad5a0f405a75..3f0381344221 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -111,8 +111,8 @@ DECLARE_WAIT_QUEUE_HEAD(usb_kill_urb_queue); */ /*-------------------------------------------------------------------------*/ -#define KERNEL_REL bin2bcd(((LINUX_VERSION_CODE >> 16) & 0x0ff)) -#define KERNEL_VER bin2bcd(((LINUX_VERSION_CODE >> 8) & 0x0ff)) +#define KERNEL_REL bin2bcd(LINUX_VERSION_MAJOR) +#define KERNEL_VER bin2bcd(LINUX_VERSION_PATCHLEVEL) /* usb 3.1 root hub device descriptor */ static const u8 usb31_rh_dev_descriptor[18] = { diff --git a/drivers/usb/gadget/udc/aspeed-vhub/hub.c b/drivers/usb/gadget/udc/aspeed-vhub/hub.c index bfd8e77788e2..5c7dea5e0ff1 100644 --- a/drivers/usb/gadget/udc/aspeed-vhub/hub.c +++ b/drivers/usb/gadget/udc/aspeed-vhub/hub.c @@ -46,8 +46,8 @@ * - Make vid/did overridable * - make it look like usb1 if usb1 mode forced */ -#define KERNEL_REL bin2bcd(((LINUX_VERSION_CODE >> 16) & 0x0ff)) -#define KERNEL_VER bin2bcd(((LINUX_VERSION_CODE >> 8) & 0x0ff)) +#define KERNEL_REL bin2bcd(LINUX_VERSION_MAJOR) +#define KERNEL_VER bin2bcd(LINUX_VERSION_PATCHLEVEL) enum { AST_VHUB_STR_INDEX_MAX = 4, diff --git a/drivers/vdpa/Kconfig b/drivers/vdpa/Kconfig index 92a6396f8a73..ffd1e098bfd2 100644 --- a/drivers/vdpa/Kconfig +++ b/drivers/vdpa/Kconfig @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only menuconfig VDPA tristate "vDPA drivers" + depends on NET help Enable this module to support vDPA device that uses a datapath which complies with virtio specifications with diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index fa1af301cf55..7c8bbfcf6c3e 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -432,7 +432,7 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id) adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa, dev, &ifc_vdpa_ops, - IFCVF_MAX_QUEUE_PAIRS * 2); + IFCVF_MAX_QUEUE_PAIRS * 2, NULL); if (adapter == NULL) { IFCVF_ERR(pdev, "Failed to allocate vDPA structure"); return -ENOMEM; diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index b5fe6d2ad22f..10e9b09932eb 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -1820,7 +1820,7 @@ static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); - if (offset + len < sizeof(struct virtio_net_config)) + if (offset + len <= sizeof(struct virtio_net_config)) memcpy(buf, (u8 *)&ndev->config + offset, len); } @@ -1982,7 +1982,7 @@ static int mlx5v_probe(struct auxiliary_device *adev, max_vqs = min_t(u32, max_vqs, MLX5_MAX_SUPPORTED_VQS); ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops, - 2 * mlx5_vdpa_max_qps(max_vqs)); + 2 * mlx5_vdpa_max_qps(max_vqs), NULL); if (IS_ERR(ndev)) return PTR_ERR(ndev); diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index c0825650c055..da67f07e24fd 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -11,9 +11,17 @@ #include <linux/idr.h> #include <linux/slab.h> #include <linux/vdpa.h> +#include <uapi/linux/vdpa.h> +#include <net/genetlink.h> +#include <linux/mod_devicetable.h> +static LIST_HEAD(mdev_head); +/* A global mutex that protects vdpa management device and device level operations. */ +static DEFINE_MUTEX(vdpa_dev_mutex); static DEFINE_IDA(vdpa_index_ida); +static struct genl_family vdpa_nl_family; + static int vdpa_dev_probe(struct device *d) { struct vdpa_device *vdev = dev_to_vdpa(d); @@ -63,6 +71,7 @@ static void vdpa_release_dev(struct device *d) * @config: the bus operations that is supported by this device * @nvqs: number of virtqueues supported by this device * @size: size of the parent structure that contains private data + * @name: name of the vdpa device; optional. * * Driver should use vdpa_alloc_device() wrapper macro instead of * using this directly. @@ -72,8 +81,7 @@ static void vdpa_release_dev(struct device *d) */ struct vdpa_device *__vdpa_alloc_device(struct device *parent, const struct vdpa_config_ops *config, - int nvqs, - size_t size) + int nvqs, size_t size, const char *name) { struct vdpa_device *vdev; int err = -EINVAL; @@ -101,7 +109,10 @@ struct vdpa_device *__vdpa_alloc_device(struct device *parent, vdev->features_valid = false; vdev->nvqs = nvqs; - err = dev_set_name(&vdev->dev, "vdpa%u", vdev->index); + if (name) + err = dev_set_name(&vdev->dev, "%s", name); + else + err = dev_set_name(&vdev->dev, "vdpa%u", vdev->index); if (err) goto err_name; @@ -118,6 +129,44 @@ err: } EXPORT_SYMBOL_GPL(__vdpa_alloc_device); +static int vdpa_name_match(struct device *dev, const void *data) +{ + struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev); + + return (strcmp(dev_name(&vdev->dev), data) == 0); +} + +static int __vdpa_register_device(struct vdpa_device *vdev) +{ + struct device *dev; + + lockdep_assert_held(&vdpa_dev_mutex); + dev = bus_find_device(&vdpa_bus, NULL, dev_name(&vdev->dev), vdpa_name_match); + if (dev) { + put_device(dev); + return -EEXIST; + } + return device_add(&vdev->dev); +} + +/** + * _vdpa_register_device - register a vDPA device with vdpa lock held + * Caller must have a succeed call of vdpa_alloc_device() before. + * Caller must invoke this routine in the management device dev_add() + * callback after setting up valid mgmtdev for this vdpa device. + * @vdev: the vdpa device to be registered to vDPA bus + * + * Returns an error when fail to add device to vDPA bus + */ +int _vdpa_register_device(struct vdpa_device *vdev) +{ + if (!vdev->mdev) + return -EINVAL; + + return __vdpa_register_device(vdev); +} +EXPORT_SYMBOL_GPL(_vdpa_register_device); + /** * vdpa_register_device - register a vDPA device * Callers must have a succeed call of vdpa_alloc_device() before. @@ -127,17 +176,38 @@ EXPORT_SYMBOL_GPL(__vdpa_alloc_device); */ int vdpa_register_device(struct vdpa_device *vdev) { - return device_add(&vdev->dev); + int err; + + mutex_lock(&vdpa_dev_mutex); + err = __vdpa_register_device(vdev); + mutex_unlock(&vdpa_dev_mutex); + return err; } EXPORT_SYMBOL_GPL(vdpa_register_device); /** + * _vdpa_unregister_device - unregister a vDPA device + * Caller must invoke this routine as part of management device dev_del() + * callback. + * @vdev: the vdpa device to be unregisted from vDPA bus + */ +void _vdpa_unregister_device(struct vdpa_device *vdev) +{ + lockdep_assert_held(&vdpa_dev_mutex); + WARN_ON(!vdev->mdev); + device_unregister(&vdev->dev); +} +EXPORT_SYMBOL_GPL(_vdpa_unregister_device); + +/** * vdpa_unregister_device - unregister a vDPA device * @vdev: the vdpa device to be unregisted from vDPA bus */ void vdpa_unregister_device(struct vdpa_device *vdev) { + mutex_lock(&vdpa_dev_mutex); device_unregister(&vdev->dev); + mutex_unlock(&vdpa_dev_mutex); } EXPORT_SYMBOL_GPL(vdpa_unregister_device); @@ -167,13 +237,436 @@ void vdpa_unregister_driver(struct vdpa_driver *drv) } EXPORT_SYMBOL_GPL(vdpa_unregister_driver); +/** + * vdpa_mgmtdev_register - register a vdpa management device + * + * @mdev: Pointer to vdpa management device + * vdpa_mgmtdev_register() register a vdpa management device which supports + * vdpa device management. + */ +int vdpa_mgmtdev_register(struct vdpa_mgmt_dev *mdev) +{ + if (!mdev->device || !mdev->ops || !mdev->ops->dev_add || !mdev->ops->dev_del) + return -EINVAL; + + INIT_LIST_HEAD(&mdev->list); + mutex_lock(&vdpa_dev_mutex); + list_add_tail(&mdev->list, &mdev_head); + mutex_unlock(&vdpa_dev_mutex); + return 0; +} +EXPORT_SYMBOL_GPL(vdpa_mgmtdev_register); + +static int vdpa_match_remove(struct device *dev, void *data) +{ + struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev); + struct vdpa_mgmt_dev *mdev = vdev->mdev; + + if (mdev == data) + mdev->ops->dev_del(mdev, vdev); + return 0; +} + +void vdpa_mgmtdev_unregister(struct vdpa_mgmt_dev *mdev) +{ + mutex_lock(&vdpa_dev_mutex); + + list_del(&mdev->list); + + /* Filter out all the entries belong to this management device and delete it. */ + bus_for_each_dev(&vdpa_bus, NULL, mdev, vdpa_match_remove); + + mutex_unlock(&vdpa_dev_mutex); +} +EXPORT_SYMBOL_GPL(vdpa_mgmtdev_unregister); + +static bool mgmtdev_handle_match(const struct vdpa_mgmt_dev *mdev, + const char *busname, const char *devname) +{ + /* Bus name is optional for simulated management device, so ignore the + * device with bus if bus attribute is provided. + */ + if ((busname && !mdev->device->bus) || (!busname && mdev->device->bus)) + return false; + + if (!busname && strcmp(dev_name(mdev->device), devname) == 0) + return true; + + if (busname && (strcmp(mdev->device->bus->name, busname) == 0) && + (strcmp(dev_name(mdev->device), devname) == 0)) + return true; + + return false; +} + +static struct vdpa_mgmt_dev *vdpa_mgmtdev_get_from_attr(struct nlattr **attrs) +{ + struct vdpa_mgmt_dev *mdev; + const char *busname = NULL; + const char *devname; + + if (!attrs[VDPA_ATTR_MGMTDEV_DEV_NAME]) + return ERR_PTR(-EINVAL); + devname = nla_data(attrs[VDPA_ATTR_MGMTDEV_DEV_NAME]); + if (attrs[VDPA_ATTR_MGMTDEV_BUS_NAME]) + busname = nla_data(attrs[VDPA_ATTR_MGMTDEV_BUS_NAME]); + + list_for_each_entry(mdev, &mdev_head, list) { + if (mgmtdev_handle_match(mdev, busname, devname)) + return mdev; + } + return ERR_PTR(-ENODEV); +} + +static int vdpa_nl_mgmtdev_handle_fill(struct sk_buff *msg, const struct vdpa_mgmt_dev *mdev) +{ + if (mdev->device->bus && + nla_put_string(msg, VDPA_ATTR_MGMTDEV_BUS_NAME, mdev->device->bus->name)) + return -EMSGSIZE; + if (nla_put_string(msg, VDPA_ATTR_MGMTDEV_DEV_NAME, dev_name(mdev->device))) + return -EMSGSIZE; + return 0; +} + +static int vdpa_mgmtdev_fill(const struct vdpa_mgmt_dev *mdev, struct sk_buff *msg, + u32 portid, u32 seq, int flags) +{ + u64 supported_classes = 0; + void *hdr; + int i = 0; + int err; + + hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags, VDPA_CMD_MGMTDEV_NEW); + if (!hdr) + return -EMSGSIZE; + err = vdpa_nl_mgmtdev_handle_fill(msg, mdev); + if (err) + goto msg_err; + + while (mdev->id_table[i].device) { + supported_classes |= BIT(mdev->id_table[i].device); + i++; + } + + if (nla_put_u64_64bit(msg, VDPA_ATTR_MGMTDEV_SUPPORTED_CLASSES, + supported_classes, VDPA_ATTR_UNSPEC)) { + err = -EMSGSIZE; + goto msg_err; + } + + genlmsg_end(msg, hdr); + return 0; + +msg_err: + genlmsg_cancel(msg, hdr); + return err; +} + +static int vdpa_nl_cmd_mgmtdev_get_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct vdpa_mgmt_dev *mdev; + struct sk_buff *msg; + int err; + + msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!msg) + return -ENOMEM; + + mutex_lock(&vdpa_dev_mutex); + mdev = vdpa_mgmtdev_get_from_attr(info->attrs); + if (IS_ERR(mdev)) { + mutex_unlock(&vdpa_dev_mutex); + NL_SET_ERR_MSG_MOD(info->extack, "Fail to find the specified mgmt device"); + err = PTR_ERR(mdev); + goto out; + } + + err = vdpa_mgmtdev_fill(mdev, msg, info->snd_portid, info->snd_seq, 0); + mutex_unlock(&vdpa_dev_mutex); + if (err) + goto out; + err = genlmsg_reply(msg, info); + return err; + +out: + nlmsg_free(msg); + return err; +} + +static int +vdpa_nl_cmd_mgmtdev_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb) +{ + struct vdpa_mgmt_dev *mdev; + int start = cb->args[0]; + int idx = 0; + int err; + + mutex_lock(&vdpa_dev_mutex); + list_for_each_entry(mdev, &mdev_head, list) { + if (idx < start) { + idx++; + continue; + } + err = vdpa_mgmtdev_fill(mdev, msg, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, NLM_F_MULTI); + if (err) + goto out; + idx++; + } +out: + mutex_unlock(&vdpa_dev_mutex); + cb->args[0] = idx; + return msg->len; +} + +static int vdpa_nl_cmd_dev_add_set_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct vdpa_mgmt_dev *mdev; + const char *name; + int err = 0; + + if (!info->attrs[VDPA_ATTR_DEV_NAME]) + return -EINVAL; + + name = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]); + + mutex_lock(&vdpa_dev_mutex); + mdev = vdpa_mgmtdev_get_from_attr(info->attrs); + if (IS_ERR(mdev)) { + NL_SET_ERR_MSG_MOD(info->extack, "Fail to find the specified management device"); + err = PTR_ERR(mdev); + goto err; + } + + err = mdev->ops->dev_add(mdev, name); +err: + mutex_unlock(&vdpa_dev_mutex); + return err; +} + +static int vdpa_nl_cmd_dev_del_set_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct vdpa_mgmt_dev *mdev; + struct vdpa_device *vdev; + struct device *dev; + const char *name; + int err = 0; + + if (!info->attrs[VDPA_ATTR_DEV_NAME]) + return -EINVAL; + name = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]); + + mutex_lock(&vdpa_dev_mutex); + dev = bus_find_device(&vdpa_bus, NULL, name, vdpa_name_match); + if (!dev) { + NL_SET_ERR_MSG_MOD(info->extack, "device not found"); + err = -ENODEV; + goto dev_err; + } + vdev = container_of(dev, struct vdpa_device, dev); + if (!vdev->mdev) { + NL_SET_ERR_MSG_MOD(info->extack, "Only user created device can be deleted by user"); + err = -EINVAL; + goto mdev_err; + } + mdev = vdev->mdev; + mdev->ops->dev_del(mdev, vdev); +mdev_err: + put_device(dev); +dev_err: + mutex_unlock(&vdpa_dev_mutex); + return err; +} + +static int +vdpa_dev_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq, + int flags, struct netlink_ext_ack *extack) +{ + u16 max_vq_size; + u32 device_id; + u32 vendor_id; + void *hdr; + int err; + + hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags, VDPA_CMD_DEV_NEW); + if (!hdr) + return -EMSGSIZE; + + err = vdpa_nl_mgmtdev_handle_fill(msg, vdev->mdev); + if (err) + goto msg_err; + + device_id = vdev->config->get_device_id(vdev); + vendor_id = vdev->config->get_vendor_id(vdev); + max_vq_size = vdev->config->get_vq_num_max(vdev); + + err = -EMSGSIZE; + if (nla_put_string(msg, VDPA_ATTR_DEV_NAME, dev_name(&vdev->dev))) + goto msg_err; + if (nla_put_u32(msg, VDPA_ATTR_DEV_ID, device_id)) + goto msg_err; + if (nla_put_u32(msg, VDPA_ATTR_DEV_VENDOR_ID, vendor_id)) + goto msg_err; + if (nla_put_u32(msg, VDPA_ATTR_DEV_MAX_VQS, vdev->nvqs)) + goto msg_err; + if (nla_put_u16(msg, VDPA_ATTR_DEV_MAX_VQ_SIZE, max_vq_size)) + goto msg_err; + + genlmsg_end(msg, hdr); + return 0; + +msg_err: + genlmsg_cancel(msg, hdr); + return err; +} + +static int vdpa_nl_cmd_dev_get_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct vdpa_device *vdev; + struct sk_buff *msg; + const char *devname; + struct device *dev; + int err; + + if (!info->attrs[VDPA_ATTR_DEV_NAME]) + return -EINVAL; + devname = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]); + msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!msg) + return -ENOMEM; + + mutex_lock(&vdpa_dev_mutex); + dev = bus_find_device(&vdpa_bus, NULL, devname, vdpa_name_match); + if (!dev) { + NL_SET_ERR_MSG_MOD(info->extack, "device not found"); + err = -ENODEV; + goto err; + } + vdev = container_of(dev, struct vdpa_device, dev); + if (!vdev->mdev) { + err = -EINVAL; + goto mdev_err; + } + err = vdpa_dev_fill(vdev, msg, info->snd_portid, info->snd_seq, 0, info->extack); + if (!err) + err = genlmsg_reply(msg, info); +mdev_err: + put_device(dev); +err: + mutex_unlock(&vdpa_dev_mutex); + if (err) + nlmsg_free(msg); + return err; +} + +struct vdpa_dev_dump_info { + struct sk_buff *msg; + struct netlink_callback *cb; + int start_idx; + int idx; +}; + +static int vdpa_dev_dump(struct device *dev, void *data) +{ + struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev); + struct vdpa_dev_dump_info *info = data; + int err; + + if (!vdev->mdev) + return 0; + if (info->idx < info->start_idx) { + info->idx++; + return 0; + } + err = vdpa_dev_fill(vdev, info->msg, NETLINK_CB(info->cb->skb).portid, + info->cb->nlh->nlmsg_seq, NLM_F_MULTI, info->cb->extack); + if (err) + return err; + + info->idx++; + return 0; +} + +static int vdpa_nl_cmd_dev_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb) +{ + struct vdpa_dev_dump_info info; + + info.msg = msg; + info.cb = cb; + info.start_idx = cb->args[0]; + info.idx = 0; + + mutex_lock(&vdpa_dev_mutex); + bus_for_each_dev(&vdpa_bus, NULL, &info, vdpa_dev_dump); + mutex_unlock(&vdpa_dev_mutex); + cb->args[0] = info.idx; + return msg->len; +} + +static const struct nla_policy vdpa_nl_policy[VDPA_ATTR_MAX + 1] = { + [VDPA_ATTR_MGMTDEV_BUS_NAME] = { .type = NLA_NUL_STRING }, + [VDPA_ATTR_MGMTDEV_DEV_NAME] = { .type = NLA_STRING }, + [VDPA_ATTR_DEV_NAME] = { .type = NLA_STRING }, +}; + +static const struct genl_ops vdpa_nl_ops[] = { + { + .cmd = VDPA_CMD_MGMTDEV_GET, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .doit = vdpa_nl_cmd_mgmtdev_get_doit, + .dumpit = vdpa_nl_cmd_mgmtdev_get_dumpit, + }, + { + .cmd = VDPA_CMD_DEV_NEW, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .doit = vdpa_nl_cmd_dev_add_set_doit, + .flags = GENL_ADMIN_PERM, + }, + { + .cmd = VDPA_CMD_DEV_DEL, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .doit = vdpa_nl_cmd_dev_del_set_doit, + .flags = GENL_ADMIN_PERM, + }, + { + .cmd = VDPA_CMD_DEV_GET, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .doit = vdpa_nl_cmd_dev_get_doit, + .dumpit = vdpa_nl_cmd_dev_get_dumpit, + }, +}; + +static struct genl_family vdpa_nl_family __ro_after_init = { + .name = VDPA_GENL_NAME, + .version = VDPA_GENL_VERSION, + .maxattr = VDPA_ATTR_MAX, + .policy = vdpa_nl_policy, + .netnsok = false, + .module = THIS_MODULE, + .ops = vdpa_nl_ops, + .n_ops = ARRAY_SIZE(vdpa_nl_ops), +}; + static int vdpa_init(void) { - return bus_register(&vdpa_bus); + int err; + + err = bus_register(&vdpa_bus); + if (err) + return err; + err = genl_register_family(&vdpa_nl_family); + if (err) + goto err; + return 0; + +err: + bus_unregister(&vdpa_bus); + return err; } static void __exit vdpa_exit(void) { + genl_unregister_family(&vdpa_nl_family); bus_unregister(&vdpa_bus); ida_destroy(&vdpa_index_ida); } diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index b3fcc67bfdf0..d5942842432d 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -235,7 +235,7 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr) ops = &vdpasim_config_ops; vdpasim = vdpa_alloc_device(struct vdpasim, vdpa, NULL, ops, - dev_attr->nvqs); + dev_attr->nvqs, dev_attr->name); if (!vdpasim) goto err_alloc; @@ -249,6 +249,7 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr) if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64))) goto err_iommu; set_dma_ops(dev, &vdpasim_dma_ops); + vdpasim->vdpa.mdev = dev_attr->mgmt_dev; vdpasim->config = kzalloc(dev_attr->config_size, GFP_KERNEL); if (!vdpasim->config) diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h index b02142293d5b..6d75444f9948 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.h +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h @@ -33,6 +33,8 @@ struct vdpasim_virtqueue { }; struct vdpasim_dev_attr { + struct vdpa_mgmt_dev *mgmt_dev; + const char *name; u64 supported_features; size_t config_size; size_t buffer_size; diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_net.c b/drivers/vdpa/vdpa_sim/vdpa_sim_net.c index c10b6981fdab..d344c5b7c914 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_net.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_net.c @@ -33,9 +33,7 @@ static char *macaddr; module_param(macaddr, charp, 0); MODULE_PARM_DESC(macaddr, "Ethernet MAC address"); -u8 macaddr_buf[ETH_ALEN]; - -static struct vdpasim *vdpasim_net_dev; +static u8 macaddr_buf[ETH_ALEN]; static void vdpasim_net_work(struct work_struct *work) { @@ -120,21 +118,23 @@ static void vdpasim_net_get_config(struct vdpasim *vdpasim, void *config) memcpy(net_config->mac, macaddr_buf, ETH_ALEN); } -static int __init vdpasim_net_init(void) +static void vdpasim_net_mgmtdev_release(struct device *dev) +{ +} + +static struct device vdpasim_net_mgmtdev = { + .init_name = "vdpasim_net", + .release = vdpasim_net_mgmtdev_release, +}; + +static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name) { struct vdpasim_dev_attr dev_attr = {}; + struct vdpasim *simdev; int ret; - if (macaddr) { - mac_pton(macaddr, macaddr_buf); - if (!is_valid_ether_addr(macaddr_buf)) { - ret = -EADDRNOTAVAIL; - goto out; - } - } else { - eth_random_addr(macaddr_buf); - } - + dev_attr.mgmt_dev = mdev; + dev_attr.name = name; dev_attr.id = VIRTIO_ID_NET; dev_attr.supported_features = VDPASIM_NET_FEATURES; dev_attr.nvqs = VDPASIM_NET_VQ_NUM; @@ -143,29 +143,75 @@ static int __init vdpasim_net_init(void) dev_attr.work_fn = vdpasim_net_work; dev_attr.buffer_size = PAGE_SIZE; - vdpasim_net_dev = vdpasim_create(&dev_attr); - if (IS_ERR(vdpasim_net_dev)) { - ret = PTR_ERR(vdpasim_net_dev); - goto out; + simdev = vdpasim_create(&dev_attr); + if (IS_ERR(simdev)) + return PTR_ERR(simdev); + + ret = _vdpa_register_device(&simdev->vdpa); + if (ret) + goto reg_err; + + return 0; + +reg_err: + put_device(&simdev->vdpa.dev); + return ret; +} + +static void vdpasim_net_dev_del(struct vdpa_mgmt_dev *mdev, + struct vdpa_device *dev) +{ + struct vdpasim *simdev = container_of(dev, struct vdpasim, vdpa); + + _vdpa_unregister_device(&simdev->vdpa); +} + +static const struct vdpa_mgmtdev_ops vdpasim_net_mgmtdev_ops = { + .dev_add = vdpasim_net_dev_add, + .dev_del = vdpasim_net_dev_del +}; + +static struct virtio_device_id id_table[] = { + { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, + { 0 }, +}; + +static struct vdpa_mgmt_dev mgmt_dev = { + .device = &vdpasim_net_mgmtdev, + .id_table = id_table, + .ops = &vdpasim_net_mgmtdev_ops, +}; + +static int __init vdpasim_net_init(void) +{ + int ret; + + if (macaddr) { + mac_pton(macaddr, macaddr_buf); + if (!is_valid_ether_addr(macaddr_buf)) + return -EADDRNOTAVAIL; + } else { + eth_random_addr(macaddr_buf); } - ret = vdpa_register_device(&vdpasim_net_dev->vdpa); + ret = device_register(&vdpasim_net_mgmtdev); if (ret) - goto put_dev; + return ret; + ret = vdpa_mgmtdev_register(&mgmt_dev); + if (ret) + goto parent_err; return 0; -put_dev: - put_device(&vdpasim_net_dev->vdpa.dev); -out: +parent_err: + device_unregister(&vdpasim_net_mgmtdev); return ret; } static void __exit vdpasim_net_exit(void) { - struct vdpa_device *vdpa = &vdpasim_net_dev->vdpa; - - vdpa_unregister_device(vdpa); + vdpa_mgmtdev_unregister(&mgmt_dev); + device_unregister(&vdpasim_net_mgmtdev); } module_init(vdpasim_net_init); diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig index 40a223381ab6..ac3c1dd3edef 100644 --- a/drivers/vfio/pci/Kconfig +++ b/drivers/vfio/pci/Kconfig @@ -45,15 +45,3 @@ config VFIO_PCI_NVLINK2 depends on VFIO_PCI && PPC_POWERNV help VFIO PCI support for P9 Witherspoon machine with NVIDIA V100 GPUs - -config VFIO_PCI_ZDEV - bool "VFIO PCI ZPCI device CLP support" - depends on VFIO_PCI && S390 - default y - help - Enabling this option exposes VFIO capabilities containing hardware - configuration for zPCI devices. This enables userspace (e.g. QEMU) - to supply proper configuration values instead of hard-coded defaults - for zPCI devices passed through via VFIO on s390. - - Say Y here. diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile index 781e0809d6ee..eff97a7cd9f1 100644 --- a/drivers/vfio/pci/Makefile +++ b/drivers/vfio/pci/Makefile @@ -3,6 +3,6 @@ vfio-pci-y := vfio_pci.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o vfio-pci-$(CONFIG_VFIO_PCI_IGD) += vfio_pci_igd.o vfio-pci-$(CONFIG_VFIO_PCI_NVLINK2) += vfio_pci_nvlink2.o -vfio-pci-$(CONFIG_VFIO_PCI_ZDEV) += vfio_pci_zdev.o +vfio-pci-$(CONFIG_S390) += vfio_pci_zdev.o obj-$(CONFIG_VFIO_PCI) += vfio-pci.o diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 706de3ef94bb..65e7e6b44578 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -807,6 +807,7 @@ static long vfio_pci_ioctl(void *device_data, struct vfio_device_info info; struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; unsigned long capsz; + int ret; minsz = offsetofend(struct vfio_device_info, num_irqs); @@ -832,13 +833,10 @@ static long vfio_pci_ioctl(void *device_data, info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; info.num_irqs = VFIO_PCI_NUM_IRQS; - if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV)) { - int ret = vfio_pci_info_zdev_add_caps(vdev, &caps); - - if (ret && ret != -ENODEV) { - pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n"); - return ret; - } + ret = vfio_pci_info_zdev_add_caps(vdev, &caps); + if (ret && ret != -ENODEV) { + pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n"); + return ret; } if (caps.size) { diff --git a/drivers/vfio/pci/vfio_pci_igd.c b/drivers/vfio/pci/vfio_pci_igd.c index 53d97f459252..e66dfb0178ed 100644 --- a/drivers/vfio/pci/vfio_pci_igd.c +++ b/drivers/vfio/pci/vfio_pci_igd.c @@ -127,7 +127,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, ret = pci_user_read_config_byte(pdev, pos, &val); if (ret) - return pcibios_err_to_errno(ret); + return ret; if (copy_to_user(buf + count - size, &val, 1)) return -EFAULT; @@ -141,7 +141,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, ret = pci_user_read_config_word(pdev, pos, &val); if (ret) - return pcibios_err_to_errno(ret); + return ret; val = cpu_to_le16(val); if (copy_to_user(buf + count - size, &val, 2)) @@ -156,7 +156,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, ret = pci_user_read_config_dword(pdev, pos, &val); if (ret) - return pcibios_err_to_errno(ret); + return ret; val = cpu_to_le32(val); if (copy_to_user(buf + count - size, &val, 4)) @@ -171,7 +171,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, ret = pci_user_read_config_word(pdev, pos, &val); if (ret) - return pcibios_err_to_errno(ret); + return ret; val = cpu_to_le16(val); if (copy_to_user(buf + count - size, &val, 2)) @@ -186,7 +186,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, ret = pci_user_read_config_byte(pdev, pos, &val); if (ret) - return pcibios_err_to_errno(ret); + return ret; if (copy_to_user(buf + count - size, &val, 1)) return -EFAULT; diff --git a/drivers/vfio/pci/vfio_pci_private.h b/drivers/vfio/pci/vfio_pci_private.h index 5c90e560c5c7..9cd1882a05af 100644 --- a/drivers/vfio/pci/vfio_pci_private.h +++ b/drivers/vfio/pci/vfio_pci_private.h @@ -214,7 +214,7 @@ static inline int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev) } #endif -#ifdef CONFIG_VFIO_PCI_ZDEV +#ifdef CONFIG_S390 extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev, struct vfio_info_cap *caps); #else diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c index 229685634031..7b011b62c766 100644 --- a/drivers/vfio/pci/vfio_pci_zdev.c +++ b/drivers/vfio/pci/vfio_pci_zdev.c @@ -24,8 +24,7 @@ /* * Add the Base PCI Function information to the device info region. */ -static int zpci_base_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, - struct vfio_info_cap *caps) +static int zpci_base_cap(struct zpci_dev *zdev, struct vfio_info_cap *caps) { struct vfio_device_info_cap_zpci_base cap = { .header.id = VFIO_DEVICE_INFO_CAP_ZPCI_BASE, @@ -45,8 +44,7 @@ static int zpci_base_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, /* * Add the Base PCI Function Group information to the device info region. */ -static int zpci_group_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, - struct vfio_info_cap *caps) +static int zpci_group_cap(struct zpci_dev *zdev, struct vfio_info_cap *caps) { struct vfio_device_info_cap_zpci_group cap = { .header.id = VFIO_DEVICE_INFO_CAP_ZPCI_GROUP, @@ -66,14 +64,15 @@ static int zpci_group_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, /* * Add the device utility string to the device info region. */ -static int zpci_util_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, - struct vfio_info_cap *caps) +static int zpci_util_cap(struct zpci_dev *zdev, struct vfio_info_cap *caps) { struct vfio_device_info_cap_zpci_util *cap; int cap_size = sizeof(*cap) + CLP_UTIL_STR_LEN; int ret; cap = kmalloc(cap_size, GFP_KERNEL); + if (!cap) + return -ENOMEM; cap->header.id = VFIO_DEVICE_INFO_CAP_ZPCI_UTIL; cap->header.version = 1; @@ -90,14 +89,15 @@ static int zpci_util_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, /* * Add the function path string to the device info region. */ -static int zpci_pfip_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, - struct vfio_info_cap *caps) +static int zpci_pfip_cap(struct zpci_dev *zdev, struct vfio_info_cap *caps) { struct vfio_device_info_cap_zpci_pfip *cap; int cap_size = sizeof(*cap) + CLP_PFIP_NR_SEGMENTS; int ret; cap = kmalloc(cap_size, GFP_KERNEL); + if (!cap) + return -ENOMEM; cap->header.id = VFIO_DEVICE_INFO_CAP_ZPCI_PFIP; cap->header.version = 1; @@ -123,21 +123,21 @@ int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev, if (!zdev) return -ENODEV; - ret = zpci_base_cap(zdev, vdev, caps); + ret = zpci_base_cap(zdev, caps); if (ret) return ret; - ret = zpci_group_cap(zdev, vdev, caps); + ret = zpci_group_cap(zdev, caps); if (ret) return ret; if (zdev->util_str_avail) { - ret = zpci_util_cap(zdev, vdev, caps); + ret = zpci_util_cap(zdev, caps); if (ret) return ret; } - ret = zpci_pfip_cap(zdev, vdev, caps); + ret = zpci_pfip_cap(zdev, caps); return ret; } diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 4ad8a35667a7..38779e6fd80c 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -1220,6 +1220,11 @@ static int vfio_fops_open(struct inode *inode, struct file *filep) static int vfio_fops_release(struct inode *inode, struct file *filep) { struct vfio_container *container = filep->private_data; + struct vfio_iommu_driver *driver = container->iommu_driver; + + if (driver && driver->ops->notify) + driver->ops->notify(container->iommu_data, + VFIO_IOMMU_CONTAINER_CLOSE); filep->private_data = NULL; diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 0b4dedaa9128..4bb162c1d649 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -24,6 +24,7 @@ #include <linux/compat.h> #include <linux/device.h> #include <linux/fs.h> +#include <linux/highmem.h> #include <linux/iommu.h> #include <linux/module.h> #include <linux/mm.h> @@ -69,11 +70,15 @@ struct vfio_iommu { struct rb_root dma_list; struct blocking_notifier_head notifier; unsigned int dma_avail; + unsigned int vaddr_invalid_count; uint64_t pgsize_bitmap; + uint64_t num_non_pinned_groups; + wait_queue_head_t vaddr_wait; bool v2; bool nesting; bool dirty_page_tracking; bool pinned_page_dirty_scope; + bool container_open; }; struct vfio_domain { @@ -92,11 +97,20 @@ struct vfio_dma { int prot; /* IOMMU_READ/WRITE */ bool iommu_mapped; bool lock_cap; /* capable(CAP_IPC_LOCK) */ + bool vaddr_invalid; struct task_struct *task; struct rb_root pfn_list; /* Ex-user pinned pfn list */ unsigned long *bitmap; }; +struct vfio_batch { + struct page **pages; /* for pin_user_pages_remote */ + struct page *fallback_page; /* if pages alloc fails */ + int capacity; /* length of pages array */ + int size; /* of batch currently */ + int offset; /* of next entry in pages */ +}; + struct vfio_group { struct iommu_group *iommu_group; struct list_head next; @@ -143,12 +157,13 @@ struct vfio_regions { #define DIRTY_BITMAP_PAGES_MAX ((u64)INT_MAX) #define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX) +#define WAITED 1 + static int put_pfn(unsigned long pfn, int prot); static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, struct iommu_group *iommu_group); -static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu); /* * This code handles mapping and unmapping of user data buffers * into DMA'ble space using the IOMMU @@ -173,6 +188,31 @@ static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu, return NULL; } +static struct rb_node *vfio_find_dma_first_node(struct vfio_iommu *iommu, + dma_addr_t start, size_t size) +{ + struct rb_node *res = NULL; + struct rb_node *node = iommu->dma_list.rb_node; + struct vfio_dma *dma_res = NULL; + + while (node) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + if (start < dma->iova + dma->size) { + res = node; + dma_res = dma; + if (start >= dma->iova) + break; + node = node->rb_left; + } else { + node = node->rb_right; + } + } + if (res && size && dma_res->iova >= start + size) + res = NULL; + return res; +} + static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new) { struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL; @@ -236,6 +276,18 @@ static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) } } +static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu) +{ + struct rb_node *n; + unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + + for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { + struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); + + bitmap_set(dma->bitmap, 0, dma->size >> pgshift); + } +} + static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize) { struct rb_node *n; @@ -415,13 +467,54 @@ static int put_pfn(unsigned long pfn, int prot) return 0; } +#define VFIO_BATCH_MAX_CAPACITY (PAGE_SIZE / sizeof(struct page *)) + +static void vfio_batch_init(struct vfio_batch *batch) +{ + batch->size = 0; + batch->offset = 0; + + if (unlikely(disable_hugepages)) + goto fallback; + + batch->pages = (struct page **) __get_free_page(GFP_KERNEL); + if (!batch->pages) + goto fallback; + + batch->capacity = VFIO_BATCH_MAX_CAPACITY; + return; + +fallback: + batch->pages = &batch->fallback_page; + batch->capacity = 1; +} + +static void vfio_batch_unpin(struct vfio_batch *batch, struct vfio_dma *dma) +{ + while (batch->size) { + unsigned long pfn = page_to_pfn(batch->pages[batch->offset]); + + put_pfn(pfn, dma->prot); + batch->offset++; + batch->size--; + } +} + +static void vfio_batch_fini(struct vfio_batch *batch) +{ + if (batch->capacity == VFIO_BATCH_MAX_CAPACITY) + free_page((unsigned long)batch->pages); +} + static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, unsigned long vaddr, unsigned long *pfn, bool write_fault) { + pte_t *ptep; + spinlock_t *ptl; int ret; - ret = follow_pfn(vma, vaddr, pfn); + ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl); if (ret) { bool unlocked = false; @@ -435,16 +528,28 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, if (ret) return ret; - ret = follow_pfn(vma, vaddr, pfn); + ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl); + if (ret) + return ret; } + if (write_fault && !pte_write(*ptep)) + ret = -EFAULT; + else + *pfn = pte_pfn(*ptep); + + pte_unmap_unlock(ptep, ptl); return ret; } -static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, - int prot, unsigned long *pfn) +/* + * Returns the positive number of pfns successfully obtained or a negative + * error code. + */ +static int vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr, + long npages, int prot, unsigned long *pfn, + struct page **pages) { - struct page *page[1]; struct vm_area_struct *vma; unsigned int flags = 0; int ret; @@ -453,11 +558,10 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, flags |= FOLL_WRITE; mmap_read_lock(mm); - ret = pin_user_pages_remote(mm, vaddr, 1, flags | FOLL_LONGTERM, - page, NULL, NULL); - if (ret == 1) { - *pfn = page_to_pfn(page[0]); - ret = 0; + ret = pin_user_pages_remote(mm, vaddr, npages, flags | FOLL_LONGTERM, + pages, NULL, NULL); + if (ret > 0) { + *pfn = page_to_pfn(pages[0]); goto done; } @@ -471,14 +575,73 @@ retry: if (ret == -EAGAIN) goto retry; - if (!ret && !is_invalid_reserved_pfn(*pfn)) - ret = -EFAULT; + if (!ret) { + if (is_invalid_reserved_pfn(*pfn)) + ret = 1; + else + ret = -EFAULT; + } } done: mmap_read_unlock(mm); return ret; } +static int vfio_wait(struct vfio_iommu *iommu) +{ + DEFINE_WAIT(wait); + + prepare_to_wait(&iommu->vaddr_wait, &wait, TASK_KILLABLE); + mutex_unlock(&iommu->lock); + schedule(); + mutex_lock(&iommu->lock); + finish_wait(&iommu->vaddr_wait, &wait); + if (kthread_should_stop() || !iommu->container_open || + fatal_signal_pending(current)) { + return -EFAULT; + } + return WAITED; +} + +/* + * Find dma struct and wait for its vaddr to be valid. iommu lock is dropped + * if the task waits, but is re-locked on return. Return result in *dma_p. + * Return 0 on success with no waiting, WAITED on success if waited, and -errno + * on error. + */ +static int vfio_find_dma_valid(struct vfio_iommu *iommu, dma_addr_t start, + size_t size, struct vfio_dma **dma_p) +{ + int ret; + + do { + *dma_p = vfio_find_dma(iommu, start, size); + if (!*dma_p) + ret = -EINVAL; + else if (!(*dma_p)->vaddr_invalid) + ret = 0; + else + ret = vfio_wait(iommu); + } while (ret > 0); + + return ret; +} + +/* + * Wait for all vaddr in the dma_list to become valid. iommu lock is dropped + * if the task waits, but is re-locked on return. Return 0 on success with no + * waiting, WAITED on success if waited, and -errno on error. + */ +static int vfio_wait_all_valid(struct vfio_iommu *iommu) +{ + int ret = 0; + + while (iommu->vaddr_invalid_count && ret >= 0) + ret = vfio_wait(iommu); + + return ret; +} + /* * Attempt to pin pages. We really don't want to track all the pfns and * the iommu can only map chunks of consecutive pfns anyway, so get the @@ -486,76 +649,102 @@ done: */ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, long npage, unsigned long *pfn_base, - unsigned long limit) + unsigned long limit, struct vfio_batch *batch) { - unsigned long pfn = 0; + unsigned long pfn; + struct mm_struct *mm = current->mm; long ret, pinned = 0, lock_acct = 0; bool rsvd; dma_addr_t iova = vaddr - dma->vaddr + dma->iova; /* This code path is only user initiated */ - if (!current->mm) + if (!mm) return -ENODEV; - ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, pfn_base); - if (ret) - return ret; - - pinned++; - rsvd = is_invalid_reserved_pfn(*pfn_base); - - /* - * Reserved pages aren't counted against the user, externally pinned - * pages are already counted against the user. - */ - if (!rsvd && !vfio_find_vpfn(dma, iova)) { - if (!dma->lock_cap && current->mm->locked_vm + 1 > limit) { - put_pfn(*pfn_base, dma->prot); - pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__, - limit << PAGE_SHIFT); - return -ENOMEM; - } - lock_acct++; + if (batch->size) { + /* Leftover pages in batch from an earlier call. */ + *pfn_base = page_to_pfn(batch->pages[batch->offset]); + pfn = *pfn_base; + rsvd = is_invalid_reserved_pfn(*pfn_base); + } else { + *pfn_base = 0; } - if (unlikely(disable_hugepages)) - goto out; + while (npage) { + if (!batch->size) { + /* Empty batch, so refill it. */ + long req_pages = min_t(long, npage, batch->capacity); - /* Lock all the consecutive pages from pfn_base */ - for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage; - pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) { - ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, &pfn); - if (ret) - break; + ret = vaddr_get_pfns(mm, vaddr, req_pages, dma->prot, + &pfn, batch->pages); + if (ret < 0) + goto unpin_out; - if (pfn != *pfn_base + pinned || - rsvd != is_invalid_reserved_pfn(pfn)) { - put_pfn(pfn, dma->prot); - break; + batch->size = ret; + batch->offset = 0; + + if (!*pfn_base) { + *pfn_base = pfn; + rsvd = is_invalid_reserved_pfn(*pfn_base); + } } - if (!rsvd && !vfio_find_vpfn(dma, iova)) { - if (!dma->lock_cap && - current->mm->locked_vm + lock_acct + 1 > limit) { - put_pfn(pfn, dma->prot); - pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", - __func__, limit << PAGE_SHIFT); - ret = -ENOMEM; - goto unpin_out; + /* + * pfn is preset for the first iteration of this inner loop and + * updated at the end to handle a VM_PFNMAP pfn. In that case, + * batch->pages isn't valid (there's no struct page), so allow + * batch->pages to be touched only when there's more than one + * pfn to check, which guarantees the pfns are from a + * !VM_PFNMAP vma. + */ + while (true) { + if (pfn != *pfn_base + pinned || + rsvd != is_invalid_reserved_pfn(pfn)) + goto out; + + /* + * Reserved pages aren't counted against the user, + * externally pinned pages are already counted against + * the user. + */ + if (!rsvd && !vfio_find_vpfn(dma, iova)) { + if (!dma->lock_cap && + mm->locked_vm + lock_acct + 1 > limit) { + pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", + __func__, limit << PAGE_SHIFT); + ret = -ENOMEM; + goto unpin_out; + } + lock_acct++; } - lock_acct++; + + pinned++; + npage--; + vaddr += PAGE_SIZE; + iova += PAGE_SIZE; + batch->offset++; + batch->size--; + + if (!batch->size) + break; + + pfn = page_to_pfn(batch->pages[batch->offset]); } + + if (unlikely(disable_hugepages)) + break; } out: ret = vfio_lock_acct(dma, lock_acct, false); unpin_out: - if (ret) { - if (!rsvd) { + if (ret < 0) { + if (pinned && !rsvd) { for (pfn = *pfn_base ; pinned ; pfn++, pinned--) put_pfn(pfn, dma->prot); } + vfio_batch_unpin(batch, dma); return ret; } @@ -587,6 +776,7 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova, static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr, unsigned long *pfn_base, bool do_accounting) { + struct page *pages[1]; struct mm_struct *mm; int ret; @@ -594,8 +784,8 @@ static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr, if (!mm) return -ENODEV; - ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base); - if (!ret && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) { + ret = vaddr_get_pfns(mm, vaddr, 1, dma->prot, pfn_base, pages); + if (ret == 1 && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) { ret = vfio_lock_acct(dma, 1, true); if (ret) { put_pfn(*pfn_base, dma->prot); @@ -640,6 +830,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, unsigned long remote_vaddr; struct vfio_dma *dma; bool do_accounting; + dma_addr_t iova; if (!iommu || !user_pfn || !phys_pfn) return -EINVAL; @@ -650,6 +841,22 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, mutex_lock(&iommu->lock); + /* + * Wait for all necessary vaddr's to be valid so they can be used in + * the main loop without dropping the lock, to avoid racing vs unmap. + */ +again: + if (iommu->vaddr_invalid_count) { + for (i = 0; i < npage; i++) { + iova = user_pfn[i] << PAGE_SHIFT; + ret = vfio_find_dma_valid(iommu, iova, PAGE_SIZE, &dma); + if (ret < 0) + goto pin_done; + if (ret == WAITED) + goto again; + } + } + /* Fail if notifier list is empty */ if (!iommu->notifier.head) { ret = -EINVAL; @@ -664,7 +871,6 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu); for (i = 0; i < npage; i++) { - dma_addr_t iova; struct vfio_pfn *vpfn; iova = user_pfn[i] << PAGE_SHIFT; @@ -714,7 +920,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, group = vfio_iommu_find_iommu_group(iommu, iommu_group); if (!group->pinned_page_dirty_scope) { group->pinned_page_dirty_scope = true; - update_pinned_page_dirty_scope(iommu); + iommu->num_non_pinned_groups--; } goto pin_done; @@ -945,10 +1151,15 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) { + WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)); vfio_unmap_unpin(iommu, dma, true); vfio_unlink_dma(iommu, dma); put_task_struct(dma->task); vfio_dma_bitmap_free(dma); + if (dma->vaddr_invalid) { + iommu->vaddr_invalid_count--; + wake_up_all(&iommu->vaddr_wait); + } kfree(dma); iommu->dma_avail++; } @@ -991,7 +1202,7 @@ static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, * mark all pages dirty if any IOMMU capable device is not able * to report dirty pages and all pages are pinned and mapped. */ - if (!iommu->pinned_page_dirty_scope && dma->iommu_mapped) + if (iommu->num_non_pinned_groups && dma->iommu_mapped) bitmap_set(dma->bitmap, 0, nbits); if (shift) { @@ -1074,34 +1285,36 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, { struct vfio_dma *dma, *dma_last = NULL; size_t unmapped = 0, pgsize; - int ret = 0, retries = 0; + int ret = -EINVAL, retries = 0; unsigned long pgshift; + dma_addr_t iova = unmap->iova; + unsigned long size = unmap->size; + bool unmap_all = unmap->flags & VFIO_DMA_UNMAP_FLAG_ALL; + bool invalidate_vaddr = unmap->flags & VFIO_DMA_UNMAP_FLAG_VADDR; + struct rb_node *n, *first_n; mutex_lock(&iommu->lock); pgshift = __ffs(iommu->pgsize_bitmap); pgsize = (size_t)1 << pgshift; - if (unmap->iova & (pgsize - 1)) { - ret = -EINVAL; + if (iova & (pgsize - 1)) goto unlock; - } - if (!unmap->size || unmap->size & (pgsize - 1)) { - ret = -EINVAL; + if (unmap_all) { + if (iova || size) + goto unlock; + size = SIZE_MAX; + } else if (!size || size & (pgsize - 1)) { goto unlock; } - if (unmap->iova + unmap->size - 1 < unmap->iova || - unmap->size > SIZE_MAX) { - ret = -EINVAL; + if (iova + size - 1 < iova || size > SIZE_MAX) goto unlock; - } /* When dirty tracking is enabled, allow only min supported pgsize */ if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) { - ret = -EINVAL; goto unlock; } @@ -1138,21 +1351,25 @@ again: * will only return success and a size of zero if there were no * mappings within the range. */ - if (iommu->v2) { - dma = vfio_find_dma(iommu, unmap->iova, 1); - if (dma && dma->iova != unmap->iova) { - ret = -EINVAL; + if (iommu->v2 && !unmap_all) { + dma = vfio_find_dma(iommu, iova, 1); + if (dma && dma->iova != iova) goto unlock; - } - dma = vfio_find_dma(iommu, unmap->iova + unmap->size - 1, 0); - if (dma && dma->iova + dma->size != unmap->iova + unmap->size) { - ret = -EINVAL; + + dma = vfio_find_dma(iommu, iova + size - 1, 0); + if (dma && dma->iova + dma->size != iova + size) goto unlock; - } } - while ((dma = vfio_find_dma(iommu, unmap->iova, unmap->size))) { - if (!iommu->v2 && unmap->iova > dma->iova) + ret = 0; + n = first_n = vfio_find_dma_first_node(iommu, iova, size); + + while (n) { + dma = rb_entry(n, struct vfio_dma, node); + if (dma->iova >= iova + size) + break; + + if (!iommu->v2 && iova > dma->iova) break; /* * Task with same address space who mapped this iova range is @@ -1161,6 +1378,27 @@ again: if (dma->task->mm != current->mm) break; + if (invalidate_vaddr) { + if (dma->vaddr_invalid) { + struct rb_node *last_n = n; + + for (n = first_n; n != last_n; n = rb_next(n)) { + dma = rb_entry(n, + struct vfio_dma, node); + dma->vaddr_invalid = false; + iommu->vaddr_invalid_count--; + } + ret = -EINVAL; + unmapped = 0; + break; + } + dma->vaddr_invalid = true; + iommu->vaddr_invalid_count++; + unmapped += dma->size; + n = rb_next(n); + continue; + } + if (!RB_EMPTY_ROOT(&dma->pfn_list)) { struct vfio_iommu_type1_dma_unmap nb_unmap; @@ -1190,12 +1428,13 @@ again: if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { ret = update_user_bitmap(bitmap->data, iommu, dma, - unmap->iova, pgsize); + iova, pgsize); if (ret) break; } unmapped += dma->size; + n = rb_next(n); vfio_remove_dma(iommu, dma); } @@ -1239,15 +1478,19 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, { dma_addr_t iova = dma->iova; unsigned long vaddr = dma->vaddr; + struct vfio_batch batch; size_t size = map_size; long npage; unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; + vfio_batch_init(&batch); + while (size) { /* Pin a contiguous chunk of memory */ npage = vfio_pin_pages_remote(dma, vaddr + dma->size, - size >> PAGE_SHIFT, &pfn, limit); + size >> PAGE_SHIFT, &pfn, limit, + &batch); if (npage <= 0) { WARN_ON(!npage); ret = (int)npage; @@ -1260,6 +1503,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, if (ret) { vfio_unpin_pages_remote(dma, iova + dma->size, pfn, npage, true); + vfio_batch_unpin(&batch, dma); break; } @@ -1267,6 +1511,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, dma->size += npage << PAGE_SHIFT; } + vfio_batch_fini(&batch); dma->iommu_mapped = true; if (ret) @@ -1299,6 +1544,7 @@ static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, static int vfio_dma_do_map(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_map *map) { + bool set_vaddr = map->flags & VFIO_DMA_MAP_FLAG_VADDR; dma_addr_t iova = map->iova; unsigned long vaddr = map->vaddr; size_t size = map->size; @@ -1316,13 +1562,16 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, if (map->flags & VFIO_DMA_MAP_FLAG_READ) prot |= IOMMU_READ; + if ((prot && set_vaddr) || (!prot && !set_vaddr)) + return -EINVAL; + mutex_lock(&iommu->lock); pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); WARN_ON((pgsize - 1) & PAGE_MASK); - if (!prot || !size || (size | iova | vaddr) & (pgsize - 1)) { + if (!size || (size | iova | vaddr) & (pgsize - 1)) { ret = -EINVAL; goto out_unlock; } @@ -1333,7 +1582,21 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, goto out_unlock; } - if (vfio_find_dma(iommu, iova, size)) { + dma = vfio_find_dma(iommu, iova, size); + if (set_vaddr) { + if (!dma) { + ret = -ENOENT; + } else if (!dma->vaddr_invalid || dma->iova != iova || + dma->size != size) { + ret = -EINVAL; + } else { + dma->vaddr = vaddr; + dma->vaddr_invalid = false; + iommu->vaddr_invalid_count--; + wake_up_all(&iommu->vaddr_wait); + } + goto out_unlock; + } else if (dma) { ret = -EEXIST; goto out_unlock; } @@ -1425,16 +1688,23 @@ static int vfio_bus_type(struct device *dev, void *data) static int vfio_iommu_replay(struct vfio_iommu *iommu, struct vfio_domain *domain) { + struct vfio_batch batch; struct vfio_domain *d = NULL; struct rb_node *n; unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret; + ret = vfio_wait_all_valid(iommu); + if (ret < 0) + return ret; + /* Arbitrarily pick the first domain in the list for lookups */ if (!list_empty(&iommu->domain_list)) d = list_first_entry(&iommu->domain_list, struct vfio_domain, next); + vfio_batch_init(&batch); + n = rb_first(&iommu->dma_list); for (; n; n = rb_next(n)) { @@ -1482,7 +1752,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, npage = vfio_pin_pages_remote(dma, vaddr, n >> PAGE_SHIFT, - &pfn, limit); + &pfn, limit, + &batch); if (npage <= 0) { WARN_ON(!npage); ret = (int)npage; @@ -1496,11 +1767,13 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, ret = iommu_map(domain->domain, iova, phys, size, dma->prot | domain->prot); if (ret) { - if (!dma->iommu_mapped) + if (!dma->iommu_mapped) { vfio_unpin_pages_remote(dma, iova, phys >> PAGE_SHIFT, size >> PAGE_SHIFT, true); + vfio_batch_unpin(&batch, dma); + } goto unwind; } @@ -1515,6 +1788,7 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, dma->iommu_mapped = true; } + vfio_batch_fini(&batch); return 0; unwind: @@ -1555,6 +1829,7 @@ unwind: } } + vfio_batch_fini(&batch); return ret; } @@ -1622,33 +1897,6 @@ static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, return group; } -static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu) -{ - struct vfio_domain *domain; - struct vfio_group *group; - - list_for_each_entry(domain, &iommu->domain_list, next) { - list_for_each_entry(group, &domain->group_list, next) { - if (!group->pinned_page_dirty_scope) { - iommu->pinned_page_dirty_scope = false; - return; - } - } - } - - if (iommu->external_domain) { - domain = iommu->external_domain; - list_for_each_entry(group, &domain->group_list, next) { - if (!group->pinned_page_dirty_scope) { - iommu->pinned_page_dirty_scope = false; - return; - } - } - } - - iommu->pinned_page_dirty_scope = true; -} - static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions, phys_addr_t *base) { @@ -2057,8 +2305,6 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, * addition of a dirty tracking group. */ group->pinned_page_dirty_scope = true; - if (!iommu->pinned_page_dirty_scope) - update_pinned_page_dirty_scope(iommu); mutex_unlock(&iommu->lock); return 0; @@ -2188,7 +2434,7 @@ done: * demotes the iommu scope until it declares itself dirty tracking * capable via the page pinning interface. */ - iommu->pinned_page_dirty_scope = false; + iommu->num_non_pinned_groups++; mutex_unlock(&iommu->lock); vfio_iommu_resv_free(&group_resv_regions); @@ -2238,23 +2484,6 @@ static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu) } } -static void vfio_sanity_check_pfn_list(struct vfio_iommu *iommu) -{ - struct rb_node *n; - - n = rb_first(&iommu->dma_list); - for (; n; n = rb_next(n)) { - struct vfio_dma *dma; - - dma = rb_entry(n, struct vfio_dma, node); - - if (WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list))) - break; - } - /* mdev vendor driver must unregister notifier */ - WARN_ON(iommu->notifier.head); -} - /* * Called when a domain is removed in detach. It is possible that * the removed domain decided the iova aperture window. Modify the @@ -2354,10 +2583,10 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, kfree(group); if (list_empty(&iommu->external_domain->group_list)) { - vfio_sanity_check_pfn_list(iommu); - - if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) + if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) { + WARN_ON(iommu->notifier.head); vfio_iommu_unmap_unpin_all(iommu); + } kfree(iommu->external_domain); iommu->external_domain = NULL; @@ -2391,10 +2620,12 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, */ if (list_empty(&domain->group_list)) { if (list_is_singular(&iommu->domain_list)) { - if (!iommu->external_domain) + if (!iommu->external_domain) { + WARN_ON(iommu->notifier.head); vfio_iommu_unmap_unpin_all(iommu); - else + } else { vfio_iommu_unmap_unpin_reaccount(iommu); + } } iommu_domain_free(domain->domain); list_del(&domain->next); @@ -2415,8 +2646,11 @@ detach_group_done: * Removal of a group without dirty tracking may allow the iommu scope * to be promoted. */ - if (update_dirty_scope) - update_pinned_page_dirty_scope(iommu); + if (update_dirty_scope) { + iommu->num_non_pinned_groups--; + if (iommu->dirty_page_tracking) + vfio_iommu_populate_bitmap_full(iommu); + } mutex_unlock(&iommu->lock); } @@ -2446,8 +2680,10 @@ static void *vfio_iommu_type1_open(unsigned long arg) INIT_LIST_HEAD(&iommu->iova_list); iommu->dma_list = RB_ROOT; iommu->dma_avail = dma_entry_limit; + iommu->container_open = true; mutex_init(&iommu->lock); BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier); + init_waitqueue_head(&iommu->vaddr_wait); return iommu; } @@ -2475,7 +2711,6 @@ static void vfio_iommu_type1_release(void *iommu_data) if (iommu->external_domain) { vfio_release_domain(iommu->external_domain, true); - vfio_sanity_check_pfn_list(iommu); kfree(iommu->external_domain); } @@ -2517,6 +2752,8 @@ static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, case VFIO_TYPE1_IOMMU: case VFIO_TYPE1v2_IOMMU: case VFIO_TYPE1_NESTING_IOMMU: + case VFIO_UNMAP_ALL: + case VFIO_UPDATE_VADDR: return 1; case VFIO_DMA_CC_IOMMU: if (!iommu) @@ -2688,7 +2925,8 @@ static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu, { struct vfio_iommu_type1_dma_map map; unsigned long minsz; - uint32_t mask = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + uint32_t mask = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE | + VFIO_DMA_MAP_FLAG_VADDR; minsz = offsetofend(struct vfio_iommu_type1_dma_map, size); @@ -2706,6 +2944,9 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, { struct vfio_iommu_type1_dma_unmap unmap; struct vfio_bitmap bitmap = { 0 }; + uint32_t mask = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP | + VFIO_DMA_UNMAP_FLAG_VADDR | + VFIO_DMA_UNMAP_FLAG_ALL; unsigned long minsz; int ret; @@ -2714,8 +2955,12 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, if (copy_from_user(&unmap, (void __user *)arg, minsz)) return -EFAULT; - if (unmap.argsz < minsz || - unmap.flags & ~VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) + if (unmap.argsz < minsz || unmap.flags & ~mask) + return -EINVAL; + + if ((unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && + (unmap.flags & (VFIO_DMA_UNMAP_FLAG_ALL | + VFIO_DMA_UNMAP_FLAG_VADDR))) return -EINVAL; if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { @@ -2906,12 +3151,13 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, struct vfio_dma *dma; bool kthread = current->mm == NULL; size_t offset; + int ret; *copied = 0; - dma = vfio_find_dma(iommu, user_iova, 1); - if (!dma) - return -EINVAL; + ret = vfio_find_dma_valid(iommu, user_iova, 1, &dma); + if (ret < 0) + return ret; if ((write && !(dma->prot & IOMMU_WRITE)) || !(dma->prot & IOMMU_READ)) @@ -3003,6 +3249,19 @@ vfio_iommu_type1_group_iommu_domain(void *iommu_data, return domain; } +static void vfio_iommu_type1_notify(void *iommu_data, + enum vfio_iommu_notify_type event) +{ + struct vfio_iommu *iommu = iommu_data; + + if (event != VFIO_IOMMU_CONTAINER_CLOSE) + return; + mutex_lock(&iommu->lock); + iommu->container_open = false; + mutex_unlock(&iommu->lock); + wake_up_all(&iommu->vaddr_wait); +} + static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = { .name = "vfio-iommu-type1", .owner = THIS_MODULE, @@ -3017,6 +3276,7 @@ static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = { .unregister_notifier = vfio_iommu_type1_unregister_notifier, .dma_rw = vfio_iommu_type1_dma_rw, .group_iommu_domain = vfio_iommu_type1_group_iommu_domain, + .notify = vfio_iommu_type1_notify, }; static int __init vfio_iommu_type1_init(void) diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 4ce9f00ae10e..5de21ad4bd05 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -1814,12 +1814,9 @@ static int vhost_scsi_open(struct inode *inode, struct file *f) struct vhost_virtqueue **vqs; int r = -ENOMEM, i; - vs = kzalloc(sizeof(*vs), GFP_KERNEL | __GFP_NOWARN | __GFP_RETRY_MAYFAIL); - if (!vs) { - vs = vzalloc(sizeof(*vs)); - if (!vs) - goto err_vs; - } + vs = kvzalloc(sizeof(*vs), GFP_KERNEL); + if (!vs) + goto err_vs; vqs = kmalloc_array(VHOST_SCSI_MAX_VQ, sizeof(*vqs), GFP_KERNEL); if (!vqs) diff --git a/drivers/video/fbdev/acornfb.c b/drivers/video/fbdev/acornfb.c index bcc92aecf666..1b72edc01cfb 100644 --- a/drivers/video/fbdev/acornfb.c +++ b/drivers/video/fbdev/acornfb.c @@ -921,40 +921,6 @@ static int acornfb_detect_monitortype(void) return 4; } -/* - * This enables the unused memory to be freed on older Acorn machines. - * We are freeing memory on behalf of the architecture initialisation - * code here. - */ -static inline void -free_unused_pages(unsigned int virtual_start, unsigned int virtual_end) -{ - int mb_freed = 0; - - /* - * Align addresses - */ - virtual_start = PAGE_ALIGN(virtual_start); - virtual_end = PAGE_ALIGN(virtual_end); - - while (virtual_start < virtual_end) { - struct page *page; - - /* - * Clear page reserved bit, - * set count to 1, and free - * the page. - */ - page = virt_to_page(virtual_start); - __free_reserved_page(page); - - virtual_start += PAGE_SIZE; - mb_freed += PAGE_SIZE / 1024; - } - - printk("acornfb: freed %dK memory\n", mb_freed); -} - static int acornfb_probe(struct platform_device *dev) { unsigned long size; diff --git a/drivers/video/fbdev/aty/atyfb_base.c b/drivers/video/fbdev/aty/atyfb_base.c index 83c8e809955a..e946903a86c2 100644 --- a/drivers/video/fbdev/aty/atyfb_base.c +++ b/drivers/video/fbdev/aty/atyfb_base.c @@ -175,6 +175,15 @@ u32 aty_ld_lcd(int index, const struct atyfb_par *par) return aty_ld_le32(LCD_DATA, par); } } +#else /* defined(CONFIG_PMAC_BACKLIGHT) || defined(CONFIG_FB_ATY_BACKLIGHT) \ + defined(CONFIG_FB_ATY_GENERIC_LCD) */ +void aty_st_lcd(int index, u32 val, const struct atyfb_par *par) +{ } + +u32 aty_ld_lcd(int index, const struct atyfb_par *par) +{ + return 0; +} #endif /* defined(CONFIG_PMAC_BACKLIGHT) || defined (CONFIG_FB_ATY_GENERIC_LCD) */ #ifdef CONFIG_FB_ATY_GENERIC_LCD diff --git a/drivers/virt/Kconfig b/drivers/virt/Kconfig index 80c5f9c16ec1..8061e8ef449f 100644 --- a/drivers/virt/Kconfig +++ b/drivers/virt/Kconfig @@ -34,4 +34,6 @@ config FSL_HV_MANAGER source "drivers/virt/vboxguest/Kconfig" source "drivers/virt/nitro_enclaves/Kconfig" + +source "drivers/virt/acrn/Kconfig" endif diff --git a/drivers/virt/Makefile b/drivers/virt/Makefile index f28425ce4b39..3e272ea60cd9 100644 --- a/drivers/virt/Makefile +++ b/drivers/virt/Makefile @@ -7,3 +7,4 @@ obj-$(CONFIG_FSL_HV_MANAGER) += fsl_hypervisor.o obj-y += vboxguest/ obj-$(CONFIG_NITRO_ENCLAVES) += nitro_enclaves/ +obj-$(CONFIG_ACRN_HSM) += acrn/ diff --git a/drivers/virt/acrn/Kconfig b/drivers/virt/acrn/Kconfig new file mode 100644 index 000000000000..3e1a61c9d8d8 --- /dev/null +++ b/drivers/virt/acrn/Kconfig @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0 +config ACRN_HSM + tristate "ACRN Hypervisor Service Module" + depends on ACRN_GUEST + select EVENTFD + help + ACRN Hypervisor Service Module (HSM) is a kernel module which + communicates with ACRN userspace through ioctls and talks to + the ACRN Hypervisor through hypercalls. HSM will only run in + a privileged management VM, called Service VM, to manage User + VMs and do I/O emulation. Not required for simply running + under ACRN as a User VM. + + To compile as a module, choose M, the module will be called + acrn. If unsure, say N. diff --git a/drivers/virt/acrn/Makefile b/drivers/virt/acrn/Makefile new file mode 100644 index 000000000000..08ce641dcfa1 --- /dev/null +++ b/drivers/virt/acrn/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_ACRN_HSM) := acrn.o +acrn-y := hsm.o vm.o mm.o ioreq.o ioeventfd.o irqfd.o diff --git a/drivers/virt/acrn/acrn_drv.h b/drivers/virt/acrn/acrn_drv.h new file mode 100644 index 000000000000..1be54efa666c --- /dev/null +++ b/drivers/virt/acrn/acrn_drv.h @@ -0,0 +1,227 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ACRN_HSM_DRV_H +#define __ACRN_HSM_DRV_H + +#include <linux/acrn.h> +#include <linux/dev_printk.h> +#include <linux/miscdevice.h> +#include <linux/types.h> + +#include "hypercall.h" + +extern struct miscdevice acrn_dev; + +#define ACRN_NAME_LEN 16 +#define ACRN_MEM_MAPPING_MAX 256 + +#define ACRN_MEM_REGION_ADD 0 +#define ACRN_MEM_REGION_DEL 2 + +struct acrn_vm; +struct acrn_ioreq_client; + +/** + * struct vm_memory_region_op - Hypervisor memory operation + * @type: Operation type (ACRN_MEM_REGION_*) + * @attr: Memory attribute (ACRN_MEM_TYPE_* | ACRN_MEM_ACCESS_*) + * @user_vm_pa: Physical address of User VM to be mapped. + * @service_vm_pa: Physical address of Service VM to be mapped. + * @size: Size of this region. + * + * Structure containing needed information that is provided to ACRN Hypervisor + * to manage the EPT mappings of a single memory region of the User VM. Several + * &struct vm_memory_region_op can be batched to ACRN Hypervisor, see &struct + * vm_memory_region_batch. + */ +struct vm_memory_region_op { + u32 type; + u32 attr; + u64 user_vm_pa; + u64 service_vm_pa; + u64 size; +}; + +/** + * struct vm_memory_region_batch - A batch of vm_memory_region_op. + * @vmid: A User VM ID. + * @reserved: Reserved. + * @regions_num: The number of vm_memory_region_op. + * @regions_gpa: Physical address of a vm_memory_region_op array. + * + * HC_VM_SET_MEMORY_REGIONS uses this structure to manage EPT mappings of + * multiple memory regions of a User VM. A &struct vm_memory_region_batch + * contains multiple &struct vm_memory_region_op for batch processing in the + * ACRN Hypervisor. + */ +struct vm_memory_region_batch { + u16 vmid; + u16 reserved[3]; + u32 regions_num; + u64 regions_gpa; +}; + +/** + * struct vm_memory_mapping - Memory map between a User VM and the Service VM + * @pages: Pages in Service VM kernel. + * @npages: Number of pages. + * @service_vm_va: Virtual address in Service VM kernel. + * @user_vm_pa: Physical address in User VM. + * @size: Size of this memory region. + * + * HSM maintains memory mappings between a User VM GPA and the Service VM + * kernel VA for accelerating the User VM GPA translation. + */ +struct vm_memory_mapping { + struct page **pages; + int npages; + void *service_vm_va; + u64 user_vm_pa; + size_t size; +}; + +/** + * struct acrn_ioreq_buffer - Data for setting the ioreq buffer of User VM + * @ioreq_buf: The GPA of the IO request shared buffer of a VM + * + * The parameter for the HC_SET_IOREQ_BUFFER hypercall used to set up + * the shared I/O request buffer between Service VM and ACRN hypervisor. + */ +struct acrn_ioreq_buffer { + u64 ioreq_buf; +}; + +struct acrn_ioreq_range { + struct list_head list; + u32 type; + u64 start; + u64 end; +}; + +#define ACRN_IOREQ_CLIENT_DESTROYING 0U +typedef int (*ioreq_handler_t)(struct acrn_ioreq_client *client, + struct acrn_io_request *req); +/** + * struct acrn_ioreq_client - Structure of I/O client. + * @name: Client name + * @vm: The VM that the client belongs to + * @list: List node for this acrn_ioreq_client + * @is_default: If this client is the default one + * @flags: Flags (ACRN_IOREQ_CLIENT_*) + * @range_list: I/O ranges + * @range_lock: Lock to protect range_list + * @ioreqs_map: The pending I/O requests bitmap. + * @handler: I/O requests handler of this client + * @thread: The thread which executes the handler + * @wq: The wait queue for the handler thread parking + * @priv: Data for the thread + */ +struct acrn_ioreq_client { + char name[ACRN_NAME_LEN]; + struct acrn_vm *vm; + struct list_head list; + bool is_default; + unsigned long flags; + struct list_head range_list; + rwlock_t range_lock; + DECLARE_BITMAP(ioreqs_map, ACRN_IO_REQUEST_MAX); + ioreq_handler_t handler; + struct task_struct *thread; + wait_queue_head_t wq; + void *priv; +}; + +#define ACRN_INVALID_VMID (0xffffU) + +#define ACRN_VM_FLAG_DESTROYED 0U +#define ACRN_VM_FLAG_CLEARING_IOREQ 1U +extern struct list_head acrn_vm_list; +extern rwlock_t acrn_vm_list_lock; +/** + * struct acrn_vm - Properties of ACRN User VM. + * @list: Entry within global list of all VMs. + * @vmid: User VM ID. + * @vcpu_num: Number of virtual CPUs in the VM. + * @flags: Flags (ACRN_VM_FLAG_*) of the VM. This is VM + * flag management in HSM which is different + * from the &acrn_vm_creation.vm_flag. + * @regions_mapping_lock: Lock to protect &acrn_vm.regions_mapping and + * &acrn_vm.regions_mapping_count. + * @regions_mapping: Memory mappings of this VM. + * @regions_mapping_count: Number of memory mapping of this VM. + * @ioreq_clients_lock: Lock to protect ioreq_clients and default_client + * @ioreq_clients: The I/O request clients list of this VM + * @default_client: The default I/O request client + * @ioreq_buf: I/O request shared buffer + * @ioreq_page: The page of the I/O request shared buffer + * @pci_conf_addr: Address of a PCI configuration access emulation + * @monitor_page: Page of interrupt statistics of User VM + * @ioeventfds_lock: Lock to protect ioeventfds list + * @ioeventfds: List to link all hsm_ioeventfd + * @ioeventfd_client: I/O client for ioeventfds of the VM + * @irqfds_lock: Lock to protect irqfds list + * @irqfds: List to link all hsm_irqfd + * @irqfd_wq: Workqueue for irqfd async shutdown + */ +struct acrn_vm { + struct list_head list; + u16 vmid; + int vcpu_num; + unsigned long flags; + struct mutex regions_mapping_lock; + struct vm_memory_mapping regions_mapping[ACRN_MEM_MAPPING_MAX]; + int regions_mapping_count; + spinlock_t ioreq_clients_lock; + struct list_head ioreq_clients; + struct acrn_ioreq_client *default_client; + struct acrn_io_request_buffer *ioreq_buf; + struct page *ioreq_page; + u32 pci_conf_addr; + struct page *monitor_page; + struct mutex ioeventfds_lock; + struct list_head ioeventfds; + struct acrn_ioreq_client *ioeventfd_client; + struct mutex irqfds_lock; + struct list_head irqfds; + struct workqueue_struct *irqfd_wq; +}; + +struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, + struct acrn_vm_creation *vm_param); +int acrn_vm_destroy(struct acrn_vm *vm); +int acrn_mm_region_add(struct acrn_vm *vm, u64 user_gpa, u64 service_gpa, + u64 size, u32 mem_type, u32 mem_access_right); +int acrn_mm_region_del(struct acrn_vm *vm, u64 user_gpa, u64 size); +int acrn_vm_memseg_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap); +int acrn_vm_memseg_unmap(struct acrn_vm *vm, struct acrn_vm_memmap *memmap); +int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap); +void acrn_vm_all_ram_unmap(struct acrn_vm *vm); + +int acrn_ioreq_init(struct acrn_vm *vm, u64 buf_vma); +void acrn_ioreq_deinit(struct acrn_vm *vm); +int acrn_ioreq_intr_setup(void); +void acrn_ioreq_intr_remove(void); +void acrn_ioreq_request_clear(struct acrn_vm *vm); +int acrn_ioreq_client_wait(struct acrn_ioreq_client *client); +int acrn_ioreq_request_default_complete(struct acrn_vm *vm, u16 vcpu); +struct acrn_ioreq_client *acrn_ioreq_client_create(struct acrn_vm *vm, + ioreq_handler_t handler, + void *data, bool is_default, + const char *name); +void acrn_ioreq_client_destroy(struct acrn_ioreq_client *client); +int acrn_ioreq_range_add(struct acrn_ioreq_client *client, + u32 type, u64 start, u64 end); +void acrn_ioreq_range_del(struct acrn_ioreq_client *client, + u32 type, u64 start, u64 end); + +int acrn_msi_inject(struct acrn_vm *vm, u64 msi_addr, u64 msi_data); + +int acrn_ioeventfd_init(struct acrn_vm *vm); +int acrn_ioeventfd_config(struct acrn_vm *vm, struct acrn_ioeventfd *args); +void acrn_ioeventfd_deinit(struct acrn_vm *vm); + +int acrn_irqfd_init(struct acrn_vm *vm); +int acrn_irqfd_config(struct acrn_vm *vm, struct acrn_irqfd *args); +void acrn_irqfd_deinit(struct acrn_vm *vm); + +#endif /* __ACRN_HSM_DRV_H */ diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c new file mode 100644 index 000000000000..1f6b7c54a1a4 --- /dev/null +++ b/drivers/virt/acrn/hsm.c @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ACRN Hypervisor Service Module (HSM) + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * + * Authors: + * Fengwei Yin <fengwei.yin@intel.com> + * Yakui Zhao <yakui.zhao@intel.com> + */ + +#include <linux/cpu.h> +#include <linux/io.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/slab.h> + +#include <asm/acrn.h> +#include <asm/hypervisor.h> + +#include "acrn_drv.h" + +/* + * When /dev/acrn_hsm is opened, a 'struct acrn_vm' object is created to + * represent a VM instance and continues to be associated with the opened file + * descriptor. All ioctl operations on this file descriptor will be targeted to + * the VM instance. Release of this file descriptor will destroy the object. + */ +static int acrn_dev_open(struct inode *inode, struct file *filp) +{ + struct acrn_vm *vm; + + vm = kzalloc(sizeof(*vm), GFP_KERNEL); + if (!vm) + return -ENOMEM; + + vm->vmid = ACRN_INVALID_VMID; + filp->private_data = vm; + return 0; +} + +static int pmcmd_ioctl(u64 cmd, void __user *uptr) +{ + struct acrn_pstate_data *px_data; + struct acrn_cstate_data *cx_data; + u64 *pm_info; + int ret = 0; + + switch (cmd & PMCMD_TYPE_MASK) { + case ACRN_PMCMD_GET_PX_CNT: + case ACRN_PMCMD_GET_CX_CNT: + pm_info = kmalloc(sizeof(u64), GFP_KERNEL); + if (!pm_info) + return -ENOMEM; + + ret = hcall_get_cpu_state(cmd, virt_to_phys(pm_info)); + if (ret < 0) { + kfree(pm_info); + break; + } + + if (copy_to_user(uptr, pm_info, sizeof(u64))) + ret = -EFAULT; + kfree(pm_info); + break; + case ACRN_PMCMD_GET_PX_DATA: + px_data = kmalloc(sizeof(*px_data), GFP_KERNEL); + if (!px_data) + return -ENOMEM; + + ret = hcall_get_cpu_state(cmd, virt_to_phys(px_data)); + if (ret < 0) { + kfree(px_data); + break; + } + + if (copy_to_user(uptr, px_data, sizeof(*px_data))) + ret = -EFAULT; + kfree(px_data); + break; + case ACRN_PMCMD_GET_CX_DATA: + cx_data = kmalloc(sizeof(*cx_data), GFP_KERNEL); + if (!cx_data) + return -ENOMEM; + + ret = hcall_get_cpu_state(cmd, virt_to_phys(cx_data)); + if (ret < 0) { + kfree(cx_data); + break; + } + + if (copy_to_user(uptr, cx_data, sizeof(*cx_data))) + ret = -EFAULT; + kfree(cx_data); + break; + default: + break; + } + + return ret; +} + +/* + * HSM relies on hypercall layer of the ACRN hypervisor to do the + * sanity check against the input parameters. + */ +static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, + unsigned long ioctl_param) +{ + struct acrn_vm *vm = filp->private_data; + struct acrn_vm_creation *vm_param; + struct acrn_vcpu_regs *cpu_regs; + struct acrn_ioreq_notify notify; + struct acrn_ptdev_irq *irq_info; + struct acrn_ioeventfd ioeventfd; + struct acrn_vm_memmap memmap; + struct acrn_msi_entry *msi; + struct acrn_pcidev *pcidev; + struct acrn_irqfd irqfd; + struct page *page; + u64 cstate_cmd; + int i, ret = 0; + + if (vm->vmid == ACRN_INVALID_VMID && cmd != ACRN_IOCTL_CREATE_VM) { + dev_dbg(acrn_dev.this_device, + "ioctl 0x%x: Invalid VM state!\n", cmd); + return -EINVAL; + } + + switch (cmd) { + case ACRN_IOCTL_CREATE_VM: + vm_param = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_vm_creation)); + if (IS_ERR(vm_param)) + return PTR_ERR(vm_param); + + if ((vm_param->reserved0 | vm_param->reserved1) != 0) + return -EINVAL; + + vm = acrn_vm_create(vm, vm_param); + if (!vm) { + ret = -EINVAL; + kfree(vm_param); + break; + } + + if (copy_to_user((void __user *)ioctl_param, vm_param, + sizeof(struct acrn_vm_creation))) { + acrn_vm_destroy(vm); + ret = -EFAULT; + } + + kfree(vm_param); + break; + case ACRN_IOCTL_START_VM: + ret = hcall_start_vm(vm->vmid); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to start VM %u!\n", vm->vmid); + break; + case ACRN_IOCTL_PAUSE_VM: + ret = hcall_pause_vm(vm->vmid); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to pause VM %u!\n", vm->vmid); + break; + case ACRN_IOCTL_RESET_VM: + ret = hcall_reset_vm(vm->vmid); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to restart VM %u!\n", vm->vmid); + break; + case ACRN_IOCTL_DESTROY_VM: + ret = acrn_vm_destroy(vm); + break; + case ACRN_IOCTL_SET_VCPU_REGS: + cpu_regs = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_vcpu_regs)); + if (IS_ERR(cpu_regs)) + return PTR_ERR(cpu_regs); + + for (i = 0; i < ARRAY_SIZE(cpu_regs->reserved); i++) + if (cpu_regs->reserved[i]) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(cpu_regs->vcpu_regs.reserved_32); i++) + if (cpu_regs->vcpu_regs.reserved_32[i]) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(cpu_regs->vcpu_regs.reserved_64); i++) + if (cpu_regs->vcpu_regs.reserved_64[i]) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(cpu_regs->vcpu_regs.gdt.reserved); i++) + if (cpu_regs->vcpu_regs.gdt.reserved[i] | + cpu_regs->vcpu_regs.idt.reserved[i]) + return -EINVAL; + + ret = hcall_set_vcpu_regs(vm->vmid, virt_to_phys(cpu_regs)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to set regs state of VM%u!\n", + vm->vmid); + kfree(cpu_regs); + break; + case ACRN_IOCTL_SET_MEMSEG: + if (copy_from_user(&memmap, (void __user *)ioctl_param, + sizeof(memmap))) + return -EFAULT; + + ret = acrn_vm_memseg_map(vm, &memmap); + break; + case ACRN_IOCTL_UNSET_MEMSEG: + if (copy_from_user(&memmap, (void __user *)ioctl_param, + sizeof(memmap))) + return -EFAULT; + + ret = acrn_vm_memseg_unmap(vm, &memmap); + break; + case ACRN_IOCTL_ASSIGN_PCIDEV: + pcidev = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_pcidev)); + if (IS_ERR(pcidev)) + return PTR_ERR(pcidev); + + ret = hcall_assign_pcidev(vm->vmid, virt_to_phys(pcidev)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to assign pci device!\n"); + kfree(pcidev); + break; + case ACRN_IOCTL_DEASSIGN_PCIDEV: + pcidev = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_pcidev)); + if (IS_ERR(pcidev)) + return PTR_ERR(pcidev); + + ret = hcall_deassign_pcidev(vm->vmid, virt_to_phys(pcidev)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to deassign pci device!\n"); + kfree(pcidev); + break; + case ACRN_IOCTL_SET_PTDEV_INTR: + irq_info = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_ptdev_irq)); + if (IS_ERR(irq_info)) + return PTR_ERR(irq_info); + + ret = hcall_set_ptdev_intr(vm->vmid, virt_to_phys(irq_info)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to configure intr for ptdev!\n"); + kfree(irq_info); + break; + case ACRN_IOCTL_RESET_PTDEV_INTR: + irq_info = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_ptdev_irq)); + if (IS_ERR(irq_info)) + return PTR_ERR(irq_info); + + ret = hcall_reset_ptdev_intr(vm->vmid, virt_to_phys(irq_info)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to reset intr for ptdev!\n"); + kfree(irq_info); + break; + case ACRN_IOCTL_SET_IRQLINE: + ret = hcall_set_irqline(vm->vmid, ioctl_param); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to set interrupt line!\n"); + break; + case ACRN_IOCTL_INJECT_MSI: + msi = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_msi_entry)); + if (IS_ERR(msi)) + return PTR_ERR(msi); + + ret = hcall_inject_msi(vm->vmid, virt_to_phys(msi)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to inject MSI!\n"); + kfree(msi); + break; + case ACRN_IOCTL_VM_INTR_MONITOR: + ret = pin_user_pages_fast(ioctl_param, 1, + FOLL_WRITE | FOLL_LONGTERM, &page); + if (unlikely(ret != 1)) { + dev_dbg(acrn_dev.this_device, + "Failed to pin intr hdr buffer!\n"); + return -EFAULT; + } + + ret = hcall_vm_intr_monitor(vm->vmid, page_to_phys(page)); + if (ret < 0) { + unpin_user_page(page); + dev_dbg(acrn_dev.this_device, + "Failed to monitor intr data!\n"); + return ret; + } + if (vm->monitor_page) + unpin_user_page(vm->monitor_page); + vm->monitor_page = page; + break; + case ACRN_IOCTL_CREATE_IOREQ_CLIENT: + if (vm->default_client) + return -EEXIST; + if (!acrn_ioreq_client_create(vm, NULL, NULL, true, "acrndm")) + ret = -EINVAL; + break; + case ACRN_IOCTL_DESTROY_IOREQ_CLIENT: + if (vm->default_client) + acrn_ioreq_client_destroy(vm->default_client); + break; + case ACRN_IOCTL_ATTACH_IOREQ_CLIENT: + if (vm->default_client) + ret = acrn_ioreq_client_wait(vm->default_client); + else + ret = -ENODEV; + break; + case ACRN_IOCTL_NOTIFY_REQUEST_FINISH: + if (copy_from_user(¬ify, (void __user *)ioctl_param, + sizeof(struct acrn_ioreq_notify))) + return -EFAULT; + + if (notify.reserved != 0) + return -EINVAL; + + ret = acrn_ioreq_request_default_complete(vm, notify.vcpu); + break; + case ACRN_IOCTL_CLEAR_VM_IOREQ: + acrn_ioreq_request_clear(vm); + break; + case ACRN_IOCTL_PM_GET_CPU_STATE: + if (copy_from_user(&cstate_cmd, (void *)ioctl_param, + sizeof(cstate_cmd))) + return -EFAULT; + + ret = pmcmd_ioctl(cstate_cmd, (void __user *)ioctl_param); + break; + case ACRN_IOCTL_IOEVENTFD: + if (copy_from_user(&ioeventfd, (void __user *)ioctl_param, + sizeof(ioeventfd))) + return -EFAULT; + + if (ioeventfd.reserved != 0) + return -EINVAL; + + ret = acrn_ioeventfd_config(vm, &ioeventfd); + break; + case ACRN_IOCTL_IRQFD: + if (copy_from_user(&irqfd, (void __user *)ioctl_param, + sizeof(irqfd))) + return -EFAULT; + ret = acrn_irqfd_config(vm, &irqfd); + break; + default: + dev_dbg(acrn_dev.this_device, "Unknown IOCTL 0x%x!\n", cmd); + ret = -ENOTTY; + } + + return ret; +} + +static int acrn_dev_release(struct inode *inode, struct file *filp) +{ + struct acrn_vm *vm = filp->private_data; + + acrn_vm_destroy(vm); + kfree(vm); + return 0; +} + +static ssize_t remove_cpu_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + u64 cpu, lapicid; + int ret; + + if (kstrtoull(buf, 0, &cpu) < 0) + return -EINVAL; + + if (cpu >= num_possible_cpus() || cpu == 0 || !cpu_is_hotpluggable(cpu)) + return -EINVAL; + + if (cpu_online(cpu)) + remove_cpu(cpu); + + lapicid = cpu_data(cpu).apicid; + dev_dbg(dev, "Try to remove cpu %lld with lapicid %lld\n", cpu, lapicid); + ret = hcall_sos_remove_cpu(lapicid); + if (ret < 0) { + dev_err(dev, "Failed to remove cpu %lld!\n", cpu); + goto fail_remove; + } + + return count; + +fail_remove: + add_cpu(cpu); + return ret; +} +static DEVICE_ATTR_WO(remove_cpu); + +static struct attribute *acrn_attrs[] = { + &dev_attr_remove_cpu.attr, + NULL +}; + +static struct attribute_group acrn_attr_group = { + .attrs = acrn_attrs, +}; + +static const struct attribute_group *acrn_attr_groups[] = { + &acrn_attr_group, + NULL +}; + +static const struct file_operations acrn_fops = { + .owner = THIS_MODULE, + .open = acrn_dev_open, + .release = acrn_dev_release, + .unlocked_ioctl = acrn_dev_ioctl, +}; + +struct miscdevice acrn_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "acrn_hsm", + .fops = &acrn_fops, + .groups = acrn_attr_groups, +}; + +static int __init hsm_init(void) +{ + int ret; + + if (x86_hyper_type != X86_HYPER_ACRN) + return -ENODEV; + + if (!(cpuid_eax(ACRN_CPUID_FEATURES) & ACRN_FEATURE_PRIVILEGED_VM)) + return -EPERM; + + ret = misc_register(&acrn_dev); + if (ret) { + pr_err("Create misc dev failed!\n"); + return ret; + } + + ret = acrn_ioreq_intr_setup(); + if (ret) { + pr_err("Setup I/O request handler failed!\n"); + misc_deregister(&acrn_dev); + return ret; + } + return 0; +} + +static void __exit hsm_exit(void) +{ + acrn_ioreq_intr_remove(); + misc_deregister(&acrn_dev); +} +module_init(hsm_init); +module_exit(hsm_exit); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("ACRN Hypervisor Service Module (HSM)"); diff --git a/drivers/virt/acrn/hypercall.h b/drivers/virt/acrn/hypercall.h new file mode 100644 index 000000000000..0cfad05bd1a9 --- /dev/null +++ b/drivers/virt/acrn/hypercall.h @@ -0,0 +1,254 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ACRN HSM: hypercalls of ACRN Hypervisor + */ +#ifndef __ACRN_HSM_HYPERCALL_H +#define __ACRN_HSM_HYPERCALL_H +#include <asm/acrn.h> + +/* + * Hypercall IDs of the ACRN Hypervisor + */ +#define _HC_ID(x, y) (((x) << 24) | (y)) + +#define HC_ID 0x80UL + +#define HC_ID_GEN_BASE 0x0UL +#define HC_SOS_REMOVE_CPU _HC_ID(HC_ID, HC_ID_GEN_BASE + 0x01) + +#define HC_ID_VM_BASE 0x10UL +#define HC_CREATE_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x00) +#define HC_DESTROY_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x01) +#define HC_START_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x02) +#define HC_PAUSE_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x03) +#define HC_RESET_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x05) +#define HC_SET_VCPU_REGS _HC_ID(HC_ID, HC_ID_VM_BASE + 0x06) + +#define HC_ID_IRQ_BASE 0x20UL +#define HC_INJECT_MSI _HC_ID(HC_ID, HC_ID_IRQ_BASE + 0x03) +#define HC_VM_INTR_MONITOR _HC_ID(HC_ID, HC_ID_IRQ_BASE + 0x04) +#define HC_SET_IRQLINE _HC_ID(HC_ID, HC_ID_IRQ_BASE + 0x05) + +#define HC_ID_IOREQ_BASE 0x30UL +#define HC_SET_IOREQ_BUFFER _HC_ID(HC_ID, HC_ID_IOREQ_BASE + 0x00) +#define HC_NOTIFY_REQUEST_FINISH _HC_ID(HC_ID, HC_ID_IOREQ_BASE + 0x01) + +#define HC_ID_MEM_BASE 0x40UL +#define HC_VM_SET_MEMORY_REGIONS _HC_ID(HC_ID, HC_ID_MEM_BASE + 0x02) + +#define HC_ID_PCI_BASE 0x50UL +#define HC_SET_PTDEV_INTR _HC_ID(HC_ID, HC_ID_PCI_BASE + 0x03) +#define HC_RESET_PTDEV_INTR _HC_ID(HC_ID, HC_ID_PCI_BASE + 0x04) +#define HC_ASSIGN_PCIDEV _HC_ID(HC_ID, HC_ID_PCI_BASE + 0x05) +#define HC_DEASSIGN_PCIDEV _HC_ID(HC_ID, HC_ID_PCI_BASE + 0x06) + +#define HC_ID_PM_BASE 0x80UL +#define HC_PM_GET_CPU_STATE _HC_ID(HC_ID, HC_ID_PM_BASE + 0x00) + +/** + * hcall_sos_remove_cpu() - Remove a vCPU of Service VM + * @cpu: The vCPU to be removed + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_sos_remove_cpu(u64 cpu) +{ + return acrn_hypercall1(HC_SOS_REMOVE_CPU, cpu); +} + +/** + * hcall_create_vm() - Create a User VM + * @vminfo: Service VM GPA of info of User VM creation + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_create_vm(u64 vminfo) +{ + return acrn_hypercall1(HC_CREATE_VM, vminfo); +} + +/** + * hcall_start_vm() - Start a User VM + * @vmid: User VM ID + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_start_vm(u64 vmid) +{ + return acrn_hypercall1(HC_START_VM, vmid); +} + +/** + * hcall_pause_vm() - Pause a User VM + * @vmid: User VM ID + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_pause_vm(u64 vmid) +{ + return acrn_hypercall1(HC_PAUSE_VM, vmid); +} + +/** + * hcall_destroy_vm() - Destroy a User VM + * @vmid: User VM ID + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_destroy_vm(u64 vmid) +{ + return acrn_hypercall1(HC_DESTROY_VM, vmid); +} + +/** + * hcall_reset_vm() - Reset a User VM + * @vmid: User VM ID + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_reset_vm(u64 vmid) +{ + return acrn_hypercall1(HC_RESET_VM, vmid); +} + +/** + * hcall_set_vcpu_regs() - Set up registers of virtual CPU of a User VM + * @vmid: User VM ID + * @regs_state: Service VM GPA of registers state + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_set_vcpu_regs(u64 vmid, u64 regs_state) +{ + return acrn_hypercall2(HC_SET_VCPU_REGS, vmid, regs_state); +} + +/** + * hcall_inject_msi() - Deliver a MSI interrupt to a User VM + * @vmid: User VM ID + * @msi: Service VM GPA of MSI message + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_inject_msi(u64 vmid, u64 msi) +{ + return acrn_hypercall2(HC_INJECT_MSI, vmid, msi); +} + +/** + * hcall_vm_intr_monitor() - Set a shared page for User VM interrupt statistics + * @vmid: User VM ID + * @addr: Service VM GPA of the shared page + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_vm_intr_monitor(u64 vmid, u64 addr) +{ + return acrn_hypercall2(HC_VM_INTR_MONITOR, vmid, addr); +} + +/** + * hcall_set_irqline() - Set or clear an interrupt line + * @vmid: User VM ID + * @op: Service VM GPA of interrupt line operations + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_set_irqline(u64 vmid, u64 op) +{ + return acrn_hypercall2(HC_SET_IRQLINE, vmid, op); +} + +/** + * hcall_set_ioreq_buffer() - Set up the shared buffer for I/O Requests. + * @vmid: User VM ID + * @buffer: Service VM GPA of the shared buffer + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_set_ioreq_buffer(u64 vmid, u64 buffer) +{ + return acrn_hypercall2(HC_SET_IOREQ_BUFFER, vmid, buffer); +} + +/** + * hcall_notify_req_finish() - Notify ACRN Hypervisor of I/O request completion. + * @vmid: User VM ID + * @vcpu: The vCPU which initiated the I/O request + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_notify_req_finish(u64 vmid, u64 vcpu) +{ + return acrn_hypercall2(HC_NOTIFY_REQUEST_FINISH, vmid, vcpu); +} + +/** + * hcall_set_memory_regions() - Inform the hypervisor to set up EPT mappings + * @regions_pa: Service VM GPA of &struct vm_memory_region_batch + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_set_memory_regions(u64 regions_pa) +{ + return acrn_hypercall1(HC_VM_SET_MEMORY_REGIONS, regions_pa); +} + +/** + * hcall_assign_pcidev() - Assign a PCI device to a User VM + * @vmid: User VM ID + * @addr: Service VM GPA of the &struct acrn_pcidev + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_assign_pcidev(u64 vmid, u64 addr) +{ + return acrn_hypercall2(HC_ASSIGN_PCIDEV, vmid, addr); +} + +/** + * hcall_deassign_pcidev() - De-assign a PCI device from a User VM + * @vmid: User VM ID + * @addr: Service VM GPA of the &struct acrn_pcidev + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_deassign_pcidev(u64 vmid, u64 addr) +{ + return acrn_hypercall2(HC_DEASSIGN_PCIDEV, vmid, addr); +} + +/** + * hcall_set_ptdev_intr() - Configure an interrupt for an assigned PCI device. + * @vmid: User VM ID + * @irq: Service VM GPA of the &struct acrn_ptdev_irq + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_set_ptdev_intr(u64 vmid, u64 irq) +{ + return acrn_hypercall2(HC_SET_PTDEV_INTR, vmid, irq); +} + +/** + * hcall_reset_ptdev_intr() - Reset an interrupt for an assigned PCI device. + * @vmid: User VM ID + * @irq: Service VM GPA of the &struct acrn_ptdev_irq + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_reset_ptdev_intr(u64 vmid, u64 irq) +{ + return acrn_hypercall2(HC_RESET_PTDEV_INTR, vmid, irq); +} + +/* + * hcall_get_cpu_state() - Get P-states and C-states info from the hypervisor + * @state: Service VM GPA of buffer of P-states and C-states + */ +static inline long hcall_get_cpu_state(u64 cmd, u64 state) +{ + return acrn_hypercall2(HC_PM_GET_CPU_STATE, cmd, state); +} + +#endif /* __ACRN_HSM_HYPERCALL_H */ diff --git a/drivers/virt/acrn/ioeventfd.c b/drivers/virt/acrn/ioeventfd.c new file mode 100644 index 000000000000..ac4037e9f947 --- /dev/null +++ b/drivers/virt/acrn/ioeventfd.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ACRN HSM eventfd - use eventfd objects to signal expected I/O requests + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * + * Authors: + * Shuo Liu <shuo.a.liu@intel.com> + * Yakui Zhao <yakui.zhao@intel.com> + */ + +#include <linux/eventfd.h> +#include <linux/slab.h> + +#include "acrn_drv.h" + +/** + * struct hsm_ioeventfd - Properties of HSM ioeventfd + * @list: Entry within &acrn_vm.ioeventfds of ioeventfds of a VM + * @eventfd: Eventfd of the HSM ioeventfd + * @addr: Address of I/O range + * @data: Data for matching + * @length: Length of I/O range + * @type: Type of I/O range (ACRN_IOREQ_TYPE_MMIO/ACRN_IOREQ_TYPE_PORTIO) + * @wildcard: Data matching or not + */ +struct hsm_ioeventfd { + struct list_head list; + struct eventfd_ctx *eventfd; + u64 addr; + u64 data; + int length; + int type; + bool wildcard; +}; + +static inline int ioreq_type_from_flags(int flags) +{ + return flags & ACRN_IOEVENTFD_FLAG_PIO ? + ACRN_IOREQ_TYPE_PORTIO : ACRN_IOREQ_TYPE_MMIO; +} + +static void acrn_ioeventfd_shutdown(struct acrn_vm *vm, struct hsm_ioeventfd *p) +{ + lockdep_assert_held(&vm->ioeventfds_lock); + + eventfd_ctx_put(p->eventfd); + list_del(&p->list); + kfree(p); +} + +static bool hsm_ioeventfd_is_conflict(struct acrn_vm *vm, + struct hsm_ioeventfd *ioeventfd) +{ + struct hsm_ioeventfd *p; + + lockdep_assert_held(&vm->ioeventfds_lock); + + /* Either one is wildcard, the data matching will be skipped. */ + list_for_each_entry(p, &vm->ioeventfds, list) + if (p->eventfd == ioeventfd->eventfd && + p->addr == ioeventfd->addr && + p->type == ioeventfd->type && + (p->wildcard || ioeventfd->wildcard || + p->data == ioeventfd->data)) + return true; + + return false; +} + +/* + * Assign an eventfd to a VM and create a HSM ioeventfd associated with the + * eventfd. The properties of the HSM ioeventfd are built from a &struct + * acrn_ioeventfd. + */ +static int acrn_ioeventfd_assign(struct acrn_vm *vm, + struct acrn_ioeventfd *args) +{ + struct eventfd_ctx *eventfd; + struct hsm_ioeventfd *p; + int ret; + + /* Check for range overflow */ + if (args->addr + args->len < args->addr) + return -EINVAL; + + /* + * Currently, acrn_ioeventfd is used to support vhost. 1,2,4,8 width + * accesses can cover vhost's requirements. + */ + if (!(args->len == 1 || args->len == 2 || + args->len == 4 || args->len == 8)) + return -EINVAL; + + eventfd = eventfd_ctx_fdget(args->fd); + if (IS_ERR(eventfd)) + return PTR_ERR(eventfd); + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) { + ret = -ENOMEM; + goto fail; + } + + INIT_LIST_HEAD(&p->list); + p->addr = args->addr; + p->length = args->len; + p->eventfd = eventfd; + p->type = ioreq_type_from_flags(args->flags); + + /* + * ACRN_IOEVENTFD_FLAG_DATAMATCH flag is set in virtio 1.0 support, the + * writing of notification register of each virtqueue may trigger the + * notification. There is no data matching requirement. + */ + if (args->flags & ACRN_IOEVENTFD_FLAG_DATAMATCH) + p->data = args->data; + else + p->wildcard = true; + + mutex_lock(&vm->ioeventfds_lock); + + if (hsm_ioeventfd_is_conflict(vm, p)) { + ret = -EEXIST; + goto unlock_fail; + } + + /* register the I/O range into ioreq client */ + ret = acrn_ioreq_range_add(vm->ioeventfd_client, p->type, + p->addr, p->addr + p->length - 1); + if (ret < 0) + goto unlock_fail; + + list_add_tail(&p->list, &vm->ioeventfds); + mutex_unlock(&vm->ioeventfds_lock); + + return 0; + +unlock_fail: + mutex_unlock(&vm->ioeventfds_lock); + kfree(p); +fail: + eventfd_ctx_put(eventfd); + return ret; +} + +static int acrn_ioeventfd_deassign(struct acrn_vm *vm, + struct acrn_ioeventfd *args) +{ + struct hsm_ioeventfd *p; + struct eventfd_ctx *eventfd; + + eventfd = eventfd_ctx_fdget(args->fd); + if (IS_ERR(eventfd)) + return PTR_ERR(eventfd); + + mutex_lock(&vm->ioeventfds_lock); + list_for_each_entry(p, &vm->ioeventfds, list) { + if (p->eventfd != eventfd) + continue; + + acrn_ioreq_range_del(vm->ioeventfd_client, p->type, + p->addr, p->addr + p->length - 1); + acrn_ioeventfd_shutdown(vm, p); + break; + } + mutex_unlock(&vm->ioeventfds_lock); + + eventfd_ctx_put(eventfd); + return 0; +} + +static struct hsm_ioeventfd *hsm_ioeventfd_match(struct acrn_vm *vm, u64 addr, + u64 data, int len, int type) +{ + struct hsm_ioeventfd *p = NULL; + + lockdep_assert_held(&vm->ioeventfds_lock); + + list_for_each_entry(p, &vm->ioeventfds, list) { + if (p->type == type && p->addr == addr && p->length >= len && + (p->wildcard || p->data == data)) + return p; + } + + return NULL; +} + +static int acrn_ioeventfd_handler(struct acrn_ioreq_client *client, + struct acrn_io_request *req) +{ + struct hsm_ioeventfd *p; + u64 addr, val; + int size; + + if (req->type == ACRN_IOREQ_TYPE_MMIO) { + /* + * I/O requests are dispatched by range check only, so a + * acrn_ioreq_client need process both READ and WRITE accesses + * of same range. READ accesses are safe to be ignored here + * because virtio PCI devices write the notify registers for + * notification. + */ + if (req->reqs.mmio_request.direction == ACRN_IOREQ_DIR_READ) { + /* reading does nothing and return 0 */ + req->reqs.mmio_request.value = 0; + return 0; + } + addr = req->reqs.mmio_request.address; + size = req->reqs.mmio_request.size; + val = req->reqs.mmio_request.value; + } else { + if (req->reqs.pio_request.direction == ACRN_IOREQ_DIR_READ) { + /* reading does nothing and return 0 */ + req->reqs.pio_request.value = 0; + return 0; + } + addr = req->reqs.pio_request.address; + size = req->reqs.pio_request.size; + val = req->reqs.pio_request.value; + } + + mutex_lock(&client->vm->ioeventfds_lock); + p = hsm_ioeventfd_match(client->vm, addr, val, size, req->type); + if (p) + eventfd_signal(p->eventfd, 1); + mutex_unlock(&client->vm->ioeventfds_lock); + + return 0; +} + +int acrn_ioeventfd_config(struct acrn_vm *vm, struct acrn_ioeventfd *args) +{ + int ret; + + if (args->flags & ACRN_IOEVENTFD_FLAG_DEASSIGN) + ret = acrn_ioeventfd_deassign(vm, args); + else + ret = acrn_ioeventfd_assign(vm, args); + + return ret; +} + +int acrn_ioeventfd_init(struct acrn_vm *vm) +{ + char name[ACRN_NAME_LEN]; + + mutex_init(&vm->ioeventfds_lock); + INIT_LIST_HEAD(&vm->ioeventfds); + snprintf(name, sizeof(name), "ioeventfd-%u", vm->vmid); + vm->ioeventfd_client = acrn_ioreq_client_create(vm, + acrn_ioeventfd_handler, + NULL, false, name); + if (!vm->ioeventfd_client) { + dev_err(acrn_dev.this_device, "Failed to create ioeventfd ioreq client!\n"); + return -EINVAL; + } + + dev_dbg(acrn_dev.this_device, "VM %u ioeventfd init.\n", vm->vmid); + return 0; +} + +void acrn_ioeventfd_deinit(struct acrn_vm *vm) +{ + struct hsm_ioeventfd *p, *next; + + dev_dbg(acrn_dev.this_device, "VM %u ioeventfd deinit.\n", vm->vmid); + acrn_ioreq_client_destroy(vm->ioeventfd_client); + mutex_lock(&vm->ioeventfds_lock); + list_for_each_entry_safe(p, next, &vm->ioeventfds, list) + acrn_ioeventfd_shutdown(vm, p); + mutex_unlock(&vm->ioeventfds_lock); +} diff --git a/drivers/virt/acrn/ioreq.c b/drivers/virt/acrn/ioreq.c new file mode 100644 index 000000000000..80b2e3f0e276 --- /dev/null +++ b/drivers/virt/acrn/ioreq.c @@ -0,0 +1,657 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ACRN_HSM: Handle I/O requests + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * + * Authors: + * Jason Chen CJ <jason.cj.chen@intel.com> + * Fengwei Yin <fengwei.yin@intel.com> + */ + +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kthread.h> +#include <linux/mm.h> +#include <linux/slab.h> + +#include <asm/acrn.h> + +#include "acrn_drv.h" + +static void ioreq_pause(void); +static void ioreq_resume(void); + +static void ioreq_dispatcher(struct work_struct *work); +static struct workqueue_struct *ioreq_wq; +static DECLARE_WORK(ioreq_work, ioreq_dispatcher); + +static inline bool has_pending_request(struct acrn_ioreq_client *client) +{ + return !bitmap_empty(client->ioreqs_map, ACRN_IO_REQUEST_MAX); +} + +static inline bool is_destroying(struct acrn_ioreq_client *client) +{ + return test_bit(ACRN_IOREQ_CLIENT_DESTROYING, &client->flags); +} + +static int ioreq_complete_request(struct acrn_vm *vm, u16 vcpu, + struct acrn_io_request *acrn_req) +{ + bool polling_mode; + int ret = 0; + + polling_mode = acrn_req->completion_polling; + /* Add barrier() to make sure the writes are done before completion */ + smp_store_release(&acrn_req->processed, ACRN_IOREQ_STATE_COMPLETE); + + /* + * To fulfill the requirement of real-time in several industry + * scenarios, like automotive, ACRN can run under the partition mode, + * in which User VMs and Service VM are bound to dedicated CPU cores. + * Polling mode of handling the I/O request is introduced to achieve a + * faster I/O request handling. In polling mode, the hypervisor polls + * I/O request's completion. Once an I/O request is marked as + * ACRN_IOREQ_STATE_COMPLETE, hypervisor resumes from the polling point + * to continue the I/O request flow. Thus, the completion notification + * from HSM of I/O request is not needed. Please note, + * completion_polling needs to be read before the I/O request being + * marked as ACRN_IOREQ_STATE_COMPLETE to avoid racing with the + * hypervisor. + */ + if (!polling_mode) { + ret = hcall_notify_req_finish(vm->vmid, vcpu); + if (ret < 0) + dev_err(acrn_dev.this_device, + "Notify I/O request finished failed!\n"); + } + + return ret; +} + +static int acrn_ioreq_complete_request(struct acrn_ioreq_client *client, + u16 vcpu, + struct acrn_io_request *acrn_req) +{ + int ret; + + if (vcpu >= client->vm->vcpu_num) + return -EINVAL; + + clear_bit(vcpu, client->ioreqs_map); + if (!acrn_req) { + acrn_req = (struct acrn_io_request *)client->vm->ioreq_buf; + acrn_req += vcpu; + } + + ret = ioreq_complete_request(client->vm, vcpu, acrn_req); + + return ret; +} + +int acrn_ioreq_request_default_complete(struct acrn_vm *vm, u16 vcpu) +{ + int ret = 0; + + spin_lock_bh(&vm->ioreq_clients_lock); + if (vm->default_client) + ret = acrn_ioreq_complete_request(vm->default_client, + vcpu, NULL); + spin_unlock_bh(&vm->ioreq_clients_lock); + + return ret; +} + +/** + * acrn_ioreq_range_add() - Add an iorange monitored by an ioreq client + * @client: The ioreq client + * @type: Type (ACRN_IOREQ_TYPE_MMIO or ACRN_IOREQ_TYPE_PORTIO) + * @start: Start address of iorange + * @end: End address of iorange + * + * Return: 0 on success, <0 on error + */ +int acrn_ioreq_range_add(struct acrn_ioreq_client *client, + u32 type, u64 start, u64 end) +{ + struct acrn_ioreq_range *range; + + if (end < start) { + dev_err(acrn_dev.this_device, + "Invalid IO range [0x%llx,0x%llx]\n", start, end); + return -EINVAL; + } + + range = kzalloc(sizeof(*range), GFP_KERNEL); + if (!range) + return -ENOMEM; + + range->type = type; + range->start = start; + range->end = end; + + write_lock_bh(&client->range_lock); + list_add(&range->list, &client->range_list); + write_unlock_bh(&client->range_lock); + + return 0; +} + +/** + * acrn_ioreq_range_del() - Del an iorange monitored by an ioreq client + * @client: The ioreq client + * @type: Type (ACRN_IOREQ_TYPE_MMIO or ACRN_IOREQ_TYPE_PORTIO) + * @start: Start address of iorange + * @end: End address of iorange + */ +void acrn_ioreq_range_del(struct acrn_ioreq_client *client, + u32 type, u64 start, u64 end) +{ + struct acrn_ioreq_range *range; + + write_lock_bh(&client->range_lock); + list_for_each_entry(range, &client->range_list, list) { + if (type == range->type && + start == range->start && + end == range->end) { + list_del(&range->list); + kfree(range); + break; + } + } + write_unlock_bh(&client->range_lock); +} + +/* + * ioreq_task() is the execution entity of handler thread of an I/O client. + * The handler callback of the I/O client is called within the handler thread. + */ +static int ioreq_task(void *data) +{ + struct acrn_ioreq_client *client = data; + struct acrn_io_request *req; + unsigned long *ioreqs_map; + int vcpu, ret; + + /* + * Lockless access to ioreqs_map is safe, because + * 1) set_bit() and clear_bit() are atomic operations. + * 2) I/O requests arrives serialized. The access flow of ioreqs_map is: + * set_bit() - in ioreq_work handler + * Handler callback handles corresponding I/O request + * clear_bit() - in handler thread (include ACRN userspace) + * Mark corresponding I/O request completed + * Loop again if a new I/O request occurs + */ + ioreqs_map = client->ioreqs_map; + while (!kthread_should_stop()) { + acrn_ioreq_client_wait(client); + while (has_pending_request(client)) { + vcpu = find_first_bit(ioreqs_map, client->vm->vcpu_num); + req = client->vm->ioreq_buf->req_slot + vcpu; + ret = client->handler(client, req); + if (ret < 0) { + dev_err(acrn_dev.this_device, + "IO handle failure: %d\n", ret); + break; + } + acrn_ioreq_complete_request(client, vcpu, req); + } + } + + return 0; +} + +/* + * For the non-default I/O clients, give them chance to complete the current + * I/O requests if there are any. For the default I/O client, it is safe to + * clear all pending I/O requests because the clearing request is from ACRN + * userspace. + */ +void acrn_ioreq_request_clear(struct acrn_vm *vm) +{ + struct acrn_ioreq_client *client; + bool has_pending = false; + unsigned long vcpu; + int retry = 10; + + /* + * IO requests of this VM will be completed directly in + * acrn_ioreq_dispatch if ACRN_VM_FLAG_CLEARING_IOREQ flag is set. + */ + set_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags); + + /* + * acrn_ioreq_request_clear is only called in VM reset case. Simply + * wait 100ms in total for the IO requests' completion. + */ + do { + spin_lock_bh(&vm->ioreq_clients_lock); + list_for_each_entry(client, &vm->ioreq_clients, list) { + has_pending = has_pending_request(client); + if (has_pending) + break; + } + spin_unlock_bh(&vm->ioreq_clients_lock); + + if (has_pending) + schedule_timeout_interruptible(HZ / 100); + } while (has_pending && --retry > 0); + if (retry == 0) + dev_warn(acrn_dev.this_device, + "%s cannot flush pending request!\n", client->name); + + /* Clear all ioreqs belonging to the default client */ + spin_lock_bh(&vm->ioreq_clients_lock); + client = vm->default_client; + if (client) { + vcpu = find_next_bit(client->ioreqs_map, + ACRN_IO_REQUEST_MAX, 0); + while (vcpu < ACRN_IO_REQUEST_MAX) { + acrn_ioreq_complete_request(client, vcpu, NULL); + vcpu = find_next_bit(client->ioreqs_map, + ACRN_IO_REQUEST_MAX, vcpu + 1); + } + } + spin_unlock_bh(&vm->ioreq_clients_lock); + + /* Clear ACRN_VM_FLAG_CLEARING_IOREQ flag after the clearing */ + clear_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags); +} + +int acrn_ioreq_client_wait(struct acrn_ioreq_client *client) +{ + if (client->is_default) { + /* + * In the default client, a user space thread waits on the + * waitqueue. The is_destroying() check is used to notify user + * space the client is going to be destroyed. + */ + wait_event_interruptible(client->wq, + has_pending_request(client) || + is_destroying(client)); + if (is_destroying(client)) + return -ENODEV; + } else { + wait_event_interruptible(client->wq, + has_pending_request(client) || + kthread_should_stop()); + } + + return 0; +} + +static bool is_cfg_addr(struct acrn_io_request *req) +{ + return ((req->type == ACRN_IOREQ_TYPE_PORTIO) && + (req->reqs.pio_request.address == 0xcf8)); +} + +static bool is_cfg_data(struct acrn_io_request *req) +{ + return ((req->type == ACRN_IOREQ_TYPE_PORTIO) && + ((req->reqs.pio_request.address >= 0xcfc) && + (req->reqs.pio_request.address < (0xcfc + 4)))); +} + +/* The low 8-bit of supported pci_reg addr.*/ +#define PCI_LOWREG_MASK 0xFC +/* The high 4-bit of supported pci_reg addr */ +#define PCI_HIGHREG_MASK 0xF00 +/* Max number of supported functions */ +#define PCI_FUNCMAX 7 +/* Max number of supported slots */ +#define PCI_SLOTMAX 31 +/* Max number of supported buses */ +#define PCI_BUSMAX 255 +#define CONF1_ENABLE 0x80000000UL +/* + * A PCI configuration space access via PIO 0xCF8 and 0xCFC normally has two + * following steps: + * 1) writes address into 0xCF8 port + * 2) accesses data in/from 0xCFC + * This function combines such paired PCI configuration space I/O requests into + * one ACRN_IOREQ_TYPE_PCICFG type I/O request and continues the processing. + */ +static bool handle_cf8cfc(struct acrn_vm *vm, + struct acrn_io_request *req, u16 vcpu) +{ + int offset, pci_cfg_addr, pci_reg; + bool is_handled = false; + + if (is_cfg_addr(req)) { + WARN_ON(req->reqs.pio_request.size != 4); + if (req->reqs.pio_request.direction == ACRN_IOREQ_DIR_WRITE) + vm->pci_conf_addr = req->reqs.pio_request.value; + else + req->reqs.pio_request.value = vm->pci_conf_addr; + is_handled = true; + } else if (is_cfg_data(req)) { + if (!(vm->pci_conf_addr & CONF1_ENABLE)) { + if (req->reqs.pio_request.direction == + ACRN_IOREQ_DIR_READ) + req->reqs.pio_request.value = 0xffffffff; + is_handled = true; + } else { + offset = req->reqs.pio_request.address - 0xcfc; + + req->type = ACRN_IOREQ_TYPE_PCICFG; + pci_cfg_addr = vm->pci_conf_addr; + req->reqs.pci_request.bus = + (pci_cfg_addr >> 16) & PCI_BUSMAX; + req->reqs.pci_request.dev = + (pci_cfg_addr >> 11) & PCI_SLOTMAX; + req->reqs.pci_request.func = + (pci_cfg_addr >> 8) & PCI_FUNCMAX; + pci_reg = (pci_cfg_addr & PCI_LOWREG_MASK) + + ((pci_cfg_addr >> 16) & PCI_HIGHREG_MASK); + req->reqs.pci_request.reg = pci_reg + offset; + } + } + + if (is_handled) + ioreq_complete_request(vm, vcpu, req); + + return is_handled; +} + +static bool in_range(struct acrn_ioreq_range *range, + struct acrn_io_request *req) +{ + bool ret = false; + + if (range->type == req->type) { + switch (req->type) { + case ACRN_IOREQ_TYPE_MMIO: + if (req->reqs.mmio_request.address >= range->start && + (req->reqs.mmio_request.address + + req->reqs.mmio_request.size - 1) <= range->end) + ret = true; + break; + case ACRN_IOREQ_TYPE_PORTIO: + if (req->reqs.pio_request.address >= range->start && + (req->reqs.pio_request.address + + req->reqs.pio_request.size - 1) <= range->end) + ret = true; + break; + default: + break; + } + } + + return ret; +} + +static struct acrn_ioreq_client *find_ioreq_client(struct acrn_vm *vm, + struct acrn_io_request *req) +{ + struct acrn_ioreq_client *client, *found = NULL; + struct acrn_ioreq_range *range; + + lockdep_assert_held(&vm->ioreq_clients_lock); + + list_for_each_entry(client, &vm->ioreq_clients, list) { + read_lock_bh(&client->range_lock); + list_for_each_entry(range, &client->range_list, list) { + if (in_range(range, req)) { + found = client; + break; + } + } + read_unlock_bh(&client->range_lock); + if (found) + break; + } + return found ? found : vm->default_client; +} + +/** + * acrn_ioreq_client_create() - Create an ioreq client + * @vm: The VM that this client belongs to + * @handler: The ioreq_handler of ioreq client acrn_hsm will create a kernel + * thread and call the handler to handle I/O requests. + * @priv: Private data for the handler + * @is_default: If it is the default client + * @name: The name of ioreq client + * + * Return: acrn_ioreq_client pointer on success, NULL on error + */ +struct acrn_ioreq_client *acrn_ioreq_client_create(struct acrn_vm *vm, + ioreq_handler_t handler, + void *priv, bool is_default, + const char *name) +{ + struct acrn_ioreq_client *client; + + if (!handler && !is_default) { + dev_dbg(acrn_dev.this_device, + "Cannot create non-default client w/o handler!\n"); + return NULL; + } + client = kzalloc(sizeof(*client), GFP_KERNEL); + if (!client) + return NULL; + + client->handler = handler; + client->vm = vm; + client->priv = priv; + client->is_default = is_default; + if (name) + strncpy(client->name, name, sizeof(client->name) - 1); + rwlock_init(&client->range_lock); + INIT_LIST_HEAD(&client->range_list); + init_waitqueue_head(&client->wq); + + if (client->handler) { + client->thread = kthread_run(ioreq_task, client, "VM%u-%s", + client->vm->vmid, client->name); + if (IS_ERR(client->thread)) { + kfree(client); + return NULL; + } + } + + spin_lock_bh(&vm->ioreq_clients_lock); + if (is_default) + vm->default_client = client; + else + list_add(&client->list, &vm->ioreq_clients); + spin_unlock_bh(&vm->ioreq_clients_lock); + + dev_dbg(acrn_dev.this_device, "Created ioreq client %s.\n", name); + return client; +} + +/** + * acrn_ioreq_client_destroy() - Destroy an ioreq client + * @client: The ioreq client + */ +void acrn_ioreq_client_destroy(struct acrn_ioreq_client *client) +{ + struct acrn_ioreq_range *range, *next; + struct acrn_vm *vm = client->vm; + + dev_dbg(acrn_dev.this_device, + "Destroy ioreq client %s.\n", client->name); + ioreq_pause(); + set_bit(ACRN_IOREQ_CLIENT_DESTROYING, &client->flags); + if (client->is_default) + wake_up_interruptible(&client->wq); + else + kthread_stop(client->thread); + + spin_lock_bh(&vm->ioreq_clients_lock); + if (client->is_default) + vm->default_client = NULL; + else + list_del(&client->list); + spin_unlock_bh(&vm->ioreq_clients_lock); + + write_lock_bh(&client->range_lock); + list_for_each_entry_safe(range, next, &client->range_list, list) { + list_del(&range->list); + kfree(range); + } + write_unlock_bh(&client->range_lock); + kfree(client); + + ioreq_resume(); +} + +static int acrn_ioreq_dispatch(struct acrn_vm *vm) +{ + struct acrn_ioreq_client *client; + struct acrn_io_request *req; + int i; + + for (i = 0; i < vm->vcpu_num; i++) { + req = vm->ioreq_buf->req_slot + i; + + /* barrier the read of processed of acrn_io_request */ + if (smp_load_acquire(&req->processed) == + ACRN_IOREQ_STATE_PENDING) { + /* Complete the IO request directly in clearing stage */ + if (test_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags)) { + ioreq_complete_request(vm, i, req); + continue; + } + if (handle_cf8cfc(vm, req, i)) + continue; + + spin_lock_bh(&vm->ioreq_clients_lock); + client = find_ioreq_client(vm, req); + if (!client) { + dev_err(acrn_dev.this_device, + "Failed to find ioreq client!\n"); + spin_unlock_bh(&vm->ioreq_clients_lock); + return -EINVAL; + } + if (!client->is_default) + req->kernel_handled = 1; + else + req->kernel_handled = 0; + /* + * Add barrier() to make sure the writes are done + * before setting ACRN_IOREQ_STATE_PROCESSING + */ + smp_store_release(&req->processed, + ACRN_IOREQ_STATE_PROCESSING); + set_bit(i, client->ioreqs_map); + wake_up_interruptible(&client->wq); + spin_unlock_bh(&vm->ioreq_clients_lock); + } + } + + return 0; +} + +static void ioreq_dispatcher(struct work_struct *work) +{ + struct acrn_vm *vm; + + read_lock(&acrn_vm_list_lock); + list_for_each_entry(vm, &acrn_vm_list, list) { + if (!vm->ioreq_buf) + break; + acrn_ioreq_dispatch(vm); + } + read_unlock(&acrn_vm_list_lock); +} + +static void ioreq_intr_handler(void) +{ + queue_work(ioreq_wq, &ioreq_work); +} + +static void ioreq_pause(void) +{ + /* Flush and unarm the handler to ensure no I/O requests pending */ + acrn_remove_intr_handler(); + drain_workqueue(ioreq_wq); +} + +static void ioreq_resume(void) +{ + /* Schedule after enabling in case other clients miss interrupt */ + acrn_setup_intr_handler(ioreq_intr_handler); + queue_work(ioreq_wq, &ioreq_work); +} + +int acrn_ioreq_intr_setup(void) +{ + acrn_setup_intr_handler(ioreq_intr_handler); + ioreq_wq = alloc_workqueue("ioreq_wq", + WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_UNBOUND, 1); + if (!ioreq_wq) { + dev_err(acrn_dev.this_device, "Failed to alloc workqueue!\n"); + acrn_remove_intr_handler(); + return -ENOMEM; + } + return 0; +} + +void acrn_ioreq_intr_remove(void) +{ + if (ioreq_wq) + destroy_workqueue(ioreq_wq); + acrn_remove_intr_handler(); +} + +int acrn_ioreq_init(struct acrn_vm *vm, u64 buf_vma) +{ + struct acrn_ioreq_buffer *set_buffer; + struct page *page; + int ret; + + if (vm->ioreq_buf) + return -EEXIST; + + set_buffer = kzalloc(sizeof(*set_buffer), GFP_KERNEL); + if (!set_buffer) + return -ENOMEM; + + ret = pin_user_pages_fast(buf_vma, 1, + FOLL_WRITE | FOLL_LONGTERM, &page); + if (unlikely(ret != 1) || !page) { + dev_err(acrn_dev.this_device, "Failed to pin ioreq page!\n"); + ret = -EFAULT; + goto free_buf; + } + + vm->ioreq_buf = page_address(page); + vm->ioreq_page = page; + set_buffer->ioreq_buf = page_to_phys(page); + ret = hcall_set_ioreq_buffer(vm->vmid, virt_to_phys(set_buffer)); + if (ret < 0) { + dev_err(acrn_dev.this_device, "Failed to init ioreq buffer!\n"); + unpin_user_page(page); + vm->ioreq_buf = NULL; + goto free_buf; + } + + dev_dbg(acrn_dev.this_device, + "Init ioreq buffer %pK!\n", vm->ioreq_buf); + ret = 0; +free_buf: + kfree(set_buffer); + return ret; +} + +void acrn_ioreq_deinit(struct acrn_vm *vm) +{ + struct acrn_ioreq_client *client, *next; + + dev_dbg(acrn_dev.this_device, + "Deinit ioreq buffer %pK!\n", vm->ioreq_buf); + /* Destroy all clients belonging to this VM */ + list_for_each_entry_safe(client, next, &vm->ioreq_clients, list) + acrn_ioreq_client_destroy(client); + if (vm->default_client) + acrn_ioreq_client_destroy(vm->default_client); + + if (vm->ioreq_buf && vm->ioreq_page) { + unpin_user_page(vm->ioreq_page); + vm->ioreq_buf = NULL; + } +} diff --git a/drivers/virt/acrn/irqfd.c b/drivers/virt/acrn/irqfd.c new file mode 100644 index 000000000000..a8766d528e29 --- /dev/null +++ b/drivers/virt/acrn/irqfd.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ACRN HSM irqfd: use eventfd objects to inject virtual interrupts + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * + * Authors: + * Shuo Liu <shuo.a.liu@intel.com> + * Yakui Zhao <yakui.zhao@intel.com> + */ + +#include <linux/eventfd.h> +#include <linux/file.h> +#include <linux/poll.h> +#include <linux/slab.h> + +#include "acrn_drv.h" + +static LIST_HEAD(acrn_irqfd_clients); +static DEFINE_MUTEX(acrn_irqfds_mutex); + +/** + * struct hsm_irqfd - Properties of HSM irqfd + * @vm: Associated VM pointer + * @wait: Entry of wait-queue + * @shutdown: Async shutdown work + * @eventfd: Associated eventfd + * @list: Entry within &acrn_vm.irqfds of irqfds of a VM + * @pt: Structure for select/poll on the associated eventfd + * @msi: MSI data + */ +struct hsm_irqfd { + struct acrn_vm *vm; + wait_queue_entry_t wait; + struct work_struct shutdown; + struct eventfd_ctx *eventfd; + struct list_head list; + poll_table pt; + struct acrn_msi_entry msi; +}; + +static void acrn_irqfd_inject(struct hsm_irqfd *irqfd) +{ + struct acrn_vm *vm = irqfd->vm; + + acrn_msi_inject(vm, irqfd->msi.msi_addr, + irqfd->msi.msi_data); +} + +static void hsm_irqfd_shutdown(struct hsm_irqfd *irqfd) +{ + u64 cnt; + + lockdep_assert_held(&irqfd->vm->irqfds_lock); + + /* remove from wait queue */ + list_del_init(&irqfd->list); + eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt); + eventfd_ctx_put(irqfd->eventfd); + kfree(irqfd); +} + +static void hsm_irqfd_shutdown_work(struct work_struct *work) +{ + struct hsm_irqfd *irqfd; + struct acrn_vm *vm; + + irqfd = container_of(work, struct hsm_irqfd, shutdown); + vm = irqfd->vm; + mutex_lock(&vm->irqfds_lock); + if (!list_empty(&irqfd->list)) + hsm_irqfd_shutdown(irqfd); + mutex_unlock(&vm->irqfds_lock); +} + +/* Called with wqh->lock held and interrupts disabled */ +static int hsm_irqfd_wakeup(wait_queue_entry_t *wait, unsigned int mode, + int sync, void *key) +{ + unsigned long poll_bits = (unsigned long)key; + struct hsm_irqfd *irqfd; + struct acrn_vm *vm; + + irqfd = container_of(wait, struct hsm_irqfd, wait); + vm = irqfd->vm; + if (poll_bits & POLLIN) + /* An event has been signaled, inject an interrupt */ + acrn_irqfd_inject(irqfd); + + if (poll_bits & POLLHUP) + /* Do shutdown work in thread to hold wqh->lock */ + queue_work(vm->irqfd_wq, &irqfd->shutdown); + + return 0; +} + +static void hsm_irqfd_poll_func(struct file *file, wait_queue_head_t *wqh, + poll_table *pt) +{ + struct hsm_irqfd *irqfd; + + irqfd = container_of(pt, struct hsm_irqfd, pt); + add_wait_queue(wqh, &irqfd->wait); +} + +/* + * Assign an eventfd to a VM and create a HSM irqfd associated with the + * eventfd. The properties of the HSM irqfd are built from a &struct + * acrn_irqfd. + */ +static int acrn_irqfd_assign(struct acrn_vm *vm, struct acrn_irqfd *args) +{ + struct eventfd_ctx *eventfd = NULL; + struct hsm_irqfd *irqfd, *tmp; + unsigned int events; + struct fd f; + int ret = 0; + + irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL); + if (!irqfd) + return -ENOMEM; + + irqfd->vm = vm; + memcpy(&irqfd->msi, &args->msi, sizeof(args->msi)); + INIT_LIST_HEAD(&irqfd->list); + INIT_WORK(&irqfd->shutdown, hsm_irqfd_shutdown_work); + + f = fdget(args->fd); + if (!f.file) { + ret = -EBADF; + goto out; + } + + eventfd = eventfd_ctx_fileget(f.file); + if (IS_ERR(eventfd)) { + ret = PTR_ERR(eventfd); + goto fail; + } + + irqfd->eventfd = eventfd; + + /* + * Install custom wake-up handling to be notified whenever underlying + * eventfd is signaled. + */ + init_waitqueue_func_entry(&irqfd->wait, hsm_irqfd_wakeup); + init_poll_funcptr(&irqfd->pt, hsm_irqfd_poll_func); + + mutex_lock(&vm->irqfds_lock); + list_for_each_entry(tmp, &vm->irqfds, list) { + if (irqfd->eventfd != tmp->eventfd) + continue; + ret = -EBUSY; + mutex_unlock(&vm->irqfds_lock); + goto fail; + } + list_add_tail(&irqfd->list, &vm->irqfds); + mutex_unlock(&vm->irqfds_lock); + + /* Check the pending event in this stage */ + events = f.file->f_op->poll(f.file, &irqfd->pt); + + if (events & POLLIN) + acrn_irqfd_inject(irqfd); + + fdput(f); + return 0; +fail: + if (eventfd && !IS_ERR(eventfd)) + eventfd_ctx_put(eventfd); + + fdput(f); +out: + kfree(irqfd); + return ret; +} + +static int acrn_irqfd_deassign(struct acrn_vm *vm, + struct acrn_irqfd *args) +{ + struct hsm_irqfd *irqfd, *tmp; + struct eventfd_ctx *eventfd; + + eventfd = eventfd_ctx_fdget(args->fd); + if (IS_ERR(eventfd)) + return PTR_ERR(eventfd); + + mutex_lock(&vm->irqfds_lock); + list_for_each_entry_safe(irqfd, tmp, &vm->irqfds, list) { + if (irqfd->eventfd == eventfd) { + hsm_irqfd_shutdown(irqfd); + break; + } + } + mutex_unlock(&vm->irqfds_lock); + eventfd_ctx_put(eventfd); + + return 0; +} + +int acrn_irqfd_config(struct acrn_vm *vm, struct acrn_irqfd *args) +{ + int ret; + + if (args->flags & ACRN_IRQFD_FLAG_DEASSIGN) + ret = acrn_irqfd_deassign(vm, args); + else + ret = acrn_irqfd_assign(vm, args); + + return ret; +} + +int acrn_irqfd_init(struct acrn_vm *vm) +{ + INIT_LIST_HEAD(&vm->irqfds); + mutex_init(&vm->irqfds_lock); + vm->irqfd_wq = alloc_workqueue("acrn_irqfd-%u", 0, 0, vm->vmid); + if (!vm->irqfd_wq) + return -ENOMEM; + + dev_dbg(acrn_dev.this_device, "VM %u irqfd init.\n", vm->vmid); + return 0; +} + +void acrn_irqfd_deinit(struct acrn_vm *vm) +{ + struct hsm_irqfd *irqfd, *next; + + dev_dbg(acrn_dev.this_device, "VM %u irqfd deinit.\n", vm->vmid); + destroy_workqueue(vm->irqfd_wq); + mutex_lock(&vm->irqfds_lock); + list_for_each_entry_safe(irqfd, next, &vm->irqfds, list) + hsm_irqfd_shutdown(irqfd); + mutex_unlock(&vm->irqfds_lock); +} diff --git a/drivers/virt/acrn/mm.c b/drivers/virt/acrn/mm.c new file mode 100644 index 000000000000..c4f2e15c8a2b --- /dev/null +++ b/drivers/virt/acrn/mm.c @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ACRN: Memory mapping management + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * + * Authors: + * Fei Li <lei1.li@intel.com> + * Shuo Liu <shuo.a.liu@intel.com> + */ + +#include <linux/io.h> +#include <linux/mm.h> +#include <linux/slab.h> + +#include "acrn_drv.h" + +static int modify_region(struct acrn_vm *vm, struct vm_memory_region_op *region) +{ + struct vm_memory_region_batch *regions; + int ret; + + regions = kzalloc(sizeof(*regions), GFP_KERNEL); + if (!regions) + return -ENOMEM; + + regions->vmid = vm->vmid; + regions->regions_num = 1; + regions->regions_gpa = virt_to_phys(region); + + ret = hcall_set_memory_regions(virt_to_phys(regions)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to set memory region for VM[%u]!\n", vm->vmid); + + kfree(regions); + return ret; +} + +/** + * acrn_mm_region_add() - Set up the EPT mapping of a memory region. + * @vm: User VM. + * @user_gpa: A GPA of User VM. + * @service_gpa: A GPA of Service VM. + * @size: Size of the region. + * @mem_type: Combination of ACRN_MEM_TYPE_*. + * @mem_access_right: Combination of ACRN_MEM_ACCESS_*. + * + * Return: 0 on success, <0 on error. + */ +int acrn_mm_region_add(struct acrn_vm *vm, u64 user_gpa, u64 service_gpa, + u64 size, u32 mem_type, u32 mem_access_right) +{ + struct vm_memory_region_op *region; + int ret = 0; + + region = kzalloc(sizeof(*region), GFP_KERNEL); + if (!region) + return -ENOMEM; + + region->type = ACRN_MEM_REGION_ADD; + region->user_vm_pa = user_gpa; + region->service_vm_pa = service_gpa; + region->size = size; + region->attr = ((mem_type & ACRN_MEM_TYPE_MASK) | + (mem_access_right & ACRN_MEM_ACCESS_RIGHT_MASK)); + ret = modify_region(vm, region); + + dev_dbg(acrn_dev.this_device, + "%s: user-GPA[%pK] service-GPA[%pK] size[0x%llx].\n", + __func__, (void *)user_gpa, (void *)service_gpa, size); + kfree(region); + return ret; +} + +/** + * acrn_mm_region_del() - Del the EPT mapping of a memory region. + * @vm: User VM. + * @user_gpa: A GPA of the User VM. + * @size: Size of the region. + * + * Return: 0 on success, <0 for error. + */ +int acrn_mm_region_del(struct acrn_vm *vm, u64 user_gpa, u64 size) +{ + struct vm_memory_region_op *region; + int ret = 0; + + region = kzalloc(sizeof(*region), GFP_KERNEL); + if (!region) + return -ENOMEM; + + region->type = ACRN_MEM_REGION_DEL; + region->user_vm_pa = user_gpa; + region->service_vm_pa = 0UL; + region->size = size; + region->attr = 0U; + + ret = modify_region(vm, region); + + dev_dbg(acrn_dev.this_device, "%s: user-GPA[%pK] size[0x%llx].\n", + __func__, (void *)user_gpa, size); + kfree(region); + return ret; +} + +int acrn_vm_memseg_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) +{ + int ret; + + if (memmap->type == ACRN_MEMMAP_RAM) + return acrn_vm_ram_map(vm, memmap); + + if (memmap->type != ACRN_MEMMAP_MMIO) { + dev_dbg(acrn_dev.this_device, + "Invalid memmap type: %u\n", memmap->type); + return -EINVAL; + } + + ret = acrn_mm_region_add(vm, memmap->user_vm_pa, + memmap->service_vm_pa, memmap->len, + ACRN_MEM_TYPE_UC, memmap->attr); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Add memory region failed, VM[%u]!\n", vm->vmid); + + return ret; +} + +int acrn_vm_memseg_unmap(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) +{ + int ret; + + if (memmap->type != ACRN_MEMMAP_MMIO) { + dev_dbg(acrn_dev.this_device, + "Invalid memmap type: %u\n", memmap->type); + return -EINVAL; + } + + ret = acrn_mm_region_del(vm, memmap->user_vm_pa, memmap->len); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Del memory region failed, VM[%u]!\n", vm->vmid); + + return ret; +} + +/** + * acrn_vm_ram_map() - Create a RAM EPT mapping of User VM. + * @vm: The User VM pointer + * @memmap: Info of the EPT mapping + * + * Return: 0 on success, <0 for error. + */ +int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) +{ + struct vm_memory_region_batch *regions_info; + int nr_pages, i = 0, order, nr_regions = 0; + struct vm_memory_mapping *region_mapping; + struct vm_memory_region_op *vm_region; + struct page **pages = NULL, *page; + void *remap_vaddr; + int ret, pinned; + u64 user_vm_pa; + + if (!vm || !memmap) + return -EINVAL; + + /* Get the page number of the map region */ + nr_pages = memmap->len >> PAGE_SHIFT; + pages = vzalloc(nr_pages * sizeof(struct page *)); + if (!pages) + return -ENOMEM; + + /* Lock the pages of user memory map region */ + pinned = pin_user_pages_fast(memmap->vma_base, + nr_pages, FOLL_WRITE | FOLL_LONGTERM, + pages); + if (pinned < 0) { + ret = pinned; + goto free_pages; + } else if (pinned != nr_pages) { + ret = -EFAULT; + goto put_pages; + } + + /* Create a kernel map for the map region */ + remap_vaddr = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL); + if (!remap_vaddr) { + ret = -ENOMEM; + goto put_pages; + } + + /* Record Service VM va <-> User VM pa mapping */ + mutex_lock(&vm->regions_mapping_lock); + region_mapping = &vm->regions_mapping[vm->regions_mapping_count]; + if (vm->regions_mapping_count < ACRN_MEM_MAPPING_MAX) { + region_mapping->pages = pages; + region_mapping->npages = nr_pages; + region_mapping->size = memmap->len; + region_mapping->service_vm_va = remap_vaddr; + region_mapping->user_vm_pa = memmap->user_vm_pa; + vm->regions_mapping_count++; + } else { + dev_warn(acrn_dev.this_device, + "Run out of memory mapping slots!\n"); + ret = -ENOMEM; + mutex_unlock(&vm->regions_mapping_lock); + goto unmap_no_count; + } + mutex_unlock(&vm->regions_mapping_lock); + + /* Calculate count of vm_memory_region_op */ + while (i < nr_pages) { + page = pages[i]; + VM_BUG_ON_PAGE(PageTail(page), page); + order = compound_order(page); + nr_regions++; + i += 1 << order; + } + + /* Prepare the vm_memory_region_batch */ + regions_info = kzalloc(sizeof(*regions_info) + + sizeof(*vm_region) * nr_regions, + GFP_KERNEL); + if (!regions_info) { + ret = -ENOMEM; + goto unmap_kernel_map; + } + + /* Fill each vm_memory_region_op */ + vm_region = (struct vm_memory_region_op *)(regions_info + 1); + regions_info->vmid = vm->vmid; + regions_info->regions_num = nr_regions; + regions_info->regions_gpa = virt_to_phys(vm_region); + user_vm_pa = memmap->user_vm_pa; + i = 0; + while (i < nr_pages) { + u32 region_size; + + page = pages[i]; + VM_BUG_ON_PAGE(PageTail(page), page); + order = compound_order(page); + region_size = PAGE_SIZE << order; + vm_region->type = ACRN_MEM_REGION_ADD; + vm_region->user_vm_pa = user_vm_pa; + vm_region->service_vm_pa = page_to_phys(page); + vm_region->size = region_size; + vm_region->attr = (ACRN_MEM_TYPE_WB & ACRN_MEM_TYPE_MASK) | + (memmap->attr & ACRN_MEM_ACCESS_RIGHT_MASK); + + vm_region++; + user_vm_pa += region_size; + i += 1 << order; + } + + /* Inform the ACRN Hypervisor to set up EPT mappings */ + ret = hcall_set_memory_regions(virt_to_phys(regions_info)); + if (ret < 0) { + dev_dbg(acrn_dev.this_device, + "Failed to set regions, VM[%u]!\n", vm->vmid); + goto unset_region; + } + kfree(regions_info); + + dev_dbg(acrn_dev.this_device, + "%s: VM[%u] service-GVA[%pK] user-GPA[%pK] size[0x%llx]\n", + __func__, vm->vmid, + remap_vaddr, (void *)memmap->user_vm_pa, memmap->len); + return ret; + +unset_region: + kfree(regions_info); +unmap_kernel_map: + mutex_lock(&vm->regions_mapping_lock); + vm->regions_mapping_count--; + mutex_unlock(&vm->regions_mapping_lock); +unmap_no_count: + vunmap(remap_vaddr); +put_pages: + for (i = 0; i < pinned; i++) + unpin_user_page(pages[i]); +free_pages: + vfree(pages); + return ret; +} + +/** + * acrn_vm_all_ram_unmap() - Destroy a RAM EPT mapping of User VM. + * @vm: The User VM + */ +void acrn_vm_all_ram_unmap(struct acrn_vm *vm) +{ + struct vm_memory_mapping *region_mapping; + int i, j; + + mutex_lock(&vm->regions_mapping_lock); + for (i = 0; i < vm->regions_mapping_count; i++) { + region_mapping = &vm->regions_mapping[i]; + vunmap(region_mapping->service_vm_va); + for (j = 0; j < region_mapping->npages; j++) + unpin_user_page(region_mapping->pages[j]); + vfree(region_mapping->pages); + } + mutex_unlock(&vm->regions_mapping_lock); +} diff --git a/drivers/virt/acrn/vm.c b/drivers/virt/acrn/vm.c new file mode 100644 index 000000000000..7804a2492ad7 --- /dev/null +++ b/drivers/virt/acrn/vm.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ACRN_HSM: Virtual Machine management + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * + * Authors: + * Jason Chen CJ <jason.cj.chen@intel.com> + * Yakui Zhao <yakui.zhao@intel.com> + */ +#include <linux/io.h> +#include <linux/mm.h> +#include <linux/slab.h> + +#include "acrn_drv.h" + +/* List of VMs */ +LIST_HEAD(acrn_vm_list); +/* + * acrn_vm_list is read in a worker thread which dispatch I/O requests and + * is wrote in VM creation ioctl. Use the rwlock mechanism to protect it. + */ +DEFINE_RWLOCK(acrn_vm_list_lock); + +struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, + struct acrn_vm_creation *vm_param) +{ + int ret; + + ret = hcall_create_vm(virt_to_phys(vm_param)); + if (ret < 0 || vm_param->vmid == ACRN_INVALID_VMID) { + dev_err(acrn_dev.this_device, + "Failed to create VM! Error: %d\n", ret); + return NULL; + } + + mutex_init(&vm->regions_mapping_lock); + INIT_LIST_HEAD(&vm->ioreq_clients); + spin_lock_init(&vm->ioreq_clients_lock); + vm->vmid = vm_param->vmid; + vm->vcpu_num = vm_param->vcpu_num; + + if (acrn_ioreq_init(vm, vm_param->ioreq_buf) < 0) { + hcall_destroy_vm(vm_param->vmid); + vm->vmid = ACRN_INVALID_VMID; + return NULL; + } + + write_lock_bh(&acrn_vm_list_lock); + list_add(&vm->list, &acrn_vm_list); + write_unlock_bh(&acrn_vm_list_lock); + + acrn_ioeventfd_init(vm); + acrn_irqfd_init(vm); + dev_dbg(acrn_dev.this_device, "VM %u created.\n", vm->vmid); + return vm; +} + +int acrn_vm_destroy(struct acrn_vm *vm) +{ + int ret; + + if (vm->vmid == ACRN_INVALID_VMID || + test_and_set_bit(ACRN_VM_FLAG_DESTROYED, &vm->flags)) + return 0; + + /* Remove from global VM list */ + write_lock_bh(&acrn_vm_list_lock); + list_del_init(&vm->list); + write_unlock_bh(&acrn_vm_list_lock); + + acrn_ioeventfd_deinit(vm); + acrn_irqfd_deinit(vm); + acrn_ioreq_deinit(vm); + + if (vm->monitor_page) { + put_page(vm->monitor_page); + vm->monitor_page = NULL; + } + + ret = hcall_destroy_vm(vm->vmid); + if (ret < 0) { + dev_err(acrn_dev.this_device, + "Failed to destroy VM %u\n", vm->vmid); + clear_bit(ACRN_VM_FLAG_DESTROYED, &vm->flags); + return ret; + } + + acrn_vm_all_ram_unmap(vm); + + dev_dbg(acrn_dev.this_device, "VM %u destroyed.\n", vm->vmid); + vm->vmid = ACRN_INVALID_VMID; + return 0; +} + +/** + * acrn_inject_msi() - Inject a MSI interrupt into a User VM + * @vm: User VM + * @msi_addr: The MSI address + * @msi_data: The MSI data + * + * Return: 0 on success, <0 on error + */ +int acrn_msi_inject(struct acrn_vm *vm, u64 msi_addr, u64 msi_data) +{ + struct acrn_msi_entry *msi; + int ret; + + /* might be used in interrupt context, so use GFP_ATOMIC */ + msi = kzalloc(sizeof(*msi), GFP_ATOMIC); + if (!msi) + return -ENOMEM; + + /* + * msi_addr: addr[19:12] with dest vcpu id + * msi_data: data[7:0] with vector + */ + msi->msi_addr = msi_addr; + msi->msi_data = msi_data; + ret = hcall_inject_msi(vm->vmid, virt_to_phys(msi)); + if (ret < 0) + dev_err(acrn_dev.this_device, + "Failed to inject MSI to VM %u!\n", vm->vmid); + kfree(msi); + return ret; +} diff --git a/drivers/virt/vboxguest/vboxguest_utils.c b/drivers/virt/vboxguest/vboxguest_utils.c index ea05af41ec69..8d195e3f8301 100644 --- a/drivers/virt/vboxguest/vboxguest_utils.c +++ b/drivers/virt/vboxguest/vboxguest_utils.c @@ -468,7 +468,7 @@ static int hgcm_cancel_call(struct vbg_dev *gdev, struct vmmdev_hgcm_call *call) * Cancellation fun. */ static int vbg_hgcm_do_call(struct vbg_dev *gdev, struct vmmdev_hgcm_call *call, - u32 timeout_ms, bool *leak_it) + u32 timeout_ms, bool interruptible, bool *leak_it) { int rc, cancel_rc, ret; long timeout; @@ -495,10 +495,15 @@ static int vbg_hgcm_do_call(struct vbg_dev *gdev, struct vmmdev_hgcm_call *call, else timeout = msecs_to_jiffies(timeout_ms); - timeout = wait_event_interruptible_timeout( - gdev->hgcm_wq, - hgcm_req_done(gdev, &call->header), - timeout); + if (interruptible) { + timeout = wait_event_interruptible_timeout(gdev->hgcm_wq, + hgcm_req_done(gdev, &call->header), + timeout); + } else { + timeout = wait_event_timeout(gdev->hgcm_wq, + hgcm_req_done(gdev, &call->header), + timeout); + } /* timeout > 0 means hgcm_req_done has returned true, so success */ if (timeout > 0) @@ -631,7 +636,8 @@ int vbg_hgcm_call(struct vbg_dev *gdev, u32 requestor, u32 client_id, hgcm_call_init_call(call, client_id, function, parms, parm_count, bounce_bufs); - ret = vbg_hgcm_do_call(gdev, call, timeout_ms, &leak_it); + ret = vbg_hgcm_do_call(gdev, call, timeout_ms, + requestor & VMMDEV_REQUESTOR_USERMODE, &leak_it); if (ret == 0) { *vbox_status = call->header.result; ret = hgcm_call_copy_back_result(call, parms, parm_count, diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index 7b41130d3f35..ce1b3f6ec325 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -12,6 +12,14 @@ config ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS This option is selected if the architecture may need to enforce VIRTIO_F_ACCESS_PLATFORM +config VIRTIO_PCI_LIB + tristate + help + Modern PCI device implementation. This module implements the + basic probe and control for devices which are based on modern + PCI device with possible vendor specific extensions. Any + module that selects this module must depend on PCI. + menuconfig VIRTIO_MENU bool "Virtio drivers" default y @@ -21,6 +29,7 @@ if VIRTIO_MENU config VIRTIO_PCI tristate "PCI driver for virtio devices" depends on PCI + select VIRTIO_PCI_LIB select VIRTIO help This driver provides support for virtio based paravirtual device diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile index 591e6f72aa54..699bbea0465f 100644 --- a/drivers/virtio/Makefile +++ b/drivers/virtio/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_VIRTIO) += virtio.o virtio_ring.o +obj-$(CONFIG_VIRTIO_PCI_LIB) += virtio_pci_modern_dev.o obj-$(CONFIG_VIRTIO_MMIO) += virtio_mmio.o obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o virtio_pci-y := virtio_pci_modern.o virtio_pci_common.o diff --git a/drivers/virtio/virtio_input.c b/drivers/virtio/virtio_input.c index f1f6208edcf5..ce51ae165943 100644 --- a/drivers/virtio/virtio_input.c +++ b/drivers/virtio/virtio_input.c @@ -7,6 +7,7 @@ #include <uapi/linux/virtio_ids.h> #include <uapi/linux/virtio_input.h> +#include <linux/input/mt.h> struct virtio_input { struct virtio_device *vdev; @@ -64,6 +65,21 @@ static int virtinput_send_status(struct virtio_input *vi, unsigned long flags; int rc; + /* + * Since 29cc309d8bf1 (HID: hid-multitouch: forward MSC_TIMESTAMP), + * EV_MSC/MSC_TIMESTAMP is added to each before EV_SYN event. + * EV_MSC is configured as INPUT_PASS_TO_ALL. + * In case of touch device: + * BE pass EV_MSC/MSC_TIMESTAMP to FE on receiving event from evdev. + * FE pass EV_MSC/MSC_TIMESTAMP back to BE. + * BE writes EV_MSC/MSC_TIMESTAMP to evdev due to INPUT_PASS_TO_ALL. + * BE receives extra EV_MSC/MSC_TIMESTAMP and pass to FE. + * >>> Each new frame becomes larger and larger. + * Disable EV_MSC/MSC_TIMESTAMP forwarding for MT. + */ + if (vi->idev->mt && type == EV_MSC && code == MSC_TIMESTAMP) + return 0; + stsbuf = kzalloc(sizeof(*stsbuf), GFP_ATOMIC); if (!stsbuf) return -ENOMEM; @@ -204,7 +220,7 @@ static int virtinput_probe(struct virtio_device *vdev) struct virtio_input *vi; unsigned long flags; size_t size; - int abs, err; + int abs, err, nslots; if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) return -ENODEV; @@ -289,6 +305,13 @@ static int virtinput_probe(struct virtio_device *vdev) continue; virtinput_cfg_abs(vi, abs); } + + if (test_bit(ABS_MT_SLOT, vi->idev->absbit)) { + nslots = input_abs_get_max(vi->idev, ABS_MT_SLOT) + 1; + err = input_mt_init_slots(vi->idev, nslots, 0); + if (err) + goto err_mt_init_slots; + } } virtio_device_ready(vdev); @@ -304,6 +327,7 @@ err_input_register: spin_lock_irqsave(&vi->lock, flags); vi->ready = false; spin_unlock_irqrestore(&vi->lock, flags); +err_mt_init_slots: input_free_device(vi->idev); err_input_alloc: vdev->config->del_vqs(vdev); diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 9fc9ec4a25f5..10ec60d81e84 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -623,7 +623,7 @@ static int virtio_mem_add_memory(struct virtio_mem *vm, uint64_t addr, /* Memory might get onlined immediately. */ atomic64_add(size, &vm->offline_size); rc = add_memory_driver_managed(vm->nid, addr, size, vm->resource_name, - MEMHP_MERGE_RESOURCE); + MHP_MERGE_RESOURCE); if (rc) { atomic64_sub(size, &vm->offline_size); dev_warn(&vm->vdev->dev, "adding memory failed: %d\n", rc); @@ -2222,7 +2222,7 @@ static int virtio_mem_unplug_pending_mb(struct virtio_mem *vm) */ static void virtio_mem_refresh_config(struct virtio_mem *vm) { - const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS; + const struct range pluggable_range = mhp_get_pluggable_range(true); uint64_t new_plugged_size, usable_region_size, end_addr; /* the plugged_size is just a reflection of what _we_ did previously */ @@ -2234,15 +2234,25 @@ static void virtio_mem_refresh_config(struct virtio_mem *vm) /* calculate the last usable memory block id */ virtio_cread_le(vm->vdev, struct virtio_mem_config, usable_region_size, &usable_region_size); - end_addr = vm->addr + usable_region_size; - end_addr = min(end_addr, phys_limit); + end_addr = min(vm->addr + usable_region_size - 1, + pluggable_range.end); - if (vm->in_sbm) - vm->sbm.last_usable_mb_id = - virtio_mem_phys_to_mb_id(end_addr) - 1; - else - vm->bbm.last_usable_bb_id = - virtio_mem_phys_to_bb_id(vm, end_addr) - 1; + if (vm->in_sbm) { + vm->sbm.last_usable_mb_id = virtio_mem_phys_to_mb_id(end_addr); + if (!IS_ALIGNED(end_addr + 1, memory_block_size_bytes())) + vm->sbm.last_usable_mb_id--; + } else { + vm->bbm.last_usable_bb_id = virtio_mem_phys_to_bb_id(vm, + end_addr); + if (!IS_ALIGNED(end_addr + 1, vm->bbm.bb_size)) + vm->bbm.last_usable_bb_id--; + } + /* + * If we cannot plug any of our device memory (e.g., nothing in the + * usable region is addressable), the last usable memory block id will + * be smaller than the first usable memory block id. We'll stop + * attempting to add memory with -ENOSPC from our main loop. + */ /* see if there is a request to change the size */ virtio_cread_le(vm->vdev, struct virtio_mem_config, requested_size, @@ -2364,7 +2374,7 @@ static int virtio_mem_init_vq(struct virtio_mem *vm) static int virtio_mem_init(struct virtio_mem *vm) { - const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS; + const struct range pluggable_range = mhp_get_pluggable_range(true); uint64_t sb_size, addr; uint16_t node_id; @@ -2405,9 +2415,10 @@ static int virtio_mem_init(struct virtio_mem *vm) if (!IS_ALIGNED(vm->addr + vm->region_size, memory_block_size_bytes())) dev_warn(&vm->vdev->dev, "The alignment of the physical end address can make some memory unusable.\n"); - if (vm->addr + vm->region_size > phys_limit) + if (vm->addr < pluggable_range.start || + vm->addr + vm->region_size - 1 > pluggable_range.end) dev_warn(&vm->vdev->dev, - "Some memory is not addressable. This can make some memory unusable.\n"); + "Some device memory is not addressable/pluggable. This can make some memory unusable.\n"); /* * We want subblocks to span at least MAX_ORDER_NR_PAGES and @@ -2429,7 +2440,8 @@ static int virtio_mem_init(struct virtio_mem *vm) vm->sbm.sb_size; /* Round up to the next full memory block */ - addr = vm->addr + memory_block_size_bytes() - 1; + addr = max_t(uint64_t, vm->addr, pluggable_range.start) + + memory_block_size_bytes() - 1; vm->sbm.first_mb_id = virtio_mem_phys_to_mb_id(addr); vm->sbm.next_mb_id = vm->sbm.first_mb_id; } else { @@ -2450,7 +2462,8 @@ static int virtio_mem_init(struct virtio_mem *vm) } /* Round up to the next aligned big block */ - addr = vm->addr + vm->bbm.bb_size - 1; + addr = max_t(uint64_t, vm->addr, pluggable_range.start) + + vm->bbm.bb_size - 1; vm->bbm.first_bb_id = virtio_mem_phys_to_bb_id(vm, addr); vm->bbm.next_bb_id = vm->bbm.first_bb_id; } @@ -2577,7 +2590,7 @@ static int virtio_mem_probe(struct virtio_device *vdev) * actually in use (e.g., trying to reload the driver). */ if (vm->plugged_size) { - vm->unplug_all_required = 1; + vm->unplug_all_required = true; dev_info(&vm->vdev->dev, "unplugging all memory is required\n"); } diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 238383ff1064..a286d22b6551 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -126,7 +126,7 @@ static int vm_finalize_features(struct virtio_device *vdev) /* Give virtio_ring a chance to accept features. */ vring_transport_features(vdev); - /* Make sure there is are no mixed devices */ + /* Make sure there are no mixed devices */ if (vm_dev->version == 2 && !__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) { dev_err(&vdev->dev, "New virtio-mmio devices (version 2) must provide VIRTIO_F_VERSION_1 feature!\n"); diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index b2f0eb4067cb..beec047a8f8d 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -25,6 +25,7 @@ #include <linux/virtio_config.h> #include <linux/virtio_ring.h> #include <linux/virtio_pci.h> +#include <linux/virtio_pci_modern.h> #include <linux/highmem.h> #include <linux/spinlock.h> @@ -43,31 +44,12 @@ struct virtio_pci_vq_info { struct virtio_pci_device { struct virtio_device vdev; struct pci_dev *pci_dev; + struct virtio_pci_modern_device mdev; /* In legacy mode, these two point to within ->legacy. */ /* Where to read and clear interrupt */ u8 __iomem *isr; - /* Modern only fields */ - /* The IO mapping for the PCI config space (non-legacy mode) */ - struct virtio_pci_common_cfg __iomem *common; - /* Device-specific data (non-legacy mode) */ - void __iomem *device; - /* Base of vq notifications (non-legacy mode). */ - void __iomem *notify_base; - - /* So we can sanity-check accesses. */ - size_t notify_len; - size_t device_len; - - /* Capability for when we need to map notifications per-vq. */ - int notify_map_cap; - - /* Multiply queue_notify_off by this value. (non-legacy mode). */ - u32 notify_offset_multiplier; - - int modern_bars; - /* Legacy only field */ /* the IO mapping for the PCI config space */ void __iomem *ioaddr; diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 3d6ae5a5e252..fbd4ebc00eb6 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -19,136 +19,11 @@ #define VIRTIO_RING_NO_LEGACY #include "virtio_pci_common.h" -/* - * Type-safe wrappers for io accesses. - * Use these to enforce at compile time the following spec requirement: - * - * The driver MUST access each field using the “natural” access - * method, i.e. 32-bit accesses for 32-bit fields, 16-bit accesses - * for 16-bit fields and 8-bit accesses for 8-bit fields. - */ -static inline u8 vp_ioread8(const u8 __iomem *addr) -{ - return ioread8(addr); -} -static inline u16 vp_ioread16 (const __le16 __iomem *addr) -{ - return ioread16(addr); -} - -static inline u32 vp_ioread32(const __le32 __iomem *addr) -{ - return ioread32(addr); -} - -static inline void vp_iowrite8(u8 value, u8 __iomem *addr) -{ - iowrite8(value, addr); -} - -static inline void vp_iowrite16(u16 value, __le16 __iomem *addr) -{ - iowrite16(value, addr); -} - -static inline void vp_iowrite32(u32 value, __le32 __iomem *addr) -{ - iowrite32(value, addr); -} - -static void vp_iowrite64_twopart(u64 val, - __le32 __iomem *lo, __le32 __iomem *hi) -{ - vp_iowrite32((u32)val, lo); - vp_iowrite32(val >> 32, hi); -} - -static void __iomem *map_capability(struct pci_dev *dev, int off, - size_t minlen, - u32 align, - u32 start, u32 size, - size_t *len) -{ - u8 bar; - u32 offset, length; - void __iomem *p; - - pci_read_config_byte(dev, off + offsetof(struct virtio_pci_cap, - bar), - &bar); - pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, offset), - &offset); - pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, length), - &length); - - if (length <= start) { - dev_err(&dev->dev, - "virtio_pci: bad capability len %u (>%u expected)\n", - length, start); - return NULL; - } - - if (length - start < minlen) { - dev_err(&dev->dev, - "virtio_pci: bad capability len %u (>=%zu expected)\n", - length, minlen); - return NULL; - } - - length -= start; - - if (start + offset < offset) { - dev_err(&dev->dev, - "virtio_pci: map wrap-around %u+%u\n", - start, offset); - return NULL; - } - - offset += start; - - if (offset & (align - 1)) { - dev_err(&dev->dev, - "virtio_pci: offset %u not aligned to %u\n", - offset, align); - return NULL; - } - - if (length > size) - length = size; - - if (len) - *len = length; - - if (minlen + offset < minlen || - minlen + offset > pci_resource_len(dev, bar)) { - dev_err(&dev->dev, - "virtio_pci: map virtio %zu@%u " - "out of range on bar %i length %lu\n", - minlen, offset, - bar, (unsigned long)pci_resource_len(dev, bar)); - return NULL; - } - - p = pci_iomap_range(dev, bar, offset, length); - if (!p) - dev_err(&dev->dev, - "virtio_pci: unable to map virtio %u@%u on bar %i\n", - length, offset, bar); - return p; -} - -/* virtio config->get_features() implementation */ static u64 vp_get_features(struct virtio_device *vdev) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); - u64 features; - - vp_iowrite32(0, &vp_dev->common->device_feature_select); - features = vp_ioread32(&vp_dev->common->device_feature); - vp_iowrite32(1, &vp_dev->common->device_feature_select); - features |= ((u64)vp_ioread32(&vp_dev->common->device_feature) << 32); - return features; + return vp_modern_get_features(&vp_dev->mdev); } static void vp_transport_features(struct virtio_device *vdev, u64 features) @@ -179,10 +54,7 @@ static int vp_finalize_features(struct virtio_device *vdev) return -EINVAL; } - vp_iowrite32(0, &vp_dev->common->guest_feature_select); - vp_iowrite32((u32)vdev->features, &vp_dev->common->guest_feature); - vp_iowrite32(1, &vp_dev->common->guest_feature_select); - vp_iowrite32(vdev->features >> 32, &vp_dev->common->guest_feature); + vp_modern_set_features(&vp_dev->mdev, vdev->features); return 0; } @@ -192,29 +64,31 @@ static void vp_get(struct virtio_device *vdev, unsigned offset, void *buf, unsigned len) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; + void __iomem *device = mdev->device; u8 b; __le16 w; __le32 l; - BUG_ON(offset + len > vp_dev->device_len); + BUG_ON(offset + len > mdev->device_len); switch (len) { case 1: - b = ioread8(vp_dev->device + offset); + b = ioread8(device + offset); memcpy(buf, &b, sizeof b); break; case 2: - w = cpu_to_le16(ioread16(vp_dev->device + offset)); + w = cpu_to_le16(ioread16(device + offset)); memcpy(buf, &w, sizeof w); break; case 4: - l = cpu_to_le32(ioread32(vp_dev->device + offset)); + l = cpu_to_le32(ioread32(device + offset)); memcpy(buf, &l, sizeof l); break; case 8: - l = cpu_to_le32(ioread32(vp_dev->device + offset)); + l = cpu_to_le32(ioread32(device + offset)); memcpy(buf, &l, sizeof l); - l = cpu_to_le32(ioread32(vp_dev->device + offset + sizeof l)); + l = cpu_to_le32(ioread32(device + offset + sizeof l)); memcpy(buf + sizeof l, &l, sizeof l); break; default: @@ -228,30 +102,32 @@ static void vp_set(struct virtio_device *vdev, unsigned offset, const void *buf, unsigned len) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; + void __iomem *device = mdev->device; u8 b; __le16 w; __le32 l; - BUG_ON(offset + len > vp_dev->device_len); + BUG_ON(offset + len > mdev->device_len); switch (len) { case 1: memcpy(&b, buf, sizeof b); - iowrite8(b, vp_dev->device + offset); + iowrite8(b, device + offset); break; case 2: memcpy(&w, buf, sizeof w); - iowrite16(le16_to_cpu(w), vp_dev->device + offset); + iowrite16(le16_to_cpu(w), device + offset); break; case 4: memcpy(&l, buf, sizeof l); - iowrite32(le32_to_cpu(l), vp_dev->device + offset); + iowrite32(le32_to_cpu(l), device + offset); break; case 8: memcpy(&l, buf, sizeof l); - iowrite32(le32_to_cpu(l), vp_dev->device + offset); + iowrite32(le32_to_cpu(l), device + offset); memcpy(&l, buf + sizeof l, sizeof l); - iowrite32(le32_to_cpu(l), vp_dev->device + offset + sizeof l); + iowrite32(le32_to_cpu(l), device + offset + sizeof l); break; default: BUG(); @@ -261,35 +137,40 @@ static void vp_set(struct virtio_device *vdev, unsigned offset, static u32 vp_generation(struct virtio_device *vdev) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); - return vp_ioread8(&vp_dev->common->config_generation); + + return vp_modern_generation(&vp_dev->mdev); } /* config->{get,set}_status() implementations */ static u8 vp_get_status(struct virtio_device *vdev) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); - return vp_ioread8(&vp_dev->common->device_status); + + return vp_modern_get_status(&vp_dev->mdev); } static void vp_set_status(struct virtio_device *vdev, u8 status) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); + /* We should never be setting status to 0. */ BUG_ON(status == 0); - vp_iowrite8(status, &vp_dev->common->device_status); + vp_modern_set_status(&vp_dev->mdev, status); } static void vp_reset(struct virtio_device *vdev) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; + /* 0 status means a reset. */ - vp_iowrite8(0, &vp_dev->common->device_status); + vp_modern_set_status(mdev, 0); /* After writing 0 to device_status, the driver MUST wait for a read of * device_status to return 0 before reinitializing the device. * This will flush out the status write, and flush in device writes, * including MSI-X interrupts, if any. */ - while (vp_ioread8(&vp_dev->common->device_status)) + while (vp_modern_get_status(mdev)) msleep(1); /* Flush pending VQ/configuration callbacks. */ vp_synchronize_vectors(vdev); @@ -297,11 +178,7 @@ static void vp_reset(struct virtio_device *vdev) static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) { - /* Setup the vector used for configuration events */ - vp_iowrite16(vector, &vp_dev->common->msix_config); - /* Verify we had enough resources to assign the vector */ - /* Will also flush the write out to device */ - return vp_ioread16(&vp_dev->common->msix_config); + return vp_modern_config_vector(&vp_dev->mdev, vector); } static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, @@ -312,20 +189,18 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, bool ctx, u16 msix_vec) { - struct virtio_pci_common_cfg __iomem *cfg = vp_dev->common; + + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; struct virtqueue *vq; u16 num, off; int err; - if (index >= vp_ioread16(&cfg->num_queues)) + if (index >= vp_modern_get_num_queues(mdev)) return ERR_PTR(-ENOENT); - /* Select the queue we're interested in */ - vp_iowrite16(index, &cfg->queue_select); - /* Check if queue is either not available or already active. */ - num = vp_ioread16(&cfg->queue_size); - if (!num || vp_ioread16(&cfg->queue_enable)) + num = vp_modern_get_queue_size(mdev, index); + if (!num || vp_modern_get_queue_enable(mdev, index)) return ERR_PTR(-ENOENT); if (num & (num - 1)) { @@ -334,7 +209,7 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, } /* get offset of notification word for this vq */ - off = vp_ioread16(&cfg->queue_notify_off); + off = vp_modern_get_queue_notify_off(mdev, index); info->msix_vector = msix_vec; @@ -347,33 +222,30 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, return ERR_PTR(-ENOMEM); /* activate the queue */ - vp_iowrite16(virtqueue_get_vring_size(vq), &cfg->queue_size); - vp_iowrite64_twopart(virtqueue_get_desc_addr(vq), - &cfg->queue_desc_lo, &cfg->queue_desc_hi); - vp_iowrite64_twopart(virtqueue_get_avail_addr(vq), - &cfg->queue_avail_lo, &cfg->queue_avail_hi); - vp_iowrite64_twopart(virtqueue_get_used_addr(vq), - &cfg->queue_used_lo, &cfg->queue_used_hi); - - if (vp_dev->notify_base) { + vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq)); + vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq), + virtqueue_get_avail_addr(vq), + virtqueue_get_used_addr(vq)); + + if (mdev->notify_base) { /* offset should not wrap */ - if ((u64)off * vp_dev->notify_offset_multiplier + 2 - > vp_dev->notify_len) { - dev_warn(&vp_dev->pci_dev->dev, + if ((u64)off * mdev->notify_offset_multiplier + 2 + > mdev->notify_len) { + dev_warn(&mdev->pci_dev->dev, "bad notification offset %u (x %u) " "for queue %u > %zd", - off, vp_dev->notify_offset_multiplier, - index, vp_dev->notify_len); + off, mdev->notify_offset_multiplier, + index, mdev->notify_len); err = -EINVAL; goto err_map_notify; } - vq->priv = (void __force *)vp_dev->notify_base + - off * vp_dev->notify_offset_multiplier; + vq->priv = (void __force *)mdev->notify_base + + off * mdev->notify_offset_multiplier; } else { - vq->priv = (void __force *)map_capability(vp_dev->pci_dev, - vp_dev->notify_map_cap, 2, 2, - off * vp_dev->notify_offset_multiplier, 2, - NULL); + vq->priv = (void __force *)vp_modern_map_capability(mdev, + mdev->notify_map_cap, 2, 2, + off * mdev->notify_offset_multiplier, 2, + NULL); } if (!vq->priv) { @@ -382,8 +254,7 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, } if (msix_vec != VIRTIO_MSI_NO_VECTOR) { - vp_iowrite16(msix_vec, &cfg->queue_msix_vector); - msix_vec = vp_ioread16(&cfg->queue_msix_vector); + msix_vec = vp_modern_queue_vector(mdev, index, msix_vec); if (msix_vec == VIRTIO_MSI_NO_VECTOR) { err = -EBUSY; goto err_assign_vector; @@ -393,8 +264,8 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, return vq; err_assign_vector: - if (!vp_dev->notify_base) - pci_iounmap(vp_dev->pci_dev, (void __iomem __force *)vq->priv); + if (!mdev->notify_base) + pci_iounmap(mdev->pci_dev, (void __iomem __force *)vq->priv); err_map_notify: vring_del_virtqueue(vq); return ERR_PTR(err); @@ -416,10 +287,8 @@ static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs, /* Select and activate all queues. Has to be done last: once we do * this, there's no way to go back except reset. */ - list_for_each_entry(vq, &vdev->vqs, list) { - vp_iowrite16(vq->index, &vp_dev->common->queue_select); - vp_iowrite16(1, &vp_dev->common->queue_enable); - } + list_for_each_entry(vq, &vdev->vqs, list) + vp_modern_set_queue_enable(&vp_dev->mdev, vq->index, true); return 0; } @@ -428,18 +297,14 @@ static void del_vq(struct virtio_pci_vq_info *info) { struct virtqueue *vq = info->vq; struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; - vp_iowrite16(vq->index, &vp_dev->common->queue_select); - - if (vp_dev->msix_enabled) { - vp_iowrite16(VIRTIO_MSI_NO_VECTOR, - &vp_dev->common->queue_msix_vector); - /* Flush the write out to device */ - vp_ioread16(&vp_dev->common->queue_msix_vector); - } + if (vp_dev->msix_enabled) + vp_modern_queue_vector(mdev, vq->index, + VIRTIO_MSI_NO_VECTOR); - if (!vp_dev->notify_base) - pci_iounmap(vp_dev->pci_dev, (void __force __iomem *)vq->priv); + if (!mdev->notify_base) + pci_iounmap(mdev->pci_dev, (void __force __iomem *)vq->priv); vring_del_virtqueue(vq); } @@ -571,261 +436,36 @@ static const struct virtio_config_ops virtio_pci_config_ops = { .get_shm_region = vp_get_shm_region, }; -/** - * virtio_pci_find_capability - walk capabilities to find device info. - * @dev: the pci device - * @cfg_type: the VIRTIO_PCI_CAP_* value we seek - * @ioresource_types: IORESOURCE_MEM and/or IORESOURCE_IO. - * @bars: the bitmask of BARs - * - * Returns offset of the capability, or 0. - */ -static inline int virtio_pci_find_capability(struct pci_dev *dev, u8 cfg_type, - u32 ioresource_types, int *bars) -{ - int pos; - - for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); - pos > 0; - pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) { - u8 type, bar; - pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, - cfg_type), - &type); - pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, - bar), - &bar); - - /* Ignore structures with reserved BAR values */ - if (bar > 0x5) - continue; - - if (type == cfg_type) { - if (pci_resource_len(dev, bar) && - pci_resource_flags(dev, bar) & ioresource_types) { - *bars |= (1 << bar); - return pos; - } - } - } - return 0; -} - -/* This is part of the ABI. Don't screw with it. */ -static inline void check_offsets(void) -{ - /* Note: disk space was harmed in compilation of this function. */ - BUILD_BUG_ON(VIRTIO_PCI_CAP_VNDR != - offsetof(struct virtio_pci_cap, cap_vndr)); - BUILD_BUG_ON(VIRTIO_PCI_CAP_NEXT != - offsetof(struct virtio_pci_cap, cap_next)); - BUILD_BUG_ON(VIRTIO_PCI_CAP_LEN != - offsetof(struct virtio_pci_cap, cap_len)); - BUILD_BUG_ON(VIRTIO_PCI_CAP_CFG_TYPE != - offsetof(struct virtio_pci_cap, cfg_type)); - BUILD_BUG_ON(VIRTIO_PCI_CAP_BAR != - offsetof(struct virtio_pci_cap, bar)); - BUILD_BUG_ON(VIRTIO_PCI_CAP_OFFSET != - offsetof(struct virtio_pci_cap, offset)); - BUILD_BUG_ON(VIRTIO_PCI_CAP_LENGTH != - offsetof(struct virtio_pci_cap, length)); - BUILD_BUG_ON(VIRTIO_PCI_NOTIFY_CAP_MULT != - offsetof(struct virtio_pci_notify_cap, - notify_off_multiplier)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_DFSELECT != - offsetof(struct virtio_pci_common_cfg, - device_feature_select)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_DF != - offsetof(struct virtio_pci_common_cfg, device_feature)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_GFSELECT != - offsetof(struct virtio_pci_common_cfg, - guest_feature_select)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_GF != - offsetof(struct virtio_pci_common_cfg, guest_feature)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_MSIX != - offsetof(struct virtio_pci_common_cfg, msix_config)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_NUMQ != - offsetof(struct virtio_pci_common_cfg, num_queues)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_STATUS != - offsetof(struct virtio_pci_common_cfg, device_status)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_CFGGENERATION != - offsetof(struct virtio_pci_common_cfg, config_generation)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SELECT != - offsetof(struct virtio_pci_common_cfg, queue_select)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SIZE != - offsetof(struct virtio_pci_common_cfg, queue_size)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_MSIX != - offsetof(struct virtio_pci_common_cfg, queue_msix_vector)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_ENABLE != - offsetof(struct virtio_pci_common_cfg, queue_enable)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_NOFF != - offsetof(struct virtio_pci_common_cfg, queue_notify_off)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCLO != - offsetof(struct virtio_pci_common_cfg, queue_desc_lo)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCHI != - offsetof(struct virtio_pci_common_cfg, queue_desc_hi)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILLO != - offsetof(struct virtio_pci_common_cfg, queue_avail_lo)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILHI != - offsetof(struct virtio_pci_common_cfg, queue_avail_hi)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDLO != - offsetof(struct virtio_pci_common_cfg, queue_used_lo)); - BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDHI != - offsetof(struct virtio_pci_common_cfg, queue_used_hi)); -} - /* the PCI probing function */ int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev) { + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; struct pci_dev *pci_dev = vp_dev->pci_dev; - int err, common, isr, notify, device; - u32 notify_length; - u32 notify_offset; - - check_offsets(); - - /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */ - if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f) - return -ENODEV; - - if (pci_dev->device < 0x1040) { - /* Transitional devices: use the PCI subsystem device id as - * virtio device id, same as legacy driver always did. - */ - vp_dev->vdev.id.device = pci_dev->subsystem_device; - } else { - /* Modern devices: simply use PCI device id, but start from 0x1040. */ - vp_dev->vdev.id.device = pci_dev->device - 0x1040; - } - vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor; - - /* check for a common config: if not, use legacy mode (bar 0). */ - common = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_COMMON_CFG, - IORESOURCE_IO | IORESOURCE_MEM, - &vp_dev->modern_bars); - if (!common) { - dev_info(&pci_dev->dev, - "virtio_pci: leaving for legacy driver\n"); - return -ENODEV; - } - - /* If common is there, these should be too... */ - isr = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_ISR_CFG, - IORESOURCE_IO | IORESOURCE_MEM, - &vp_dev->modern_bars); - notify = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_NOTIFY_CFG, - IORESOURCE_IO | IORESOURCE_MEM, - &vp_dev->modern_bars); - if (!isr || !notify) { - dev_err(&pci_dev->dev, - "virtio_pci: missing capabilities %i/%i/%i\n", - common, isr, notify); - return -EINVAL; - } - - err = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)); - if (err) - err = dma_set_mask_and_coherent(&pci_dev->dev, - DMA_BIT_MASK(32)); - if (err) - dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n"); + int err; - /* Device capability is only mandatory for devices that have - * device-specific configuration. - */ - device = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_DEVICE_CFG, - IORESOURCE_IO | IORESOURCE_MEM, - &vp_dev->modern_bars); + mdev->pci_dev = pci_dev; - err = pci_request_selected_regions(pci_dev, vp_dev->modern_bars, - "virtio-pci-modern"); + err = vp_modern_probe(mdev); if (err) return err; - err = -EINVAL; - vp_dev->common = map_capability(pci_dev, common, - sizeof(struct virtio_pci_common_cfg), 4, - 0, sizeof(struct virtio_pci_common_cfg), - NULL); - if (!vp_dev->common) - goto err_map_common; - vp_dev->isr = map_capability(pci_dev, isr, sizeof(u8), 1, - 0, 1, - NULL); - if (!vp_dev->isr) - goto err_map_isr; - - /* Read notify_off_multiplier from config space. */ - pci_read_config_dword(pci_dev, - notify + offsetof(struct virtio_pci_notify_cap, - notify_off_multiplier), - &vp_dev->notify_offset_multiplier); - /* Read notify length and offset from config space. */ - pci_read_config_dword(pci_dev, - notify + offsetof(struct virtio_pci_notify_cap, - cap.length), - ¬ify_length); - - pci_read_config_dword(pci_dev, - notify + offsetof(struct virtio_pci_notify_cap, - cap.offset), - ¬ify_offset); - - /* We don't know how many VQs we'll map, ahead of the time. - * If notify length is small, map it all now. - * Otherwise, map each VQ individually later. - */ - if ((u64)notify_length + (notify_offset % PAGE_SIZE) <= PAGE_SIZE) { - vp_dev->notify_base = map_capability(pci_dev, notify, 2, 2, - 0, notify_length, - &vp_dev->notify_len); - if (!vp_dev->notify_base) - goto err_map_notify; - } else { - vp_dev->notify_map_cap = notify; - } - - /* Again, we don't know how much we should map, but PAGE_SIZE - * is more than enough for all existing devices. - */ - if (device) { - vp_dev->device = map_capability(pci_dev, device, 0, 4, - 0, PAGE_SIZE, - &vp_dev->device_len); - if (!vp_dev->device) - goto err_map_device; - + if (mdev->device) vp_dev->vdev.config = &virtio_pci_config_ops; - } else { + else vp_dev->vdev.config = &virtio_pci_config_nodev_ops; - } vp_dev->config_vector = vp_config_vector; vp_dev->setup_vq = setup_vq; vp_dev->del_vq = del_vq; + vp_dev->isr = mdev->isr; + vp_dev->vdev.id = mdev->id; return 0; - -err_map_device: - if (vp_dev->notify_base) - pci_iounmap(pci_dev, vp_dev->notify_base); -err_map_notify: - pci_iounmap(pci_dev, vp_dev->isr); -err_map_isr: - pci_iounmap(pci_dev, vp_dev->common); -err_map_common: - return err; } void virtio_pci_modern_remove(struct virtio_pci_device *vp_dev) { - struct pci_dev *pci_dev = vp_dev->pci_dev; + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; - if (vp_dev->device) - pci_iounmap(pci_dev, vp_dev->device); - if (vp_dev->notify_base) - pci_iounmap(pci_dev, vp_dev->notify_base); - pci_iounmap(pci_dev, vp_dev->isr); - pci_iounmap(pci_dev, vp_dev->common); - pci_release_selected_regions(pci_dev, vp_dev->modern_bars); + vp_modern_remove(mdev); } diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio_pci_modern_dev.c new file mode 100644 index 000000000000..cbd667496bb1 --- /dev/null +++ b/drivers/virtio/virtio_pci_modern_dev.c @@ -0,0 +1,599 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include <linux/virtio_pci_modern.h> +#include <linux/module.h> +#include <linux/pci.h> + +/* + * vp_modern_map_capability - map a part of virtio pci capability + * @mdev: the modern virtio-pci device + * @off: offset of the capability + * @minlen: minimal length of the capability + * @align: align requirement + * @start: start from the capability + * @size: map size + * @len: the length that is actually mapped + * + * Returns the io address of for the part of the capability + */ +void __iomem *vp_modern_map_capability(struct virtio_pci_modern_device *mdev, int off, + size_t minlen, + u32 align, + u32 start, u32 size, + size_t *len) +{ + struct pci_dev *dev = mdev->pci_dev; + u8 bar; + u32 offset, length; + void __iomem *p; + + pci_read_config_byte(dev, off + offsetof(struct virtio_pci_cap, + bar), + &bar); + pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, offset), + &offset); + pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, length), + &length); + + if (length <= start) { + dev_err(&dev->dev, + "virtio_pci: bad capability len %u (>%u expected)\n", + length, start); + return NULL; + } + + if (length - start < minlen) { + dev_err(&dev->dev, + "virtio_pci: bad capability len %u (>=%zu expected)\n", + length, minlen); + return NULL; + } + + length -= start; + + if (start + offset < offset) { + dev_err(&dev->dev, + "virtio_pci: map wrap-around %u+%u\n", + start, offset); + return NULL; + } + + offset += start; + + if (offset & (align - 1)) { + dev_err(&dev->dev, + "virtio_pci: offset %u not aligned to %u\n", + offset, align); + return NULL; + } + + if (length > size) + length = size; + + if (len) + *len = length; + + if (minlen + offset < minlen || + minlen + offset > pci_resource_len(dev, bar)) { + dev_err(&dev->dev, + "virtio_pci: map virtio %zu@%u " + "out of range on bar %i length %lu\n", + minlen, offset, + bar, (unsigned long)pci_resource_len(dev, bar)); + return NULL; + } + + p = pci_iomap_range(dev, bar, offset, length); + if (!p) + dev_err(&dev->dev, + "virtio_pci: unable to map virtio %u@%u on bar %i\n", + length, offset, bar); + return p; +} +EXPORT_SYMBOL_GPL(vp_modern_map_capability); + +/** + * virtio_pci_find_capability - walk capabilities to find device info. + * @dev: the pci device + * @cfg_type: the VIRTIO_PCI_CAP_* value we seek + * @ioresource_types: IORESOURCE_MEM and/or IORESOURCE_IO. + * @bars: the bitmask of BARs + * + * Returns offset of the capability, or 0. + */ +static inline int virtio_pci_find_capability(struct pci_dev *dev, u8 cfg_type, + u32 ioresource_types, int *bars) +{ + int pos; + + for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); + pos > 0; + pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) { + u8 type, bar; + pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, + cfg_type), + &type); + pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, + bar), + &bar); + + /* Ignore structures with reserved BAR values */ + if (bar > 0x5) + continue; + + if (type == cfg_type) { + if (pci_resource_len(dev, bar) && + pci_resource_flags(dev, bar) & ioresource_types) { + *bars |= (1 << bar); + return pos; + } + } + } + return 0; +} + +/* This is part of the ABI. Don't screw with it. */ +static inline void check_offsets(void) +{ + /* Note: disk space was harmed in compilation of this function. */ + BUILD_BUG_ON(VIRTIO_PCI_CAP_VNDR != + offsetof(struct virtio_pci_cap, cap_vndr)); + BUILD_BUG_ON(VIRTIO_PCI_CAP_NEXT != + offsetof(struct virtio_pci_cap, cap_next)); + BUILD_BUG_ON(VIRTIO_PCI_CAP_LEN != + offsetof(struct virtio_pci_cap, cap_len)); + BUILD_BUG_ON(VIRTIO_PCI_CAP_CFG_TYPE != + offsetof(struct virtio_pci_cap, cfg_type)); + BUILD_BUG_ON(VIRTIO_PCI_CAP_BAR != + offsetof(struct virtio_pci_cap, bar)); + BUILD_BUG_ON(VIRTIO_PCI_CAP_OFFSET != + offsetof(struct virtio_pci_cap, offset)); + BUILD_BUG_ON(VIRTIO_PCI_CAP_LENGTH != + offsetof(struct virtio_pci_cap, length)); + BUILD_BUG_ON(VIRTIO_PCI_NOTIFY_CAP_MULT != + offsetof(struct virtio_pci_notify_cap, + notify_off_multiplier)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_DFSELECT != + offsetof(struct virtio_pci_common_cfg, + device_feature_select)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_DF != + offsetof(struct virtio_pci_common_cfg, device_feature)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_GFSELECT != + offsetof(struct virtio_pci_common_cfg, + guest_feature_select)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_GF != + offsetof(struct virtio_pci_common_cfg, guest_feature)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_MSIX != + offsetof(struct virtio_pci_common_cfg, msix_config)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_NUMQ != + offsetof(struct virtio_pci_common_cfg, num_queues)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_STATUS != + offsetof(struct virtio_pci_common_cfg, device_status)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_CFGGENERATION != + offsetof(struct virtio_pci_common_cfg, config_generation)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SELECT != + offsetof(struct virtio_pci_common_cfg, queue_select)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SIZE != + offsetof(struct virtio_pci_common_cfg, queue_size)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_MSIX != + offsetof(struct virtio_pci_common_cfg, queue_msix_vector)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_ENABLE != + offsetof(struct virtio_pci_common_cfg, queue_enable)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_NOFF != + offsetof(struct virtio_pci_common_cfg, queue_notify_off)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCLO != + offsetof(struct virtio_pci_common_cfg, queue_desc_lo)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCHI != + offsetof(struct virtio_pci_common_cfg, queue_desc_hi)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILLO != + offsetof(struct virtio_pci_common_cfg, queue_avail_lo)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILHI != + offsetof(struct virtio_pci_common_cfg, queue_avail_hi)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDLO != + offsetof(struct virtio_pci_common_cfg, queue_used_lo)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDHI != + offsetof(struct virtio_pci_common_cfg, queue_used_hi)); +} + +/* + * vp_modern_probe: probe the modern virtio pci device, note that the + * caller is required to enable PCI device before calling this function. + * @mdev: the modern virtio-pci device + * + * Return 0 on succeed otherwise fail + */ +int vp_modern_probe(struct virtio_pci_modern_device *mdev) +{ + struct pci_dev *pci_dev = mdev->pci_dev; + int err, common, isr, notify, device; + u32 notify_length; + u32 notify_offset; + + check_offsets(); + + mdev->pci_dev = pci_dev; + + /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */ + if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f) + return -ENODEV; + + if (pci_dev->device < 0x1040) { + /* Transitional devices: use the PCI subsystem device id as + * virtio device id, same as legacy driver always did. + */ + mdev->id.device = pci_dev->subsystem_device; + } else { + /* Modern devices: simply use PCI device id, but start from 0x1040. */ + mdev->id.device = pci_dev->device - 0x1040; + } + mdev->id.vendor = pci_dev->subsystem_vendor; + + /* check for a common config: if not, use legacy mode (bar 0). */ + common = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_COMMON_CFG, + IORESOURCE_IO | IORESOURCE_MEM, + &mdev->modern_bars); + if (!common) { + dev_info(&pci_dev->dev, + "virtio_pci: leaving for legacy driver\n"); + return -ENODEV; + } + + /* If common is there, these should be too... */ + isr = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_ISR_CFG, + IORESOURCE_IO | IORESOURCE_MEM, + &mdev->modern_bars); + notify = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_NOTIFY_CFG, + IORESOURCE_IO | IORESOURCE_MEM, + &mdev->modern_bars); + if (!isr || !notify) { + dev_err(&pci_dev->dev, + "virtio_pci: missing capabilities %i/%i/%i\n", + common, isr, notify); + return -EINVAL; + } + + err = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)); + if (err) + err = dma_set_mask_and_coherent(&pci_dev->dev, + DMA_BIT_MASK(32)); + if (err) + dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n"); + + /* Device capability is only mandatory for devices that have + * device-specific configuration. + */ + device = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_DEVICE_CFG, + IORESOURCE_IO | IORESOURCE_MEM, + &mdev->modern_bars); + + err = pci_request_selected_regions(pci_dev, mdev->modern_bars, + "virtio-pci-modern"); + if (err) + return err; + + err = -EINVAL; + mdev->common = vp_modern_map_capability(mdev, common, + sizeof(struct virtio_pci_common_cfg), 4, + 0, sizeof(struct virtio_pci_common_cfg), + NULL); + if (!mdev->common) + goto err_map_common; + mdev->isr = vp_modern_map_capability(mdev, isr, sizeof(u8), 1, + 0, 1, + NULL); + if (!mdev->isr) + goto err_map_isr; + + /* Read notify_off_multiplier from config space. */ + pci_read_config_dword(pci_dev, + notify + offsetof(struct virtio_pci_notify_cap, + notify_off_multiplier), + &mdev->notify_offset_multiplier); + /* Read notify length and offset from config space. */ + pci_read_config_dword(pci_dev, + notify + offsetof(struct virtio_pci_notify_cap, + cap.length), + ¬ify_length); + + pci_read_config_dword(pci_dev, + notify + offsetof(struct virtio_pci_notify_cap, + cap.offset), + ¬ify_offset); + + /* We don't know how many VQs we'll map, ahead of the time. + * If notify length is small, map it all now. + * Otherwise, map each VQ individually later. + */ + if ((u64)notify_length + (notify_offset % PAGE_SIZE) <= PAGE_SIZE) { + mdev->notify_base = vp_modern_map_capability(mdev, notify, + 2, 2, + 0, notify_length, + &mdev->notify_len); + if (!mdev->notify_base) + goto err_map_notify; + } else { + mdev->notify_map_cap = notify; + } + + /* Again, we don't know how much we should map, but PAGE_SIZE + * is more than enough for all existing devices. + */ + if (device) { + mdev->device = vp_modern_map_capability(mdev, device, 0, 4, + 0, PAGE_SIZE, + &mdev->device_len); + if (!mdev->device) + goto err_map_device; + } + + return 0; + +err_map_device: + if (mdev->notify_base) + pci_iounmap(pci_dev, mdev->notify_base); +err_map_notify: + pci_iounmap(pci_dev, mdev->isr); +err_map_isr: + pci_iounmap(pci_dev, mdev->common); +err_map_common: + return err; +} +EXPORT_SYMBOL_GPL(vp_modern_probe); + +/* + * vp_modern_probe: remove and cleanup the modern virtio pci device + * @mdev: the modern virtio-pci device + */ +void vp_modern_remove(struct virtio_pci_modern_device *mdev) +{ + struct pci_dev *pci_dev = mdev->pci_dev; + + if (mdev->device) + pci_iounmap(pci_dev, mdev->device); + if (mdev->notify_base) + pci_iounmap(pci_dev, mdev->notify_base); + pci_iounmap(pci_dev, mdev->isr); + pci_iounmap(pci_dev, mdev->common); + pci_release_selected_regions(pci_dev, mdev->modern_bars); +} +EXPORT_SYMBOL_GPL(vp_modern_remove); + +/* + * vp_modern_get_features - get features from device + * @mdev: the modern virtio-pci device + * + * Returns the features read from the device + */ +u64 vp_modern_get_features(struct virtio_pci_modern_device *mdev) +{ + struct virtio_pci_common_cfg __iomem *cfg = mdev->common; + + u64 features; + + vp_iowrite32(0, &cfg->device_feature_select); + features = vp_ioread32(&cfg->device_feature); + vp_iowrite32(1, &cfg->device_feature_select); + features |= ((u64)vp_ioread32(&cfg->device_feature) << 32); + + return features; +} +EXPORT_SYMBOL_GPL(vp_modern_get_features); + +/* + * vp_modern_set_features - set features to device + * @mdev: the modern virtio-pci device + * @features: the features set to device + */ +void vp_modern_set_features(struct virtio_pci_modern_device *mdev, + u64 features) +{ + struct virtio_pci_common_cfg __iomem *cfg = mdev->common; + + vp_iowrite32(0, &cfg->guest_feature_select); + vp_iowrite32((u32)features, &cfg->guest_feature); + vp_iowrite32(1, &cfg->guest_feature_select); + vp_iowrite32(features >> 32, &cfg->guest_feature); +} +EXPORT_SYMBOL_GPL(vp_modern_set_features); + +/* + * vp_modern_generation - get the device genreation + * @mdev: the modern virtio-pci device + * + * Returns the genreation read from device + */ +u32 vp_modern_generation(struct virtio_pci_modern_device *mdev) +{ + struct virtio_pci_common_cfg __iomem *cfg = mdev->common; + + return vp_ioread8(&cfg->config_generation); +} +EXPORT_SYMBOL_GPL(vp_modern_generation); + +/* + * vp_modern_get_status - get the device status + * @mdev: the modern virtio-pci device + * + * Returns the status read from device + */ +u8 vp_modern_get_status(struct virtio_pci_modern_device *mdev) +{ + struct virtio_pci_common_cfg __iomem *cfg = mdev->common; + + return vp_ioread8(&cfg->device_status); +} +EXPORT_SYMBOL_GPL(vp_modern_get_status); + +/* + * vp_modern_set_status - set status to device + * @mdev: the modern virtio-pci device + * @status: the status set to device + */ +void vp_modern_set_status(struct virtio_pci_modern_device *mdev, + u8 status) +{ + struct virtio_pci_common_cfg __iomem *cfg = mdev->common; + + vp_iowrite8(status, &cfg->device_status); +} +EXPORT_SYMBOL_GPL(vp_modern_set_status); + +/* + * vp_modern_queue_vector - set the MSIX vector for a specific virtqueue + * @mdev: the modern virtio-pci device + * @index: queue index + * @vector: the config vector + * + * Returns the config vector read from the device + */ +u16 vp_modern_queue_vector(struct virtio_pci_modern_device *mdev, + u16 index, u16 vector) +{ + struct virtio_pci_common_cfg __iomem *cfg = mdev->common; + + vp_iowrite16(index, &cfg->queue_select); + vp_iowrite16(vector, &cfg->queue_msix_vector); + /* Flush the write out to device */ + return vp_ioread16(&cfg->queue_msix_vector); +} +EXPORT_SYMBOL_GPL(vp_modern_queue_vector); + +/* + * vp_modern_config_vector - set the vector for config interrupt + * @mdev: the modern virtio-pci device + * @vector: the config vector + * + * Returns the config vector read from the device + */ +u16 vp_modern_config_vector(struct virtio_pci_modern_device *mdev, + u16 vector) +{ + struct virtio_pci_common_cfg __iomem *cfg = mdev->common; + + /* Setup the vector used for configuration events */ + vp_iowrite16(vector, &cfg->msix_config); + /* Verify we had enough resources to assign the vector */ + /* Will also flush the write out to device */ + return vp_ioread16(&cfg->msix_config); +} +EXPORT_SYMBOL_GPL(vp_modern_config_vector); + +/* + * vp_modern_queue_address - set the virtqueue address + * @mdev: the modern virtio-pci device + * @index: the queue index + * @desc_addr: address of the descriptor area + * @driver_addr: address of the driver area + * @device_addr: address of the device area + */ +void vp_modern_queue_address(struct virtio_pci_modern_device *mdev, + u16 index, u64 desc_addr, u64 driver_addr, + u64 device_addr) +{ + struct virtio_pci_common_cfg __iomem *cfg = mdev->common; + + vp_iowrite16(index, &cfg->queue_select); + + vp_iowrite64_twopart(desc_addr, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); + vp_iowrite64_twopart(driver_addr, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); + vp_iowrite64_twopart(device_addr, &cfg->queue_used_lo, + &cfg->queue_used_hi); +} +EXPORT_SYMBOL_GPL(vp_modern_queue_address); + +/* + * vp_modern_set_queue_enable - enable a virtqueue + * @mdev: the modern virtio-pci device + * @index: the queue index + * @enable: whether the virtqueue is enable or not + */ +void vp_modern_set_queue_enable(struct virtio_pci_modern_device *mdev, + u16 index, bool enable) +{ + vp_iowrite16(index, &mdev->common->queue_select); + vp_iowrite16(enable, &mdev->common->queue_enable); +} +EXPORT_SYMBOL_GPL(vp_modern_set_queue_enable); + +/* + * vp_modern_get_queue_enable - enable a virtqueue + * @mdev: the modern virtio-pci device + * @index: the queue index + * + * Returns whether a virtqueue is enabled or not + */ +bool vp_modern_get_queue_enable(struct virtio_pci_modern_device *mdev, + u16 index) +{ + vp_iowrite16(index, &mdev->common->queue_select); + + return vp_ioread16(&mdev->common->queue_enable); +} +EXPORT_SYMBOL_GPL(vp_modern_get_queue_enable); + +/* + * vp_modern_set_queue_size - set size for a virtqueue + * @mdev: the modern virtio-pci device + * @index: the queue index + * @size: the size of the virtqueue + */ +void vp_modern_set_queue_size(struct virtio_pci_modern_device *mdev, + u16 index, u16 size) +{ + vp_iowrite16(index, &mdev->common->queue_select); + vp_iowrite16(size, &mdev->common->queue_size); + +} +EXPORT_SYMBOL_GPL(vp_modern_set_queue_size); + +/* + * vp_modern_get_queue_size - get size for a virtqueue + * @mdev: the modern virtio-pci device + * @index: the queue index + * + * Returns the size of the virtqueue + */ +u16 vp_modern_get_queue_size(struct virtio_pci_modern_device *mdev, + u16 index) +{ + vp_iowrite16(index, &mdev->common->queue_select); + + return vp_ioread16(&mdev->common->queue_size); + +} +EXPORT_SYMBOL_GPL(vp_modern_get_queue_size); + +/* + * vp_modern_get_num_queues - get the number of virtqueues + * @mdev: the modern virtio-pci device + * + * Returns the number of virtqueues + */ +u16 vp_modern_get_num_queues(struct virtio_pci_modern_device *mdev) +{ + return vp_ioread16(&mdev->common->num_queues); +} +EXPORT_SYMBOL_GPL(vp_modern_get_num_queues); + +/* + * vp_modern_get_queue_notify_off - get notification offset for a virtqueue + * @mdev: the modern virtio-pci device + * @index: the queue index + * + * Returns the notification offset for a virtqueue + */ +u16 vp_modern_get_queue_notify_off(struct virtio_pci_modern_device *mdev, + u16 index) +{ + vp_iowrite16(index, &mdev->common->queue_select); + + return vp_ioread16(&mdev->common->queue_notify_off); +} +EXPORT_SYMBOL_GPL(vp_modern_get_queue_notify_off); + +MODULE_VERSION("0.1"); +MODULE_DESCRIPTION("Modern Virtio PCI Device"); +MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 4a9ddb44b2a7..e28acf482e0c 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -225,9 +225,8 @@ static void virtio_vdpa_del_vq(struct virtqueue *vq) list_del(&info->node); spin_unlock_irqrestore(&vd_dev->lock, flags); - /* Select and deactivate the queue */ + /* Select and deactivate the queue (best effort) */ ops->set_vq_ready(vdpa, index, 0); - WARN_ON(ops->get_vq_ready(vdpa, index)); vring_del_virtqueue(vq); diff --git a/drivers/vme/vme.c b/drivers/vme/vme.c index 54d7963c1078..1b15afea28ee 100644 --- a/drivers/vme/vme.c +++ b/drivers/vme/vme.c @@ -1997,9 +1997,9 @@ static int vme_bus_remove(struct device *dev) driver = dev->platform_data; if (driver->remove) - return driver->remove(vdev); + driver->remove(vdev); - return -ENODEV; + return 0; } struct bus_type vme_bus_type = { diff --git a/drivers/w1/masters/ds2490.c b/drivers/w1/masters/ds2490.c index e17c8f70dcd0..cd8821580f71 100644 --- a/drivers/w1/masters/ds2490.c +++ b/drivers/w1/masters/ds2490.c @@ -688,12 +688,22 @@ static void ds9490r_search(void *data, struct w1_master *master, * packet size. */ const size_t bufsize = 2 * 64; - u64 *buf; + u64 *buf, *found_ids; buf = kmalloc(bufsize, GFP_KERNEL); if (!buf) return; + /* + * We are holding the bus mutex during the scan, but adding devices via the + * callback needs the bus to be unlocked. So we queue up found ids here. + */ + found_ids = kmalloc_array(master->max_slave_count, sizeof(u64), GFP_KERNEL); + if (!found_ids) { + kfree(buf); + return; + } + mutex_lock(&master->bus_mutex); /* address to start searching at */ @@ -729,13 +739,13 @@ static void ds9490r_search(void *data, struct w1_master *master, if (err < 0) break; for (i = 0; i < err/8; ++i) { - ++found; - if (found <= search_limit) - callback(master, buf[i]); + found_ids[found++] = buf[i]; /* can't know if there will be a discrepancy * value after until the next id */ - if (found == search_limit) + if (found == search_limit) { master->search_id = buf[i]; + break; + } } } @@ -759,9 +769,14 @@ static void ds9490r_search(void *data, struct w1_master *master, master->max_slave_count); set_bit(W1_WARN_MAX_COUNT, &master->flags); } + search_out: mutex_unlock(&master->bus_mutex); kfree(buf); + + for (i = 0; i < found; i++) /* run callback for all queued up IDs */ + callback(master, found_ids[i]); + kfree(found_ids); } #if 0 diff --git a/drivers/w1/slaves/w1_therm.c b/drivers/w1/slaves/w1_therm.c index 3712b1e6dc71..976eea28f268 100644 --- a/drivers/w1/slaves/w1_therm.c +++ b/drivers/w1/slaves/w1_therm.c @@ -667,28 +667,24 @@ static inline int w1_DS18B20_get_resolution(struct w1_slave *sl) */ static inline int w1_DS18B20_convert_temp(u8 rom[9]) { - int t; - u32 bv; + u16 bv; + s16 t; + + /* Signed 16-bit value to unsigned, cpu order */ + bv = le16_to_cpup((__le16 *)rom); /* Config register bit R2 = 1 - GX20MH01 in 13 or 14 bit resolution mode */ if (rom[4] & 0x80) { - /* Signed 16-bit value to unsigned, cpu order */ - bv = le16_to_cpup((__le16 *)rom); - /* Insert two temperature bits from config register */ /* Avoid arithmetic shift of signed value */ bv = (bv << 2) | (rom[4] & 3); - - t = (int) sign_extend32(bv, 17); /* Degrees, lowest bit is 2^-6 */ - return (t*1000)/64; /* Millidegrees */ + t = (s16) bv; /* Degrees, lowest bit is 2^-6 */ + return (int)t * 1000 / 64; /* Sign-extend to int; millidegrees */ } - - t = (int)le16_to_cpup((__le16 *)rom); - return t*1000/16; + t = (s16)bv; /* Degrees, lowest bit is 2^-4 */ + return (int)t * 1000 / 16; /* Sign-extend to int; millidegrees */ } - - /** * w1_DS18S20_convert_temp() - temperature computation for DS18S20 * @rom: data read from device RAM (8 data bytes + 1 CRC byte) diff --git a/drivers/w1/w1.c b/drivers/w1/w1.c index 15a2ee32f116..f2ae2e563dc5 100644 --- a/drivers/w1/w1.c +++ b/drivers/w1/w1.c @@ -25,6 +25,8 @@ #include "w1_netlink.h" #define W1_FAMILY_DEFAULT 0 +#define W1_FAMILY_DS28E04 0x1C /* for crc quirk */ + static int w1_timeout = 10; module_param_named(timeout, w1_timeout, int, 0); @@ -913,11 +915,44 @@ void w1_reconnect_slaves(struct w1_family *f, int attach) mutex_unlock(&w1_mlock); } +static int w1_addr_crc_is_valid(struct w1_master *dev, u64 rn) +{ + u64 rn_le = cpu_to_le64(rn); + struct w1_reg_num *tmp = (struct w1_reg_num *)&rn; + u8 crc; + + crc = w1_calc_crc8((u8 *)&rn_le, 7); + + /* quirk: + * DS28E04 (1w eeprom) has strapping pins to change + * address, but will not update the crc. So normal rules + * for consistent w1 addresses are violated. We test + * with the 7 LSBs of the address forced high. + * + * (char*)&rn_le = { family, addr_lsb, ..., addr_msb, crc }. + */ + if (crc != tmp->crc && tmp->family == W1_FAMILY_DS28E04) { + u64 corr_le = rn_le; + + ((u8 *)&corr_le)[1] |= 0x7f; + crc = w1_calc_crc8((u8 *)&corr_le, 7); + + dev_info(&dev->dev, "DS28E04 crc workaround on %02x.%012llx.%02x\n", + tmp->family, (unsigned long long)tmp->id, tmp->crc); + } + + if (crc != tmp->crc) { + dev_dbg(&dev->dev, "w1 addr crc mismatch: %02x.%012llx.%02x != 0x%02x.\n", + tmp->family, (unsigned long long)tmp->id, tmp->crc, crc); + return 0; + } + return 1; +} + void w1_slave_found(struct w1_master *dev, u64 rn) { struct w1_slave *sl; struct w1_reg_num *tmp; - u64 rn_le = cpu_to_le64(rn); atomic_inc(&dev->refcnt); @@ -927,7 +962,7 @@ void w1_slave_found(struct w1_master *dev, u64 rn) if (sl) { set_bit(W1_SLAVE_ACTIVE, &sl->flags); } else { - if (rn && tmp->crc == w1_calc_crc8((u8 *)&rn_le, 7)) + if (rn && w1_addr_crc_is_valid(dev, rn)) w1_attach_slave_device(dev, tmp); } diff --git a/drivers/watchdog/mei_wdt.c b/drivers/watchdog/mei_wdt.c index c5967d8b4256..e023d7d90d66 100644 --- a/drivers/watchdog/mei_wdt.c +++ b/drivers/watchdog/mei_wdt.c @@ -620,7 +620,7 @@ err_out: return ret; } -static int mei_wdt_remove(struct mei_cl_device *cldev) +static void mei_wdt_remove(struct mei_cl_device *cldev) { struct mei_wdt *wdt = mei_cldev_get_drvdata(cldev); @@ -637,8 +637,6 @@ static int mei_wdt_remove(struct mei_cl_device *cldev) dbgfs_unregister(wdt); kfree(wdt); - - return 0; } #define MEI_UUID_WD UUID_LE(0x05B79A6F, 0x4628, 0x4D7F, \ diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index b57b2067ecbf..671c71245a7b 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -331,7 +331,7 @@ static enum bp_state reserve_additional_memory(void) mutex_unlock(&balloon_mutex); /* add_memory_resource() requires the device_hotplug lock */ lock_device_hotplug(); - rc = add_memory_resource(nid, resource, MEMHP_MERGE_RESOURCE); + rc = add_memory_resource(nid, resource, MHP_MERGE_RESOURCE); unlock_device_hotplug(); mutex_lock(&balloon_mutex); diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c index b249f2d6b0cc..adb7260e94b2 100644 --- a/drivers/xen/events/events_base.c +++ b/drivers/xen/events/events_base.c @@ -323,6 +323,8 @@ static int xen_irq_info_evtchn_setup(unsigned irq, ret = xen_irq_info_common_setup(info, irq, IRQT_EVTCHN, evtchn, 0); info->u.interdomain = dev; + if (dev) + atomic_inc(&dev->event_channels); return ret; } @@ -568,18 +570,28 @@ static void xen_irq_lateeoi_locked(struct irq_info *info, bool spurious) return; if (spurious) { + struct xenbus_device *dev = info->u.interdomain; + unsigned int threshold = 1; + + if (dev && dev->spurious_threshold) + threshold = dev->spurious_threshold; + if ((1 << info->spurious_cnt) < (HZ << 2)) { if (info->spurious_cnt != 0xFF) info->spurious_cnt++; } - if (info->spurious_cnt > 1) { - delay = 1 << (info->spurious_cnt - 2); + if (info->spurious_cnt > threshold) { + delay = 1 << (info->spurious_cnt - 1 - threshold); if (delay > HZ) delay = HZ; if (!info->eoi_time) info->eoi_cpu = smp_processor_id(); info->eoi_time = get_jiffies_64() + delay; + if (dev) + atomic_add(delay, &dev->jiffies_eoi_delayed); } + if (dev) + atomic_inc(&dev->spurious_events); } else { info->spurious_cnt = 0; } @@ -908,6 +920,7 @@ static void __unbind_from_irq(unsigned int irq) if (VALID_EVTCHN(evtchn)) { unsigned int cpu = cpu_from_irq(irq); + struct xenbus_device *dev; xen_evtchn_close(evtchn); @@ -918,6 +931,11 @@ static void __unbind_from_irq(unsigned int irq) case IRQT_IPI: per_cpu(ipi_to_irq, cpu)[ipi_from_irq(irq)] = -1; break; + case IRQT_EVTCHN: + dev = info->u.interdomain; + if (dev) + atomic_dec(&dev->event_channels); + break; default: break; } @@ -1581,6 +1599,7 @@ void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl) { int irq; struct irq_info *info; + struct xenbus_device *dev; irq = get_evtchn_to_irq(port); if (irq == -1) @@ -1610,6 +1629,10 @@ void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl) info = info_for_irq(irq); + dev = (info->type == IRQT_EVTCHN) ? info->u.interdomain : NULL; + if (dev) + atomic_inc(&dev->events); + if (ctrl->defer_eoi) { info->eoi_cpu = smp_processor_id(); info->irq_epoch = __this_cpu_read(irq_epoch); diff --git a/drivers/xen/evtchn.c b/drivers/xen/evtchn.c index a7a85719a8c8..c99415a70051 100644 --- a/drivers/xen/evtchn.c +++ b/drivers/xen/evtchn.c @@ -162,6 +162,7 @@ static irqreturn_t evtchn_interrupt(int irq, void *data) { struct user_evtchn *evtchn = data; struct per_user_data *u = evtchn->user; + unsigned int prod, cons; WARN(!evtchn->enabled, "Interrupt for port %u, but apparently not enabled; per-user %p\n", @@ -171,10 +172,14 @@ static irqreturn_t evtchn_interrupt(int irq, void *data) spin_lock(&u->ring_prod_lock); - if ((u->ring_prod - u->ring_cons) < u->ring_size) { - *evtchn_ring_entry(u, u->ring_prod) = evtchn->port; - wmb(); /* Ensure ring contents visible */ - if (u->ring_cons == u->ring_prod++) { + prod = READ_ONCE(u->ring_prod); + cons = READ_ONCE(u->ring_cons); + + if ((prod - cons) < u->ring_size) { + *evtchn_ring_entry(u, prod) = evtchn->port; + smp_wmb(); /* Ensure ring contents visible */ + WRITE_ONCE(u->ring_prod, prod + 1); + if (cons == prod) { wake_up_interruptible(&u->evtchn_wait); kill_fasync(&u->evtchn_async_queue, SIGIO, POLL_IN); @@ -210,8 +215,8 @@ static ssize_t evtchn_read(struct file *file, char __user *buf, if (u->ring_overflow) goto unlock_out; - c = u->ring_cons; - p = u->ring_prod; + c = READ_ONCE(u->ring_cons); + p = READ_ONCE(u->ring_prod); if (c != p) break; @@ -221,7 +226,7 @@ static ssize_t evtchn_read(struct file *file, char __user *buf, return -EAGAIN; rc = wait_event_interruptible(u->evtchn_wait, - u->ring_cons != u->ring_prod); + READ_ONCE(u->ring_cons) != READ_ONCE(u->ring_prod)); if (rc) return rc; } @@ -245,13 +250,13 @@ static ssize_t evtchn_read(struct file *file, char __user *buf, } rc = -EFAULT; - rmb(); /* Ensure that we see the port before we copy it. */ + smp_rmb(); /* Ensure that we see the port before we copy it. */ if (copy_to_user(buf, evtchn_ring_entry(u, c), bytes1) || ((bytes2 != 0) && copy_to_user(&buf[bytes1], &u->ring[0], bytes2))) goto unlock_out; - u->ring_cons += (bytes1 + bytes2) / sizeof(evtchn_port_t); + WRITE_ONCE(u->ring_cons, c + (bytes1 + bytes2) / sizeof(evtchn_port_t)); rc = bytes1 + bytes2; unlock_out: @@ -552,7 +557,9 @@ static long evtchn_ioctl(struct file *file, /* Initialise the ring to empty. Clear errors. */ mutex_lock(&u->ring_cons_mutex); spin_lock_irq(&u->ring_prod_lock); - u->ring_cons = u->ring_prod = u->ring_overflow = 0; + WRITE_ONCE(u->ring_cons, 0); + WRITE_ONCE(u->ring_prod, 0); + u->ring_overflow = 0; spin_unlock_irq(&u->ring_prod_lock); mutex_unlock(&u->ring_cons_mutex); rc = 0; @@ -595,7 +602,7 @@ static __poll_t evtchn_poll(struct file *file, poll_table *wait) struct per_user_data *u = file->private_data; poll_wait(file, &u->evtchn_wait, wait); - if (u->ring_cons != u->ring_prod) + if (READ_ONCE(u->ring_cons) != READ_ONCE(u->ring_prod)) mask |= EPOLLIN | EPOLLRDNORM; if (u->ring_overflow) mask = EPOLLERR; diff --git a/drivers/xen/xen-acpi-processor.c b/drivers/xen/xen-acpi-processor.c index ce8ffb595a46..df7cab870be5 100644 --- a/drivers/xen/xen-acpi-processor.c +++ b/drivers/xen/xen-acpi-processor.c @@ -3,7 +3,8 @@ * Copyright 2012 by Oracle Inc * Author: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> * - * This code borrows ideas from https://lkml.org/lkml/2011/11/30/249 + * This code borrows ideas from + * https://lore.kernel.org/lkml/1322673664-14642-6-git-send-email-konrad.wilk@oracle.com * so many thanks go to Kevin Tian <kevin.tian@intel.com> * and Yu Ke <ke.yu@intel.com>. */ diff --git a/drivers/xen/xen-front-pgdir-shbuf.c b/drivers/xen/xen-front-pgdir-shbuf.c index 48a658dc7ccf..81b6e13fa5ec 100644 --- a/drivers/xen/xen-front-pgdir-shbuf.c +++ b/drivers/xen/xen-front-pgdir-shbuf.c @@ -305,11 +305,18 @@ static int backend_map(struct xen_front_pgdir_shbuf *buf) /* Save handles even if error, so we can unmap. */ for (cur_page = 0; cur_page < buf->num_pages; cur_page++) { - buf->backend_map_handles[cur_page] = map_ops[cur_page].handle; - if (unlikely(map_ops[cur_page].status != GNTST_okay)) + if (likely(map_ops[cur_page].status == GNTST_okay)) { + buf->backend_map_handles[cur_page] = + map_ops[cur_page].handle; + } else { + buf->backend_map_handles[cur_page] = + INVALID_GRANT_HANDLE; + if (!ret) + ret = -ENXIO; dev_err(&buf->xb_dev->dev, "Failed to map page %d: %d\n", cur_page, map_ops[cur_page].status); + } } if (ret) { diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c index 8a75092bb148..97f0d234482d 100644 --- a/drivers/xen/xenbus/xenbus_probe.c +++ b/drivers/xen/xenbus/xenbus_probe.c @@ -206,6 +206,65 @@ void xenbus_otherend_changed(struct xenbus_watch *watch, } EXPORT_SYMBOL_GPL(xenbus_otherend_changed); +#define XENBUS_SHOW_STAT(name) \ +static ssize_t show_##name(struct device *_dev, \ + struct device_attribute *attr, \ + char *buf) \ +{ \ + struct xenbus_device *dev = to_xenbus_device(_dev); \ + \ + return sprintf(buf, "%d\n", atomic_read(&dev->name)); \ +} \ +static DEVICE_ATTR(name, 0444, show_##name, NULL) + +XENBUS_SHOW_STAT(event_channels); +XENBUS_SHOW_STAT(events); +XENBUS_SHOW_STAT(spurious_events); +XENBUS_SHOW_STAT(jiffies_eoi_delayed); + +static ssize_t show_spurious_threshold(struct device *_dev, + struct device_attribute *attr, + char *buf) +{ + struct xenbus_device *dev = to_xenbus_device(_dev); + + return sprintf(buf, "%d\n", dev->spurious_threshold); +} + +static ssize_t set_spurious_threshold(struct device *_dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct xenbus_device *dev = to_xenbus_device(_dev); + unsigned int val; + ssize_t ret; + + ret = kstrtouint(buf, 0, &val); + if (ret) + return ret; + + dev->spurious_threshold = val; + + return count; +} + +static DEVICE_ATTR(spurious_threshold, 0644, show_spurious_threshold, + set_spurious_threshold); + +static struct attribute *xenbus_attrs[] = { + &dev_attr_event_channels.attr, + &dev_attr_events.attr, + &dev_attr_spurious_events.attr, + &dev_attr_jiffies_eoi_delayed.attr, + &dev_attr_spurious_threshold.attr, + NULL +}; + +static const struct attribute_group xenbus_group = { + .name = "xenbus", + .attrs = xenbus_attrs, +}; + int xenbus_dev_probe(struct device *_dev) { struct xenbus_device *dev = to_xenbus_device(_dev); @@ -253,6 +312,11 @@ int xenbus_dev_probe(struct device *_dev) return err; } + dev->spurious_threshold = 1; + if (sysfs_create_group(&dev->dev.kobj, &xenbus_group)) + dev_warn(&dev->dev, "sysfs_create_group on %s failed.\n", + dev->nodename); + return 0; fail_put: module_put(drv->driver.owner); @@ -269,6 +333,8 @@ int xenbus_dev_remove(struct device *_dev) DPRINTK("%s", dev->nodename); + sysfs_remove_group(&dev->dev.kobj, &xenbus_group); + free_otherend_watch(dev); if (drv->remove) { |