diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2021-07-05 13:42:16 -0700 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2021-07-05 13:42:16 -0700 |
| commit | eed0218e8cae9fcd186c30e9fcf5fe46a87e056e (patch) | |
| tree | 799a1360b947a56d05a60433fdf60a96bf3b3348 /drivers/misc | |
| parent | 3f8b8e7dbd79086ad48fcff33de9399f3da66a69 (diff) | |
| parent | 6f746d485fb9188dc67dce7de63d21f0c28a1f2e (diff) | |
Merge tag 'char-misc-5.14-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc
Pull char / misc driver updates from Greg KH:
"Here is the big set of char / misc and other driver subsystem updates
for 5.14-rc1. Included in here are:
- habanalabs driver updates
- fsl-mc driver updates
- comedi driver updates
- fpga driver updates
- extcon driver updates
- interconnect driver updates
- mei driver updates
- nvmem driver updates
- phy driver updates
- pnp driver updates
- soundwire driver updates
- lots of other tiny driver updates for char and misc drivers
This is looking more and more like the "various driver subsystems
mushed together" tree...
All of these have been in linux-next for a while with no reported
issues"
* tag 'char-misc-5.14-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc: (292 commits)
mcb: Use DEFINE_RES_MEM() helper macro and fix the end address
PNP: moved EXPORT_SYMBOL so that it immediately followed its function/variable
bus: mhi: pci-generic: Add missing 'pci_disable_pcie_error_reporting()' calls
bus: mhi: Wait for M2 state during system resume
bus: mhi: core: Fix power down latency
intel_th: Wait until port is in reset before programming it
intel_th: msu: Make contiguous buffers uncached
intel_th: Remove an unused exit point from intel_th_remove()
stm class: Spelling fix
nitro_enclaves: Set Bus Master for the NE PCI device
misc: ibmasm: Modify matricies to matrices
misc: vmw_vmci: return the correct errno code
siox: Simplify error handling via dev_err_probe()
fpga: machxo2-spi: Address warning about unused variable
lkdtm/heap: Add init_on_alloc tests
selftests/lkdtm: Enable various testable CONFIGs
lkdtm: Add CONFIG hints in errors where possible
lkdtm: Enable DOUBLE_FAULT on all architectures
lkdtm/heap: Add vmalloc linear overflow test
lkdtm/bugs: XFAIL UNALIGNED_LOAD_STORE_WRITE
...
Diffstat (limited to 'drivers/misc')
65 files changed, 3799 insertions, 1236 deletions
diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c index f40cf08a6192..066b9ef7fcd7 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.c +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -354,8 +354,7 @@ static void bcm_vk_drain_all_pend(struct device *dev, for (num = 0; num < chan->q_nr; num++) { list_for_each_entry_safe(entry, tmp, &chan->pendq[num], node) { if ((!ctx) || (entry->ctx->idx == ctx->idx)) { - list_del(&entry->node); - list_add_tail(&entry->node, &del_q); + list_move_tail(&entry->node, &del_q); } } } @@ -701,8 +700,7 @@ int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type, return -EINVAL; } - entry = kzalloc(sizeof(*entry) + - sizeof(struct vk_msg_blk), GFP_KERNEL); + entry = kzalloc(struct_size(entry, to_v_msg, 1), GFP_KERNEL); if (!entry) return -ENOMEM; diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.h b/drivers/misc/bcm-vk/bcm_vk_msg.h index 4eaad84825d6..56784c8896d8 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.h +++ b/drivers/misc/bcm-vk/bcm_vk_msg.h @@ -116,7 +116,7 @@ struct bcm_vk_wkent { u32 usr_msg_id; u32 to_v_blks; u32 seq_num; - struct vk_msg_blk to_v_msg[0]; + struct vk_msg_blk to_v_msg[]; }; /* queue stats counters */ diff --git a/drivers/misc/cardreader/alcor_pci.c b/drivers/misc/cardreader/alcor_pci.c index cd402c89189e..de6d44a158bb 100644 --- a/drivers/misc/cardreader/alcor_pci.c +++ b/drivers/misc/cardreader/alcor_pci.c @@ -139,7 +139,13 @@ static void alcor_pci_init_check_aspm(struct alcor_pci_priv *priv) u32 val32; priv->pdev_cap_off = alcor_pci_find_cap_offset(priv, priv->pdev); - priv->parent_cap_off = alcor_pci_find_cap_offset(priv, + /* + * A device might be attached to root complex directly and + * priv->parent_pdev will be NULL. In this case we don't check its + * capability and disable ASPM completely. + */ + if (priv->parent_pdev) + priv->parent_cap_off = alcor_pci_find_cap_offset(priv, priv->parent_pdev); if ((priv->pdev_cap_off == 0) || (priv->parent_cap_off == 0)) { diff --git a/drivers/misc/cxl/file.c b/drivers/misc/cxl/file.c index bd3bd32333c5..3dbdce96fae0 100644 --- a/drivers/misc/cxl/file.c +++ b/drivers/misc/cxl/file.c @@ -569,7 +569,8 @@ static int cxl_add_chardev(struct cxl_afu *afu, dev_t devt, struct cdev *cdev, int rc; cdev_init(cdev, fops); - if ((rc = cdev_add(cdev, devt, 1))) { + rc = cdev_add(cdev, devt, 1); + if (rc) { dev_err(&afu->dev, "Unable to add %s chardev: %i\n", desc, rc); return rc; } @@ -577,8 +578,8 @@ static int cxl_add_chardev(struct cxl_afu *afu, dev_t devt, struct cdev *cdev, dev = device_create(cxl_class, &afu->dev, devt, afu, "afu%i.%i%s", afu->adapter->adapter_num, afu->slice, postfix); if (IS_ERR(dev)) { - dev_err(&afu->dev, "Unable to create %s chardev in sysfs: %i\n", desc, rc); rc = PTR_ERR(dev); + dev_err(&afu->dev, "Unable to create %s chardev in sysfs: %i\n", desc, rc); goto err; } diff --git a/drivers/misc/eeprom/Kconfig b/drivers/misc/eeprom/Kconfig index 0f791bfdc1f5..f0a7531f354c 100644 --- a/drivers/misc/eeprom/Kconfig +++ b/drivers/misc/eeprom/Kconfig @@ -32,12 +32,13 @@ config EEPROM_AT24 will be called at24. config EEPROM_AT25 - tristate "SPI EEPROMs from most vendors" + tristate "SPI EEPROMs (FRAMs) from most vendors" depends on SPI && SYSFS select NVMEM select NVMEM_SYSFS help - Enable this driver to get read/write support to most SPI EEPROMs, + Enable this driver to get read/write support to most SPI EEPROMs + and Cypress FRAMs, after you configure the board init code to know about each eeprom on your target board. diff --git a/drivers/misc/eeprom/at25.c b/drivers/misc/eeprom/at25.c index b76e4901b4a4..4d09b672ac3c 100644 --- a/drivers/misc/eeprom/at25.c +++ b/drivers/misc/eeprom/at25.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* * at25.c -- support most SPI EEPROMs, such as Atmel AT25 models + * and Cypress FRAMs FM25 models * * Copyright (C) 2006 David Brownell */ @@ -16,6 +17,9 @@ #include <linux/spi/spi.h> #include <linux/spi/eeprom.h> #include <linux/property.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/math.h> /* * NOTE: this is an *EEPROM* driver. The vagaries of product naming @@ -27,6 +31,7 @@ * AT25M02, AT25128B */ +#define FM25_SN_LEN 8 /* serial number length */ struct at25_data { struct spi_device *spi; struct mutex lock; @@ -34,6 +39,7 @@ struct at25_data { unsigned addrlen; struct nvmem_config nvmem_config; struct nvmem_device *nvmem; + u8 sernum[FM25_SN_LEN]; }; #define AT25_WREN 0x06 /* latch the write enable */ @@ -42,6 +48,9 @@ struct at25_data { #define AT25_WRSR 0x01 /* write status register */ #define AT25_READ 0x03 /* read byte(s) */ #define AT25_WRITE 0x02 /* write byte(s)/sector */ +#define FM25_SLEEP 0xb9 /* enter sleep mode */ +#define FM25_RDID 0x9f /* read device ID */ +#define FM25_RDSN 0xc3 /* read S/N */ #define AT25_SR_nRDY 0x01 /* nRDY = write-in-progress */ #define AT25_SR_WEN 0x02 /* write enable (latched) */ @@ -51,6 +60,8 @@ struct at25_data { #define AT25_INSTR_BIT3 0x08 /* Additional address bit in instr */ +#define FM25_ID_LEN 9 /* ID length */ + #define EE_MAXADDRLEN 3 /* 24 bit addresses, up to 2 MBytes */ /* Specs often allow 5 msec for a page write, sometimes 20 msec; @@ -129,6 +140,51 @@ static int at25_ee_read(void *priv, unsigned int offset, return status; } +/* + * read extra registers as ID or serial number + */ +static int fm25_aux_read(struct at25_data *at25, u8 *buf, uint8_t command, + int len) +{ + int status; + struct spi_transfer t[2]; + struct spi_message m; + + spi_message_init(&m); + memset(t, 0, sizeof(t)); + + t[0].tx_buf = &command; + t[0].len = 1; + spi_message_add_tail(&t[0], &m); + + t[1].rx_buf = buf; + t[1].len = len; + spi_message_add_tail(&t[1], &m); + + mutex_lock(&at25->lock); + + status = spi_sync(at25->spi, &m); + dev_dbg(&at25->spi->dev, "read %d aux bytes --> %d\n", len, status); + + mutex_unlock(&at25->lock); + return status; +} + +static ssize_t sernum_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct at25_data *at25; + + at25 = dev_get_drvdata(dev); + return sysfs_emit(buf, "%*ph\n", (int)sizeof(at25->sernum), at25->sernum); +} +static DEVICE_ATTR_RO(sernum); + +static struct attribute *sernum_attrs[] = { + &dev_attr_sernum.attr, + NULL, +}; +ATTRIBUTE_GROUPS(sernum); + static int at25_ee_write(void *priv, unsigned int off, void *val, size_t count) { struct at25_data *at25 = priv; @@ -303,34 +359,39 @@ static int at25_fw_to_chip(struct device *dev, struct spi_eeprom *chip) return 0; } +static const struct of_device_id at25_of_match[] = { + { .compatible = "atmel,at25",}, + { .compatible = "cypress,fm25",}, + { } +}; +MODULE_DEVICE_TABLE(of, at25_of_match); + static int at25_probe(struct spi_device *spi) { struct at25_data *at25 = NULL; struct spi_eeprom chip; int err; int sr; - int addrlen; + u8 id[FM25_ID_LEN]; + u8 sernum[FM25_SN_LEN]; + int i; + const struct of_device_id *match; + bool is_fram = 0; + + match = of_match_device(of_match_ptr(at25_of_match), &spi->dev); + if (match && !strcmp(match->compatible, "cypress,fm25")) + is_fram = 1; /* Chip description */ if (!spi->dev.platform_data) { - err = at25_fw_to_chip(&spi->dev, &chip); - if (err) - return err; + if (!is_fram) { + err = at25_fw_to_chip(&spi->dev, &chip); + if (err) + return err; + } } else chip = *(struct spi_eeprom *)spi->dev.platform_data; - /* For now we only support 8/16/24 bit addressing */ - if (chip.flags & EE_ADDR1) - addrlen = 1; - else if (chip.flags & EE_ADDR2) - addrlen = 2; - else if (chip.flags & EE_ADDR3) - addrlen = 3; - else { - dev_dbg(&spi->dev, "unsupported address type\n"); - return -EINVAL; - } - /* Ping the chip ... the status register is pretty portable, * unlike probing manufacturer IDs. We do expect that system * firmware didn't write it in the past few milliseconds! @@ -349,9 +410,51 @@ static int at25_probe(struct spi_device *spi) at25->chip = chip; at25->spi = spi; spi_set_drvdata(spi, at25); - at25->addrlen = addrlen; - at25->nvmem_config.type = NVMEM_TYPE_EEPROM; + if (is_fram) { + /* Get ID of chip */ + fm25_aux_read(at25, id, FM25_RDID, FM25_ID_LEN); + if (id[6] != 0xc2) { + dev_err(&spi->dev, + "Error: no Cypress FRAM (id %02x)\n", id[6]); + return -ENODEV; + } + /* set size found in ID */ + if (id[7] < 0x21 || id[7] > 0x26) { + dev_err(&spi->dev, "Error: unsupported size (id %02x)\n", id[7]); + return -ENODEV; + } + chip.byte_len = int_pow(2, id[7] - 0x21 + 4) * 1024; + + if (at25->chip.byte_len > 64 * 1024) + at25->chip.flags |= EE_ADDR3; + else + at25->chip.flags |= EE_ADDR2; + + if (id[8]) { + fm25_aux_read(at25, sernum, FM25_RDSN, FM25_SN_LEN); + /* swap byte order */ + for (i = 0; i < FM25_SN_LEN; i++) + at25->sernum[i] = sernum[FM25_SN_LEN - 1 - i]; + } + + at25->chip.page_size = PAGE_SIZE; + strncpy(at25->chip.name, "fm25", sizeof(at25->chip.name)); + } + + /* For now we only support 8/16/24 bit addressing */ + if (at25->chip.flags & EE_ADDR1) + at25->addrlen = 1; + else if (at25->chip.flags & EE_ADDR2) + at25->addrlen = 2; + else if (at25->chip.flags & EE_ADDR3) + at25->addrlen = 3; + else { + dev_dbg(&spi->dev, "unsupported address type\n"); + return -EINVAL; + } + + at25->nvmem_config.type = is_fram ? NVMEM_TYPE_FRAM : NVMEM_TYPE_EEPROM; at25->nvmem_config.name = dev_name(&spi->dev); at25->nvmem_config.dev = &spi->dev; at25->nvmem_config.read_only = chip.flags & EE_READONLY; @@ -370,27 +473,22 @@ static int at25_probe(struct spi_device *spi) if (IS_ERR(at25->nvmem)) return PTR_ERR(at25->nvmem); - dev_info(&spi->dev, "%d %s %s eeprom%s, pagesize %u\n", - (chip.byte_len < 1024) ? chip.byte_len : (chip.byte_len / 1024), - (chip.byte_len < 1024) ? "Byte" : "KByte", - at25->chip.name, - (chip.flags & EE_READONLY) ? " (readonly)" : "", - at25->chip.page_size); + dev_info(&spi->dev, "%d %s %s %s%s, pagesize %u\n", + (chip.byte_len < 1024) ? chip.byte_len : (chip.byte_len / 1024), + (chip.byte_len < 1024) ? "Byte" : "KByte", + at25->chip.name, is_fram ? "fram" : "eeprom", + (chip.flags & EE_READONLY) ? " (readonly)" : "", + at25->chip.page_size); return 0; } /*-------------------------------------------------------------------------*/ -static const struct of_device_id at25_of_match[] = { - { .compatible = "atmel,at25", }, - { } -}; -MODULE_DEVICE_TABLE(of, at25_of_match); - static struct spi_driver at25_driver = { .driver = { .name = "at25", .of_match_table = at25_of_match, + .dev_groups = sernum_groups, }, .probe = at25_probe, }; diff --git a/drivers/misc/eeprom/ee1004.c b/drivers/misc/eeprom/ee1004.c index 252e15ba65e1..bb9c4512c968 100644 --- a/drivers/misc/eeprom/ee1004.c +++ b/drivers/misc/eeprom/ee1004.c @@ -32,16 +32,17 @@ */ #define EE1004_ADDR_SET_PAGE 0x36 -#define EE1004_EEPROM_SIZE 512 +#define EE1004_NUM_PAGES 2 #define EE1004_PAGE_SIZE 256 #define EE1004_PAGE_SHIFT 8 +#define EE1004_EEPROM_SIZE (EE1004_PAGE_SIZE * EE1004_NUM_PAGES) /* * Mutex protects ee1004_set_page and ee1004_dev_count, and must be held * from page selection to end of read. */ static DEFINE_MUTEX(ee1004_bus_lock); -static struct i2c_client *ee1004_set_page[2]; +static struct i2c_client *ee1004_set_page[EE1004_NUM_PAGES]; static unsigned int ee1004_dev_count; static int ee1004_current_page; @@ -71,40 +72,58 @@ static int ee1004_get_current_page(void) return 0; } +static int ee1004_set_current_page(struct device *dev, int page) +{ + int ret; + + if (page == ee1004_current_page) + return 0; + + /* Data is ignored */ + ret = i2c_smbus_write_byte(ee1004_set_page[page], 0x00); + /* + * Don't give up just yet. Some memory modules will select the page + * but not ack the command. Check which page is selected now. + */ + if (ret == -ENXIO && ee1004_get_current_page() == page) + ret = 0; + if (ret < 0) { + dev_err(dev, "Failed to select page %d (%d)\n", page, ret); + return ret; + } + + dev_dbg(dev, "Selected page %d\n", page); + ee1004_current_page = page; + + return 0; +} + static ssize_t ee1004_eeprom_read(struct i2c_client *client, char *buf, unsigned int offset, size_t count) { - int status; + int status, page; + + page = offset >> EE1004_PAGE_SHIFT; + offset &= (1 << EE1004_PAGE_SHIFT) - 1; + + status = ee1004_set_current_page(&client->dev, page); + if (status) + return status; - if (count > I2C_SMBUS_BLOCK_MAX) - count = I2C_SMBUS_BLOCK_MAX; /* Can't cross page boundaries */ - if (unlikely(offset + count > EE1004_PAGE_SIZE)) + if (offset + count > EE1004_PAGE_SIZE) count = EE1004_PAGE_SIZE - offset; - status = i2c_smbus_read_i2c_block_data_or_emulated(client, offset, - count, buf); - dev_dbg(&client->dev, "read %zu@%d --> %d\n", count, offset, status); - - return status; + return i2c_smbus_read_i2c_block_data_or_emulated(client, offset, count, buf); } -static ssize_t ee1004_read(struct file *filp, struct kobject *kobj, +static ssize_t eeprom_read(struct file *filp, struct kobject *kobj, struct bin_attribute *bin_attr, char *buf, loff_t off, size_t count) { - struct device *dev = kobj_to_dev(kobj); - struct i2c_client *client = to_i2c_client(dev); + struct i2c_client *client = kobj_to_i2c_client(kobj); size_t requested = count; - int page; - - if (unlikely(!count)) - return count; - - page = off >> EE1004_PAGE_SHIFT; - if (unlikely(page > 1)) - return 0; - off &= (1 << EE1004_PAGE_SHIFT) - 1; + int ret = 0; /* * Read data from chip, protecting against concurrent access to @@ -113,139 +132,85 @@ static ssize_t ee1004_read(struct file *filp, struct kobject *kobj, mutex_lock(&ee1004_bus_lock); while (count) { - int status; - - /* Select page */ - if (page != ee1004_current_page) { - /* Data is ignored */ - status = i2c_smbus_write_byte(ee1004_set_page[page], - 0x00); - if (status == -ENXIO) { - /* - * Don't give up just yet. Some memory - * modules will select the page but not - * ack the command. Check which page is - * selected now. - */ - if (ee1004_get_current_page() == page) - status = 0; - } - if (status < 0) { - dev_err(dev, "Failed to select page %d (%d)\n", - page, status); - mutex_unlock(&ee1004_bus_lock); - return status; - } - dev_dbg(dev, "Selected page %d\n", page); - ee1004_current_page = page; - } - - status = ee1004_eeprom_read(client, buf, off, count); - if (status < 0) { - mutex_unlock(&ee1004_bus_lock); - return status; - } - buf += status; - off += status; - count -= status; + ret = ee1004_eeprom_read(client, buf, off, count); + if (ret < 0) + goto out; - if (off == EE1004_PAGE_SIZE) { - page++; - off = 0; - } + buf += ret; + off += ret; + count -= ret; } - +out: mutex_unlock(&ee1004_bus_lock); - return requested; + return ret < 0 ? ret : requested; } -static const struct bin_attribute eeprom_attr = { - .attr = { - .name = "eeprom", - .mode = 0444, - }, - .size = EE1004_EEPROM_SIZE, - .read = ee1004_read, +static BIN_ATTR_RO(eeprom, EE1004_EEPROM_SIZE); + +static struct bin_attribute *ee1004_attrs[] = { + &bin_attr_eeprom, + NULL }; -static int ee1004_probe(struct i2c_client *client, - const struct i2c_device_id *id) +BIN_ATTRIBUTE_GROUPS(ee1004); + +static void ee1004_cleanup(int idx) +{ + if (--ee1004_dev_count == 0) + while (--idx >= 0) { + i2c_unregister_device(ee1004_set_page[idx]); + ee1004_set_page[idx] = NULL; + } +} + +static int ee1004_probe(struct i2c_client *client) { int err, cnr = 0; - const char *slow = NULL; /* Make sure we can operate on this adapter */ if (!i2c_check_functionality(client->adapter, - I2C_FUNC_SMBUS_READ_BYTE | - I2C_FUNC_SMBUS_READ_I2C_BLOCK)) { - if (i2c_check_functionality(client->adapter, - I2C_FUNC_SMBUS_READ_BYTE | - I2C_FUNC_SMBUS_READ_WORD_DATA)) - slow = "word"; - else if (i2c_check_functionality(client->adapter, - I2C_FUNC_SMBUS_READ_BYTE | - I2C_FUNC_SMBUS_READ_BYTE_DATA)) - slow = "byte"; - else - return -EPFNOSUPPORT; - } + I2C_FUNC_SMBUS_BYTE | I2C_FUNC_SMBUS_READ_I2C_BLOCK) && + !i2c_check_functionality(client->adapter, + I2C_FUNC_SMBUS_BYTE | I2C_FUNC_SMBUS_READ_BYTE_DATA)) + return -EPFNOSUPPORT; /* Use 2 dummy devices for page select command */ mutex_lock(&ee1004_bus_lock); if (++ee1004_dev_count == 1) { - for (cnr = 0; cnr < 2; cnr++) { - ee1004_set_page[cnr] = i2c_new_dummy_device(client->adapter, - EE1004_ADDR_SET_PAGE + cnr); - if (IS_ERR(ee1004_set_page[cnr])) { - dev_err(&client->dev, - "address 0x%02x unavailable\n", - EE1004_ADDR_SET_PAGE + cnr); - err = PTR_ERR(ee1004_set_page[cnr]); + for (cnr = 0; cnr < EE1004_NUM_PAGES; cnr++) { + struct i2c_client *cl; + + cl = i2c_new_dummy_device(client->adapter, EE1004_ADDR_SET_PAGE + cnr); + if (IS_ERR(cl)) { + err = PTR_ERR(cl); goto err_clients; } + ee1004_set_page[cnr] = cl; } - } else if (i2c_adapter_id(client->adapter) != - i2c_adapter_id(ee1004_set_page[0]->adapter)) { + + /* Remember current page to avoid unneeded page select */ + err = ee1004_get_current_page(); + if (err < 0) + goto err_clients; + dev_dbg(&client->dev, "Currently selected page: %d\n", err); + ee1004_current_page = err; + } else if (client->adapter != ee1004_set_page[0]->adapter) { dev_err(&client->dev, "Driver only supports devices on a single I2C bus\n"); err = -EOPNOTSUPP; goto err_clients; } - - /* Remember current page to avoid unneeded page select */ - err = ee1004_get_current_page(); - if (err < 0) - goto err_clients; - ee1004_current_page = err; - dev_dbg(&client->dev, "Currently selected page: %d\n", - ee1004_current_page); mutex_unlock(&ee1004_bus_lock); - /* Create the sysfs eeprom file */ - err = sysfs_create_bin_file(&client->dev.kobj, &eeprom_attr); - if (err) - goto err_clients_lock; - dev_info(&client->dev, "%u byte EE1004-compliant SPD EEPROM, read-only\n", EE1004_EEPROM_SIZE); - if (slow) - dev_notice(&client->dev, - "Falling back to %s reads, performance will suffer\n", - slow); return 0; - err_clients_lock: - mutex_lock(&ee1004_bus_lock); err_clients: - if (--ee1004_dev_count == 0) { - for (cnr--; cnr >= 0; cnr--) { - i2c_unregister_device(ee1004_set_page[cnr]); - ee1004_set_page[cnr] = NULL; - } - } + ee1004_cleanup(cnr); mutex_unlock(&ee1004_bus_lock); return err; @@ -253,18 +218,9 @@ static int ee1004_probe(struct i2c_client *client, static int ee1004_remove(struct i2c_client *client) { - int i; - - sysfs_remove_bin_file(&client->dev.kobj, &eeprom_attr); - /* Remove page select clients if this is the last device */ mutex_lock(&ee1004_bus_lock); - if (--ee1004_dev_count == 0) { - for (i = 0; i < 2; i++) { - i2c_unregister_device(ee1004_set_page[i]); - ee1004_set_page[i] = NULL; - } - } + ee1004_cleanup(EE1004_NUM_PAGES); mutex_unlock(&ee1004_bus_lock); return 0; @@ -275,8 +231,9 @@ static int ee1004_remove(struct i2c_client *client) static struct i2c_driver ee1004_driver = { .driver = { .name = "ee1004", + .dev_groups = ee1004_groups, }, - .probe = ee1004_probe, + .probe_new = ee1004_probe, .remove = ee1004_remove, .id_table = ee1004_ids, }; diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c index 80114f4c80ad..29d8971ec558 100644 --- a/drivers/misc/eeprom/eeprom_93xx46.c +++ b/drivers/misc/eeprom/eeprom_93xx46.c @@ -9,6 +9,7 @@ #include <linux/device.h> #include <linux/gpio/consumer.h> #include <linux/kernel.h> +#include <linux/log2.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/of.h> @@ -28,14 +29,29 @@ struct eeprom_93xx46_devtype_data { unsigned int quirks; + unsigned char flags; +}; + +static const struct eeprom_93xx46_devtype_data at93c46_data = { + .flags = EE_SIZE1K, +}; + +static const struct eeprom_93xx46_devtype_data at93c56_data = { + .flags = EE_SIZE2K, +}; + +static const struct eeprom_93xx46_devtype_data at93c66_data = { + .flags = EE_SIZE4K, }; static const struct eeprom_93xx46_devtype_data atmel_at93c46d_data = { + .flags = EE_SIZE1K, .quirks = EEPROM_93XX46_QUIRK_SINGLE_WORD_READ | EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH, }; static const struct eeprom_93xx46_devtype_data microchip_93lc46b_data = { + .flags = EE_SIZE1K, .quirks = EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE, }; @@ -70,6 +86,7 @@ static int eeprom_93xx46_read(void *priv, unsigned int off, struct eeprom_93xx46_dev *edev = priv; char *buf = val; int err = 0; + int bits; if (unlikely(off >= edev->size)) return 0; @@ -83,21 +100,21 @@ static int eeprom_93xx46_read(void *priv, unsigned int off, if (edev->pdata->prepare) edev->pdata->prepare(edev); + /* The opcode in front of the address is three bits. */ + bits = edev->addrlen + 3; + while (count) { struct spi_message m; struct spi_transfer t[2] = { { 0 } }; u16 cmd_addr = OP_READ << edev->addrlen; size_t nbytes = count; - int bits; - if (edev->addrlen == 7) { - cmd_addr |= off & 0x7f; - bits = 10; + if (edev->pdata->flags & EE_ADDR8) { + cmd_addr |= off; if (has_quirk_single_word_read(edev)) nbytes = 1; } else { - cmd_addr |= (off >> 1) & 0x3f; - bits = 9; + cmd_addr |= (off >> 1); if (has_quirk_single_word_read(edev)) nbytes = 2; } @@ -152,14 +169,14 @@ static int eeprom_93xx46_ew(struct eeprom_93xx46_dev *edev, int is_on) int bits, ret; u16 cmd_addr; + /* The opcode in front of the address is three bits. */ + bits = edev->addrlen + 3; + cmd_addr = OP_START << edev->addrlen; - if (edev->addrlen == 7) { + if (edev->pdata->flags & EE_ADDR8) cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS) << 1; - bits = 10; - } else { + else cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS); - bits = 9; - } if (has_quirk_instruction_length(edev)) { cmd_addr <<= 2; @@ -205,15 +222,19 @@ eeprom_93xx46_write_word(struct eeprom_93xx46_dev *edev, int bits, data_len, ret; u16 cmd_addr; + if (unlikely(off >= edev->size)) + return -EINVAL; + + /* The opcode in front of the address is three bits. */ + bits = edev->addrlen + 3; + cmd_addr = OP_WRITE << edev->addrlen; - if (edev->addrlen == 7) { - cmd_addr |= off & 0x7f; - bits = 10; + if (edev->pdata->flags & EE_ADDR8) { + cmd_addr |= off; data_len = 1; } else { - cmd_addr |= (off >> 1) & 0x3f; - bits = 9; + cmd_addr |= (off >> 1); data_len = 2; } @@ -253,7 +274,7 @@ static int eeprom_93xx46_write(void *priv, unsigned int off, return count; /* only write even number of bytes on 16-bit devices */ - if (edev->addrlen == 6) { + if (edev->pdata->flags & EE_ADDR16) { step = 2; count &= ~1; } @@ -295,14 +316,14 @@ static int eeprom_93xx46_eral(struct eeprom_93xx46_dev *edev) int bits, ret; u16 cmd_addr; + /* The opcode in front of the address is three bits. */ + bits = edev->addrlen + 3; + cmd_addr = OP_START << edev->addrlen; - if (edev->addrlen == 7) { + if (edev->pdata->flags & EE_ADDR8) cmd_addr |= ADDR_ERAL << 1; - bits = 10; - } else { + else cmd_addr |= ADDR_ERAL; - bits = 9; - } if (has_quirk_instruction_length(edev)) { cmd_addr <<= 2; @@ -375,8 +396,11 @@ static void select_deassert(void *context) } static const struct of_device_id eeprom_93xx46_of_table[] = { - { .compatible = "eeprom-93xx46", }, + { .compatible = "eeprom-93xx46", .data = &at93c46_data, }, + { .compatible = "atmel,at93c46", .data = &at93c46_data, }, { .compatible = "atmel,at93c46d", .data = &atmel_at93c46d_data, }, + { .compatible = "atmel,at93c56", .data = &at93c56_data, }, + { .compatible = "atmel,at93c66", .data = &at93c66_data, }, { .compatible = "microchip,93lc46b", .data = µchip_93lc46b_data, }, {} }; @@ -426,6 +450,7 @@ static int eeprom_93xx46_probe_dt(struct spi_device *spi) const struct eeprom_93xx46_devtype_data *data = of_id->data; pd->quirks = data->quirks; + pd->flags |= data->flags; } spi->dev.platform_data = pd; @@ -455,10 +480,21 @@ static int eeprom_93xx46_probe(struct spi_device *spi) if (!edev) return -ENOMEM; + if (pd->flags & EE_SIZE1K) + edev->size = 128; + else if (pd->flags & EE_SIZE2K) + edev->size = 256; + else if (pd->flags & EE_SIZE4K) + edev->size = 512; + else { + dev_err(&spi->dev, "unspecified size\n"); + return -EINVAL; + } + if (pd->flags & EE_ADDR8) - edev->addrlen = 7; + edev->addrlen = ilog2(edev->size); else if (pd->flags & EE_ADDR16) - edev->addrlen = 6; + edev->addrlen = ilog2(edev->size) - 1; else { dev_err(&spi->dev, "unspecified address type\n"); return -EINVAL; @@ -469,7 +505,6 @@ static int eeprom_93xx46_probe(struct spi_device *spi) edev->spi = spi; edev->pdata = pd; - edev->size = 128; edev->nvmem_config.type = NVMEM_TYPE_EEPROM; edev->nvmem_config.name = dev_name(&spi->dev); edev->nvmem_config.dev = &spi->dev; @@ -489,8 +524,9 @@ static int eeprom_93xx46_probe(struct spi_device *spi) if (IS_ERR(edev->nvmem)) return PTR_ERR(edev->nvmem); - dev_info(&spi->dev, "%d-bit eeprom %s\n", + dev_info(&spi->dev, "%d-bit eeprom containing %d bytes %s\n", (pd->flags & EE_ADDR8) ? 8 : 16, + edev->size, (pd->flags & EE_READONLY) ? "(readonly)" : ""); if (!(pd->flags & EE_READONLY)) { diff --git a/drivers/misc/eeprom/idt_89hpesx.c b/drivers/misc/eeprom/idt_89hpesx.c index 81c70e5bc168..b0cff4b152da 100644 --- a/drivers/misc/eeprom/idt_89hpesx.c +++ b/drivers/misc/eeprom/idt_89hpesx.c @@ -1,38 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-only /* - * This file is provided under a GPLv2 license. When using or - * redistributing this file, you may do so under that license. - * - * GPL LICENSE SUMMARY - * * Copyright (C) 2016 T-Platforms. All Rights Reserved. * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, it can be found <http://www.gnu.org/licenses/>. - * - * The full GNU General Public License is included in this distribution in - * the file called "COPYING". - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * * IDT PCIe-switch NTB Linux driver * * Contact Information: @@ -1126,11 +1095,10 @@ static void idt_get_fw_data(struct idt_89hpesx_dev *pdev) device_for_each_child_node(dev, fwnode) { ee_id = idt_ee_match_id(fwnode); - if (!ee_id) { - dev_warn(dev, "Skip unsupported EEPROM device"); - continue; - } else + if (ee_id) break; + + dev_warn(dev, "Skip unsupported EEPROM device %pfw\n", fwnode); } /* If there is no fwnode EEPROM device, then set zero size */ @@ -1161,6 +1129,7 @@ static void idt_get_fw_data(struct idt_89hpesx_dev *pdev) else /* if (!fwnode_property_read_bool(node, "read-only")) */ pdev->eero = false; + fwnode_handle_put(fwnode); dev_info(dev, "EEPROM of %d bytes found by 0x%x", pdev->eesize, pdev->eeaddr); } diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index af3c497defb1..80c60fb41bbc 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -556,6 +556,13 @@ out: else if (!cs->submitted) cs->fence->error = -EBUSY; + if (unlikely(cs->skip_reset_on_timeout)) { + dev_err(hdev->dev, + "Command submission %llu completed after %llu (s)\n", + cs->sequence, + div_u64(jiffies - cs->submission_time_jiffies, HZ)); + } + if (cs->timestamp) cs->fence->timestamp = ktime_get(); complete_all(&cs->fence->completion); @@ -571,6 +578,8 @@ static void cs_timedout(struct work_struct *work) int rc; struct hl_cs *cs = container_of(work, struct hl_cs, work_tdr.work); + bool skip_reset_on_timeout = cs->skip_reset_on_timeout; + rc = cs_get_unless_zero(cs); if (!rc) return; @@ -581,7 +590,8 @@ static void cs_timedout(struct work_struct *work) } /* Mark the CS is timed out so we won't try to cancel its TDR */ - cs->timedout = true; + if (likely(!skip_reset_on_timeout)) + cs->timedout = true; hdev = cs->ctx->hdev; @@ -613,10 +623,12 @@ static void cs_timedout(struct work_struct *work) cs_put(cs); - if (hdev->reset_on_lockup) - hl_device_reset(hdev, 0); - else - hdev->needs_reset = true; + if (likely(!skip_reset_on_timeout)) { + if (hdev->reset_on_lockup) + hl_device_reset(hdev, HL_RESET_TDR); + else + hdev->needs_reset = true; + } } static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, @@ -650,6 +662,10 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, cs->type = cs_type; cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP); cs->timeout_jiffies = timeout; + cs->skip_reset_on_timeout = + hdev->skip_reset_on_timeout || + !!(flags & HL_CS_FLAGS_SKIP_RESET_ON_TIMEOUT); + cs->submission_time_jiffies = jiffies; INIT_LIST_HEAD(&cs->job_list); INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout); kref_init(&cs->refcount); @@ -1481,6 +1497,61 @@ out: return rc; } +/* + * hl_cs_signal_sob_wraparound_handler: handle SOB value wrapaound case. + * if the SOB value reaches the max value move to the other SOB reserved + * to the queue. + * Note that this function must be called while hw_queues_lock is taken. + */ +int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, + struct hl_hw_sob **hw_sob, u32 count) +{ + struct hl_sync_stream_properties *prop; + struct hl_hw_sob *sob = *hw_sob, *other_sob; + u8 other_sob_offset; + + prop = &hdev->kernel_queues[q_idx].sync_stream_prop; + + kref_get(&sob->kref); + + /* check for wraparound */ + if (prop->next_sob_val + count >= HL_MAX_SOB_VAL) { + /* + * Decrement as we reached the max value. + * The release function won't be called here as we've + * just incremented the refcount right before calling this + * function. + */ + kref_put(&sob->kref, hl_sob_reset_error); + + /* + * check the other sob value, if it still in use then fail + * otherwise make the switch + */ + other_sob_offset = (prop->curr_sob_offset + 1) % HL_RSVD_SOBS; + other_sob = &prop->hw_sob[other_sob_offset]; + + if (kref_read(&other_sob->kref) != 1) { + dev_err(hdev->dev, "error: Cannot switch SOBs q_idx: %d\n", + q_idx); + return -EINVAL; + } + + prop->next_sob_val = 1; + + /* only two SOBs are currently in use */ + prop->curr_sob_offset = other_sob_offset; + *hw_sob = other_sob; + + dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n", + prop->curr_sob_offset, q_idx); + } else { + prop->next_sob_val += count; + } + + return 0; +} + static int cs_ioctl_extract_signal_seq(struct hl_device *hdev, struct hl_cs_chunk *chunk, u64 *signal_seq, struct hl_ctx *ctx) { diff --git a/drivers/misc/habanalabs/common/context.c b/drivers/misc/habanalabs/common/context.c index 62d705889ca8..19b6b045219e 100644 --- a/drivers/misc/habanalabs/common/context.c +++ b/drivers/misc/habanalabs/common/context.c @@ -12,7 +12,6 @@ static void hl_ctx_fini(struct hl_ctx *ctx) { struct hl_device *hdev = ctx->hdev; - u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; int i; /* Release all allocated pending cb's, those cb's were never @@ -57,14 +56,6 @@ static void hl_ctx_fini(struct hl_ctx *ctx) /* Scrub both SRAM and DRAM */ hdev->asic_funcs->scrub_device_mem(hdev, 0, 0); - - if ((!hdev->pldm) && (hdev->pdev) && - (!hdev->asic_funcs->is_device_idle(hdev, - idle_mask, - HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL))) - dev_notice(hdev->dev, - "device not idle after user context is closed (0x%llx, 0x%llx)\n", - idle_mask[0], idle_mask[1]); } else { dev_dbg(hdev->dev, "closing kernel context\n"); hdev->asic_funcs->ctx_fini(ctx); diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c index 8381155578a0..703d79fb6f3f 100644 --- a/drivers/misc/habanalabs/common/debugfs.c +++ b/drivers/misc/habanalabs/common/debugfs.c @@ -1278,6 +1278,11 @@ void hl_debugfs_add_device(struct hl_device *hdev) dev_entry->root, &dev_entry->blob_desc); + debugfs_create_x8("skip_reset_on_timeout", + 0644, + dev_entry->root, + &hdev->skip_reset_on_timeout); + for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) { debugfs_create_file(hl_debugfs_list[i].name, 0444, diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index 00e92b678828..ff4cbde289c0 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -51,6 +51,8 @@ bool hl_device_operational(struct hl_device *hdev, static void hpriv_release(struct kref *ref) { + u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; + bool device_is_idle = true; struct hl_fpriv *hpriv; struct hl_device *hdev; @@ -71,8 +73,20 @@ static void hpriv_release(struct kref *ref) kfree(hpriv); - if (hdev->reset_upon_device_release) - hl_device_reset(hdev, 0); + if ((!hdev->pldm) && (hdev->pdev) && + (!hdev->asic_funcs->is_device_idle(hdev, + idle_mask, + HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL))) { + dev_err(hdev->dev, + "device not idle after user context is closed (0x%llx_%llx)\n", + idle_mask[1], idle_mask[0]); + + device_is_idle = false; + } + + if ((hdev->reset_if_device_not_idle && !device_is_idle) + || hdev->reset_upon_device_release) + hl_device_reset(hdev, HL_RESET_DEVICE_RELEASE); } void hl_hpriv_get(struct hl_fpriv *hpriv) @@ -118,6 +132,9 @@ static int hl_device_release(struct inode *inode, struct file *filp) dev_warn(hdev->dev, "Device is still in use because there are live CS and/or memory mappings\n"); + hdev->last_open_session_duration_jif = + jiffies - hdev->last_successful_open_jif; + return 0; } @@ -868,7 +885,7 @@ static void device_disable_open_processes(struct hl_device *hdev) int hl_device_reset(struct hl_device *hdev, u32 flags) { u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; - bool hard_reset, from_hard_reset_thread; + bool hard_reset, from_hard_reset_thread, hard_instead_soft = false; int i, rc; if (!hdev->init_done) { @@ -880,11 +897,28 @@ int hl_device_reset(struct hl_device *hdev, u32 flags) hard_reset = (flags & HL_RESET_HARD) != 0; from_hard_reset_thread = (flags & HL_RESET_FROM_RESET_THREAD) != 0; - if ((!hard_reset) && (!hdev->supports_soft_reset)) { - dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n"); + if (!hard_reset && !hdev->supports_soft_reset) { + hard_instead_soft = true; + hard_reset = true; + } + + if (hdev->reset_upon_device_release && + (flags & HL_RESET_DEVICE_RELEASE)) { + dev_dbg(hdev->dev, + "Perform %s-reset upon device release\n", + hard_reset ? "hard" : "soft"); + goto do_reset; + } + + if (!hard_reset && !hdev->allow_external_soft_reset) { + hard_instead_soft = true; hard_reset = true; } + if (hard_instead_soft) + dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n"); + +do_reset: /* Re-entry of reset thread */ if (from_hard_reset_thread && hdev->process_kill_trial_cnt) goto kill_processes; @@ -901,6 +935,19 @@ int hl_device_reset(struct hl_device *hdev, u32 flags) return 0; /* + * 'reset cause' is being updated here, because getting here + * means that it's the 1st time and the last time we're here + * ('in_reset' makes sure of it). This makes sure that + * 'reset_cause' will continue holding its 1st recorded reason! + */ + if (flags & HL_RESET_HEARTBEAT) + hdev->curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT; + else if (flags & HL_RESET_TDR) + hdev->curr_reset_cause = HL_RESET_CAUSE_TDR; + else + hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; + + /* * if reset is due to heartbeat, device CPU is no responsive in * which case no point sending PCI disable message to it */ @@ -943,9 +990,8 @@ again: hdev->process_kill_trial_cnt = 0; /* - * Because the reset function can't run from interrupt or - * from heartbeat work, we need to call the reset function - * from a dedicated work + * Because the reset function can't run from heartbeat work, + * we need to call the reset function from a dedicated work. */ queue_delayed_work(hdev->device_reset_work.wq, &hdev->device_reset_work.reset_work, 0); @@ -1096,8 +1142,8 @@ kill_processes: if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask, HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)) { dev_err(hdev->dev, - "device is not idle (mask %#llx %#llx) after reset\n", - idle_mask[0], idle_mask[1]); + "device is not idle (mask 0x%llx_%llx) after reset\n", + idle_mask[1], idle_mask[0]); rc = -EIO; goto out_err; } @@ -1334,8 +1380,9 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) } /* - * From this point, in case of an error, add char devices and create - * sysfs nodes as part of the error flow, to allow debugging. + * From this point, override rc (=0) in case of an error to allow + * debugging (by adding char devices and create sysfs nodes as part of + * the error flow). */ add_cdev_sysfs_on_err = true; @@ -1369,7 +1416,7 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n", hdev->asic_name, - hdev->asic_prop.dram_size / 1024 / 1024 / 1024); + hdev->asic_prop.dram_size / SZ_1G); rc = hl_vm_init(hdev); if (rc) { @@ -1475,6 +1522,7 @@ out_disabled: void hl_device_fini(struct hl_device *hdev) { ktime_t timeout; + u64 reset_sec; int i, rc; dev_info(hdev->dev, "Removing device\n"); @@ -1482,6 +1530,11 @@ void hl_device_fini(struct hl_device *hdev) hdev->device_fini_pending = 1; flush_delayed_work(&hdev->device_reset_work.reset_work); + if (hdev->pldm) + reset_sec = HL_PLDM_HARD_RESET_MAX_TIMEOUT; + else + reset_sec = HL_HARD_RESET_MAX_TIMEOUT; + /* * This function is competing with the reset function, so try to * take the reset atomic and if we are already in middle of reset, @@ -1490,8 +1543,7 @@ void hl_device_fini(struct hl_device *hdev) * ports, the hard reset could take between 10-30 seconds */ - timeout = ktime_add_us(ktime_get(), - HL_HARD_RESET_MAX_TIMEOUT * 1000 * 1000); + timeout = ktime_add_us(ktime_get(), reset_sec * 1000 * 1000); rc = atomic_cmpxchg(&hdev->in_reset, 0, 1); while (rc) { usleep_range(50, 200); diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c index 0713b2c12d54..2e4d04ec6b53 100644 --- a/drivers/misc/habanalabs/common/firmware_if.c +++ b/drivers/misc/habanalabs/common/firmware_if.c @@ -9,42 +9,63 @@ #include "../include/common/hl_boot_if.h" #include <linux/firmware.h> +#include <linux/crc32.h> #include <linux/slab.h> +#include <linux/ctype.h> -#define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */ -/** - * hl_fw_load_fw_to_device() - Load F/W code to device's memory. - * - * @hdev: pointer to hl_device structure. - * @fw_name: the firmware image name - * @dst: IO memory mapped address space to copy firmware to - * @src_offset: offset in src FW to copy from - * @size: amount of bytes to copy (0 to copy the whole binary) - * - * Copy fw code from firmware file to device memory. - * - * Return: 0 on success, non-zero for failure. - */ -int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, - void __iomem *dst, u32 src_offset, u32 size) +#define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */ + +#define FW_CPU_STATUS_POLL_INTERVAL_USEC 10000 + +static char *extract_fw_ver_from_str(const char *fw_str) +{ + char *str, *fw_ver, *whitespace; + + fw_ver = kmalloc(16, GFP_KERNEL); + if (!fw_ver) + return NULL; + + str = strnstr(fw_str, "fw-", VERSION_MAX_LEN); + if (!str) + goto free_fw_ver; + + /* Skip the fw- part */ + str += 3; + + /* Copy until the next whitespace */ + whitespace = strnstr(str, " ", 15); + if (!whitespace) + goto free_fw_ver; + + strscpy(fw_ver, str, whitespace - str + 1); + + return fw_ver; + +free_fw_ver: + kfree(fw_ver); + return NULL; +} + +static int hl_request_fw(struct hl_device *hdev, + const struct firmware **firmware_p, + const char *fw_name) { - const struct firmware *fw; - const void *fw_data; size_t fw_size; int rc; - rc = request_firmware(&fw, fw_name, hdev->dev); + rc = request_firmware(firmware_p, fw_name, hdev->dev); if (rc) { - dev_err(hdev->dev, "Firmware file %s is not found!\n", fw_name); + dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n", + fw_name, rc); goto out; } - fw_size = fw->size; + fw_size = (*firmware_p)->size; if ((fw_size % 4) != 0) { dev_err(hdev->dev, "Illegal %s firmware size %zu\n", - fw_name, fw_size); + fw_name, fw_size); rc = -EINVAL; - goto out; + goto release_fw; } dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size); @@ -54,26 +75,125 @@ int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, "FW file size %zu exceeds maximum of %u bytes\n", fw_size, FW_FILE_MAX_SIZE); rc = -EINVAL; - goto out; + goto release_fw; } - if (size - src_offset > fw_size) { + return 0; + +release_fw: + release_firmware(*firmware_p); +out: + return rc; +} + +/** + * hl_release_firmware() - release FW + * + * @fw: fw descriptor + * + * note: this inline function added to serve as a comprehensive mirror for the + * hl_request_fw function. + */ +static inline void hl_release_firmware(const struct firmware *fw) +{ + release_firmware(fw); +} + +/** + * hl_fw_copy_fw_to_device() - copy FW to device + * + * @hdev: pointer to hl_device structure. + * @fw: fw descriptor + * @dst: IO memory mapped address space to copy firmware to + * @src_offset: offset in src FW to copy from + * @size: amount of bytes to copy (0 to copy the whole binary) + * + * actual copy of FW binary data to device, shared by static and dynamic loaders + */ +static int hl_fw_copy_fw_to_device(struct hl_device *hdev, + const struct firmware *fw, void __iomem *dst, + u32 src_offset, u32 size) +{ + const void *fw_data; + + /* size 0 indicates to copy the whole file */ + if (!size) + size = fw->size; + + if (src_offset + size > fw->size) { dev_err(hdev->dev, "size to copy(%u) and offset(%u) are invalid\n", size, src_offset); - rc = -EINVAL; - goto out; + return -EINVAL; } - if (size) - fw_size = size; - fw_data = (const void *) fw->data; - memcpy_toio(dst, fw_data + src_offset, fw_size); + memcpy_toio(dst, fw_data + src_offset, size); + return 0; +} -out: - release_firmware(fw); +/** + * hl_fw_copy_msg_to_device() - copy message to device + * + * @hdev: pointer to hl_device structure. + * @msg: message + * @dst: IO memory mapped address space to copy firmware to + * @src_offset: offset in src message to copy from + * @size: amount of bytes to copy (0 to copy the whole binary) + * + * actual copy of message data to device. + */ +static int hl_fw_copy_msg_to_device(struct hl_device *hdev, + struct lkd_msg_comms *msg, void __iomem *dst, + u32 src_offset, u32 size) +{ + void *msg_data; + + /* size 0 indicates to copy the whole file */ + if (!size) + size = sizeof(struct lkd_msg_comms); + + if (src_offset + size > sizeof(struct lkd_msg_comms)) { + dev_err(hdev->dev, + "size to copy(%u) and offset(%u) are invalid\n", + size, src_offset); + return -EINVAL; + } + + msg_data = (void *) msg; + + memcpy_toio(dst, msg_data + src_offset, size); + + return 0; +} + +/** + * hl_fw_load_fw_to_device() - Load F/W code to device's memory. + * + * @hdev: pointer to hl_device structure. + * @fw_name: the firmware image name + * @dst: IO memory mapped address space to copy firmware to + * @src_offset: offset in src FW to copy from + * @size: amount of bytes to copy (0 to copy the whole binary) + * + * Copy fw code from firmware file to device memory. + * + * Return: 0 on success, non-zero for failure. + */ +int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, + void __iomem *dst, u32 src_offset, u32 size) +{ + const struct firmware *fw; + int rc; + + rc = hl_request_fw(hdev, &fw, fw_name); + if (rc) + return rc; + + rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size); + + hl_release_firmware(fw); return rc; } @@ -91,6 +211,7 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, u16 len, u32 timeout, u64 *result) { struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id]; + struct asic_fixed_properties *prop = &hdev->asic_prop; struct cpucp_packet *pkt; dma_addr_t pkt_dma_addr; u32 tmp, expected_ack_val; @@ -117,7 +238,7 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, } /* set fence to a non valid value */ - pkt->fence = UINT_MAX; + pkt->fence = cpu_to_le32(UINT_MAX); rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, len, pkt_dma_addr); if (rc) { @@ -125,8 +246,7 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, goto out; } - if (hdev->asic_prop.fw_app_security_map & - CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN) + if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN) expected_ack_val = queue->pi; else expected_ack_val = CPUCP_PACKET_FENCE_VAL; @@ -272,10 +392,11 @@ void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, int hl_fw_send_heartbeat(struct hl_device *hdev) { - struct cpucp_packet hb_pkt = {}; + struct cpucp_packet hb_pkt; u64 result; int rc; + memset(&hb_pkt, 0, sizeof(hb_pkt)); hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST << CPUCP_PKT_CTL_OPCODE_SHIFT); hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL); @@ -284,29 +405,24 @@ int hl_fw_send_heartbeat(struct hl_device *hdev) sizeof(hb_pkt), 0, &result); if ((rc) || (result != CPUCP_PACKET_FENCE_VAL)) + return -EIO; + + if (le32_to_cpu(hb_pkt.status_mask) & + CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) { + dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n"); rc = -EIO; + } return rc; } -static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, - u32 cpu_security_boot_status_reg) +static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val, + u32 sts_val) { - u32 err_val, security_val; bool err_exists = false; - /* Some of the firmware status codes are deprecated in newer f/w - * versions. In those versions, the errors are reported - * in different registers. Therefore, we need to check those - * registers and print the exact errors. Moreover, there - * may be multiple errors, so we need to report on each error - * separately. Some of the error codes might indicate a state - * that is not an error per-se, but it is an error in production - * environment - */ - err_val = RREG32(boot_err0_reg); if (!(err_val & CPU_BOOT_ERR0_ENABLED)) - return 0; + return false; if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) { dev_err(hdev->dev, @@ -377,44 +493,120 @@ static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, err_exists = true; } + if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) { + dev_warn(hdev->dev, + "Device boot warning - Failed to load preboot primary image\n"); + /* This is a warning so we don't want it to disable the + * device as we have a secondary preboot image + */ + err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL; + } + + if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) { + dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n"); + err_exists = true; + } + if (err_val & CPU_BOOT_ERR0_PLL_FAIL) { dev_err(hdev->dev, "Device boot error - PLL failure\n"); err_exists = true; } if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) { - dev_err(hdev->dev, - "Device boot error - device unusable\n"); - err_exists = true; + /* Ignore this bit, don't prevent driver loading */ + dev_dbg(hdev->dev, "device unusable status is set\n"); + err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL; } - security_val = RREG32(cpu_security_boot_status_reg); - if (security_val & CPU_BOOT_DEV_STS0_ENABLED) - dev_dbg(hdev->dev, "Device security status %#x\n", - security_val); + if (sts_val & CPU_BOOT_DEV_STS0_ENABLED) + dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val); if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) { dev_err(hdev->dev, - "Device boot error - unknown error 0x%08x\n", - err_val); + "Device boot error - unknown ERR0 error 0x%08x\n", err_val); err_exists = true; } + /* return error only if it's in the predefined mask */ if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) & lower_32_bits(hdev->boot_error_status_mask))) + return true; + + return false; +} + +/* placeholder for ERR1 as no errors defined there yet */ +static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val, + u32 sts_val) +{ + /* + * keep this variable to preserve the logic of the function. + * this way it would require less modifications when error will be + * added to DEV_ERR1 + */ + bool err_exists = false; + + if (!(err_val & CPU_BOOT_ERR1_ENABLED)) + return false; + + if (sts_val & CPU_BOOT_DEV_STS1_ENABLED) + dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val); + + if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) { + dev_err(hdev->dev, + "Device boot error - unknown ERR1 error 0x%08x\n", + err_val); + err_exists = true; + } + + /* return error only if it's in the predefined mask */ + if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) & + upper_32_bits(hdev->boot_error_status_mask))) + return true; + + return false; +} + +static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, + u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg, + u32 cpu_boot_dev_status1_reg) +{ + u32 err_val, status_val; + bool err_exists = false; + + /* Some of the firmware status codes are deprecated in newer f/w + * versions. In those versions, the errors are reported + * in different registers. Therefore, we need to check those + * registers and print the exact errors. Moreover, there + * may be multiple errors, so we need to report on each error + * separately. Some of the error codes might indicate a state + * that is not an error per-se, but it is an error in production + * environment + */ + err_val = RREG32(boot_err0_reg); + status_val = RREG32(cpu_boot_dev_status0_reg); + err_exists = fw_report_boot_dev0(hdev, err_val, status_val); + + err_val = RREG32(boot_err1_reg); + status_val = RREG32(cpu_boot_dev_status1_reg); + err_exists |= fw_report_boot_dev1(hdev, err_val, status_val); + + if (err_exists) return -EIO; return 0; } int hl_fw_cpucp_info_get(struct hl_device *hdev, - u32 cpu_security_boot_status_reg, - u32 boot_err0_reg) + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, + u32 boot_err1_reg) { struct asic_fixed_properties *prop = &hdev->asic_prop; struct cpucp_packet pkt = {}; - void *cpucp_info_cpu_addr; dma_addr_t cpucp_info_dma_addr; + void *cpucp_info_cpu_addr; + char *kernel_ver; u64 result; int rc; @@ -443,7 +635,8 @@ int hl_fw_cpucp_info_get(struct hl_device *hdev, goto out; } - rc = fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg); + rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg, + sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg); if (rc) { dev_err(hdev->dev, "Errors in device boot\n"); goto out; @@ -460,10 +653,27 @@ int hl_fw_cpucp_info_get(struct hl_device *hdev, goto out; } + kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version); + if (kernel_ver) { + dev_info(hdev->dev, "Linux version %s", kernel_ver); + kfree(kernel_ver); + } + + /* assume EQ code doesn't need to check eqe index */ + hdev->event_queue.check_eqe_index = false; + /* Read FW application security bits again */ - if (hdev->asic_prop.fw_security_status_valid) - hdev->asic_prop.fw_app_security_map = - RREG32(cpu_security_boot_status_reg); + if (hdev->asic_prop.fw_cpu_boot_dev_sts0_valid) { + hdev->asic_prop.fw_app_cpu_boot_dev_sts0 = + RREG32(sts_boot_dev_sts0_reg); + if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_EQ_INDEX_EN) + hdev->event_queue.check_eqe_index = true; + } + + if (hdev->asic_prop.fw_cpu_boot_dev_sts1_valid) + hdev->asic_prop.fw_app_cpu_boot_dev_sts1 = + RREG32(sts_boot_dev_sts1_reg); out: hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, @@ -501,7 +711,8 @@ static int hl_fw_send_msi_info_msg(struct hl_device *hdev) pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES); - hdev->asic_funcs->get_msi_info((u32 *)&pkt->data); + memset((void *) &pkt->data, 0xFF, data_size); + hdev->asic_funcs->get_msi_info(pkt->data); pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET << CPUCP_PKT_CTL_OPCODE_SHIFT); @@ -526,13 +737,15 @@ static int hl_fw_send_msi_info_msg(struct hl_device *hdev) } int hl_fw_cpucp_handshake(struct hl_device *hdev, - u32 cpu_security_boot_status_reg, - u32 boot_err0_reg) + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, + u32 boot_err1_reg) { int rc; - rc = hl_fw_cpucp_info_get(hdev, cpu_security_boot_status_reg, - boot_err0_reg); + rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg, + sts_boot_dev_sts1_reg, boot_err0_reg, + boot_err1_reg); if (rc) return rc; @@ -667,8 +880,8 @@ int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index, bool dynamic_pll; int fw_pll_idx; - dynamic_pll = prop->fw_security_status_valid && - (prop->fw_app_security_map & CPU_BOOT_DEV_STS0_DYN_PLL_EN); + dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_DYN_PLL_EN); if (!dynamic_pll) { /* @@ -759,6 +972,47 @@ int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power) return rc; } +void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev) +{ + struct static_fw_load_mgr *static_loader = + &hdev->fw_loader.static_loader; + int rc; + + if (hdev->asic_prop.dynamic_fw_load) { + rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader, + COMMS_RST_DEV, 0, false, + hdev->fw_loader.cpu_timeout); + if (rc) + dev_warn(hdev->dev, "Failed sending COMMS_RST_DEV\n"); + } else { + WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV); + } +} + +void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev) +{ + struct static_fw_load_mgr *static_loader = + &hdev->fw_loader.static_loader; + int rc; + + if (hdev->device_cpu_is_halted) + return; + + /* Stop device CPU to make sure nothing bad happens */ + if (hdev->asic_prop.dynamic_fw_load) { + rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader, + COMMS_GOTO_WFE, 0, true, + hdev->fw_loader.cpu_timeout); + if (rc) + dev_warn(hdev->dev, "Failed sending COMMS_GOTO_WFE\n"); + } else { + WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE); + msleep(static_loader->cpu_reset_wait_msec); + } + + hdev->device_cpu_is_halted = true; +} + static void detect_cpu_boot_status(struct hl_device *hdev, u32 status) { /* Some of the status codes below are deprecated in newer f/w @@ -767,63 +1021,59 @@ static void detect_cpu_boot_status(struct hl_device *hdev, u32 status) switch (status) { case CPU_BOOT_STATUS_NA: dev_err(hdev->dev, - "Device boot error - BTL did NOT run\n"); + "Device boot progress - BTL did NOT run\n"); break; case CPU_BOOT_STATUS_IN_WFE: dev_err(hdev->dev, - "Device boot error - Stuck inside WFE loop\n"); + "Device boot progress - Stuck inside WFE loop\n"); break; case CPU_BOOT_STATUS_IN_BTL: dev_err(hdev->dev, - "Device boot error - Stuck in BTL\n"); + "Device boot progress - Stuck in BTL\n"); break; case CPU_BOOT_STATUS_IN_PREBOOT: dev_err(hdev->dev, - "Device boot error - Stuck in Preboot\n"); + "Device boot progress - Stuck in Preboot\n"); break; case CPU_BOOT_STATUS_IN_SPL: dev_err(hdev->dev, - "Device boot error - Stuck in SPL\n"); + "Device boot progress - Stuck in SPL\n"); break; case CPU_BOOT_STATUS_IN_UBOOT: dev_err(hdev->dev, - "Device boot error - Stuck in u-boot\n"); + "Device boot progress - Stuck in u-boot\n"); break; case CPU_BOOT_STATUS_DRAM_INIT_FAIL: dev_err(hdev->dev, - "Device boot error - DRAM initialization failed\n"); + "Device boot progress - DRAM initialization failed\n"); break; case CPU_BOOT_STATUS_UBOOT_NOT_READY: dev_err(hdev->dev, - "Device boot error - u-boot stopped by user\n"); + "Device boot progress - Cannot boot\n"); break; case CPU_BOOT_STATUS_TS_INIT_FAIL: dev_err(hdev->dev, - "Device boot error - Thermal Sensor initialization failed\n"); + "Device boot progress - Thermal Sensor initialization failed\n"); break; default: dev_err(hdev->dev, - "Device boot error - Invalid status code %d\n", + "Device boot progress - Invalid status code %d\n", status); break; } } -int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, - u32 cpu_security_boot_status_reg, u32 boot_err0_reg, - u32 timeout) +static int hl_fw_read_preboot_caps(struct hl_device *hdev, + u32 cpu_boot_status_reg, + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, + u32 boot_err0_reg, u32 boot_err1_reg, + u32 timeout) { struct asic_fixed_properties *prop = &hdev->asic_prop; - u32 status, security_status; + u32 status, reg_val; int rc; - /* pldm was added for cases in which we use preboot on pldm and want - * to load boot fit, but we can't wait for preboot because it runs - * very slowly - */ - if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) || hdev->pldm) - return 0; - /* Need to check two possible scenarios: * * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where @@ -842,29 +1092,145 @@ int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, (status == CPU_BOOT_STATUS_READY_TO_BOOT) || (status == CPU_BOOT_STATUS_SRAM_AVAIL) || (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT), - 10000, + FW_CPU_STATUS_POLL_INTERVAL_USEC, timeout); if (rc) { - dev_err(hdev->dev, "Failed to read preboot version\n"); + dev_err(hdev->dev, "CPU boot ready status timeout\n"); detect_cpu_boot_status(hdev, status); /* If we read all FF, then something is totally wrong, no point * of reading specific errors */ if (status != -1) - fw_read_errors(hdev, boot_err0_reg, - cpu_security_boot_status_reg); + fw_read_errors(hdev, boot_err0_reg, boot_err1_reg, + sts_boot_dev_sts0_reg, + sts_boot_dev_sts1_reg); return -EIO; } - rc = hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_PREBOOT); - if (rc) - return rc; + /* + * the registers DEV_STS* contain FW capabilities/features. + * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED + * is set. + * In the first read of this register we store the value of this + * register ONLY if the register is enabled (which will be propagated + * to next stages) and also mark the register as valid. + * In case it is not enabled the stored value will be left 0- all + * caps/features are off + */ + reg_val = RREG32(sts_boot_dev_sts0_reg); + if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) { + prop->fw_cpu_boot_dev_sts0_valid = true; + prop->fw_preboot_cpu_boot_dev_sts0 = reg_val; + } + + reg_val = RREG32(sts_boot_dev_sts1_reg); + if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) { + prop->fw_cpu_boot_dev_sts1_valid = true; + prop->fw_preboot_cpu_boot_dev_sts1 = reg_val; + } - security_status = RREG32(cpu_security_boot_status_reg); + prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_FW_LD_COM_EN); + + /* initialize FW loader once we know what load protocol is used */ + hdev->asic_funcs->init_firmware_loader(hdev); + + dev_dbg(hdev->dev, "Attempting %s FW load\n", + prop->dynamic_fw_load ? "dynamic" : "legacy"); + return 0; +} - /* We read security status multiple times during boot: +static int hl_fw_static_read_device_fw_version(struct hl_device *hdev, + enum hl_fw_component fwc) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct fw_load_mgr *fw_loader = &hdev->fw_loader; + struct static_fw_load_mgr *static_loader; + char *dest, *boot_ver, *preboot_ver; + u32 ver_off, limit; + const char *name; + char btl_ver[32]; + + static_loader = &hdev->fw_loader.static_loader; + + switch (fwc) { + case FW_COMP_BOOT_FIT: + ver_off = RREG32(static_loader->boot_fit_version_offset_reg); + dest = prop->uboot_ver; + name = "Boot-fit"; + limit = static_loader->boot_fit_version_max_off; + break; + case FW_COMP_PREBOOT: + ver_off = RREG32(static_loader->preboot_version_offset_reg); + dest = prop->preboot_ver; + name = "Preboot"; + limit = static_loader->preboot_version_max_off; + break; + default: + dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc); + return -EIO; + } + + ver_off &= static_loader->sram_offset_mask; + + if (ver_off < limit) { + memcpy_fromio(dest, + hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off, + VERSION_MAX_LEN); + } else { + dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n", + name, ver_off); + strscpy(dest, "unavailable", VERSION_MAX_LEN); + return -EIO; + } + + if (fwc == FW_COMP_BOOT_FIT) { + boot_ver = extract_fw_ver_from_str(prop->uboot_ver); + if (boot_ver) { + dev_info(hdev->dev, "boot-fit version %s\n", boot_ver); + kfree(boot_ver); + } + } else if (fwc == FW_COMP_PREBOOT) { + preboot_ver = strnstr(prop->preboot_ver, "Preboot", + VERSION_MAX_LEN); + if (preboot_ver && preboot_ver != prop->preboot_ver) { + strscpy(btl_ver, prop->preboot_ver, + min((int) (preboot_ver - prop->preboot_ver), + 31)); + dev_info(hdev->dev, "%s\n", btl_ver); + } + + preboot_ver = extract_fw_ver_from_str(prop->preboot_ver); + if (preboot_ver) { + dev_info(hdev->dev, "preboot version %s\n", + preboot_ver); + kfree(preboot_ver); + } + } + + return 0; +} + +/** + * hl_fw_preboot_update_state - update internal data structures during + * handshake with preboot + * + * + * @hdev: pointer to the habanalabs device structure + * + * @return 0 on success, otherwise non-zero error code + */ +static void hl_fw_preboot_update_state(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1; + + cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0; + cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1; + + /* We read boot_dev_sts registers multiple times during boot: * 1. preboot - a. Check whether the security status bits are valid * b. Check whether fw security is enabled * c. Check whether hard reset is done by preboot @@ -874,48 +1240,1121 @@ int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, * b. Check whether hard reset is done by fw app * * Preboot: - * Check security status bit (CPU_BOOT_DEV_STS0_ENABLED), if it is set + * Check security status bit (CPU_BOOT_DEV_STS0_ENABLED). If set, then- * check security enabled bit (CPU_BOOT_DEV_STS0_SECURITY_EN) + * If set, then mark GIC controller to be disabled. */ - if (security_status & CPU_BOOT_DEV_STS0_ENABLED) { - prop->fw_security_status_valid = 1; + prop->hard_reset_done_by_fw = + !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN); - /* FW security should be derived from PCI ID, we keep this - * check for backward compatibility - */ - if (security_status & CPU_BOOT_DEV_STS0_SECURITY_EN) - prop->fw_security_disabled = false; + dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n", + cpu_boot_dev_sts0); + + dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n", + cpu_boot_dev_sts1); + + dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n", + prop->hard_reset_done_by_fw ? "enabled" : "disabled"); + + dev_dbg(hdev->dev, "firmware-level security is %s\n", + prop->fw_security_enabled ? "enabled" : "disabled"); + + dev_dbg(hdev->dev, "GIC controller is %s\n", + prop->gic_interrupts_enable ? "enabled" : "disabled"); +} + +static int hl_fw_static_read_preboot_status(struct hl_device *hdev) +{ + int rc; + + rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT); + if (rc) + return rc; + + return 0; +} + +int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, + u32 boot_err1_reg, u32 timeout) +{ + int rc; + + /* pldm was added for cases in which we use preboot on pldm and want + * to load boot fit, but we can't wait for preboot because it runs + * very slowly + */ + if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) || hdev->pldm) + return 0; + + /* + * In order to determine boot method (static VS dymanic) we need to + * read the boot caps register + */ + rc = hl_fw_read_preboot_caps(hdev, cpu_boot_status_reg, + sts_boot_dev_sts0_reg, + sts_boot_dev_sts1_reg, boot_err0_reg, + boot_err1_reg, timeout); + if (rc) + return rc; + + hl_fw_preboot_update_state(hdev); - if (security_status & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) + /* no need to read preboot status in dynamic load */ + if (hdev->asic_prop.dynamic_fw_load) + return 0; + + return hl_fw_static_read_preboot_status(hdev); +} + +/* associate string with COMM status */ +static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = { + [COMMS_STS_NOOP] = "NOOP", + [COMMS_STS_ACK] = "ACK", + [COMMS_STS_OK] = "OK", + [COMMS_STS_ERR] = "ERR", + [COMMS_STS_VALID_ERR] = "VALID_ERR", + [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR", +}; + +/** + * hl_fw_dynamic_report_error_status - report error status + * + * @hdev: pointer to the habanalabs device structure + * @status: value of FW status register + * @expected_status: the expected status + */ +static void hl_fw_dynamic_report_error_status(struct hl_device *hdev, + u32 status, + enum comms_sts expected_status) +{ + enum comms_sts comm_status = + FIELD_GET(COMMS_STATUS_STATUS_MASK, status); + + if (comm_status < COMMS_STS_INVLD_LAST) + dev_err(hdev->dev, "Device status %s, expected status: %s\n", + hl_dynamic_fw_status_str[comm_status], + hl_dynamic_fw_status_str[expected_status]); + else + dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n", + comm_status, + hl_dynamic_fw_status_str[expected_status]); +} + +/** + * hl_fw_dynamic_send_cmd - send LKD to FW cmd + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @cmd: LKD to FW cmd code + * @size: size of next FW component to be loaded (0 if not necessary) + * + * LDK to FW exact command layout is defined at struct comms_command. + * note: the size argument is used only when the next FW component should be + * loaded, otherwise it shall be 0. the size is used by the FW in later + * protocol stages and when sending only indicating the amount of memory + * to be allocated by the FW to receive the next boot component. + */ +static void hl_fw_dynamic_send_cmd(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + enum comms_cmd cmd, unsigned int size) +{ + struct cpu_dyn_regs *dyn_regs; + u32 val; + + dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs; + + val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd); + val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size); + + WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val); +} + +/** + * hl_fw_dynamic_extract_fw_response - update the FW response + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @response: FW response + * @status: the status read from CPU status register + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + struct fw_response *response, + u32 status) +{ + response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status); + response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) << + COMMS_STATUS_OFFSET_ALIGN_SHIFT; + response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status); + + if ((response->ram_type != COMMS_SRAM) && + (response->ram_type != COMMS_DRAM)) { + dev_err(hdev->dev, "FW status: invalid RAM type %u\n", + response->ram_type); + return -EIO; + } + + return 0; +} + +/** + * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @expected_status: expected status to wait for + * @timeout: timeout for status wait + * + * @return 0 on success, otherwise non-zero error code + * + * waiting for status from FW include polling the FW status register until + * expected status is received or timeout occurs (whatever occurs first). + */ +static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + enum comms_sts expected_status, + u32 timeout) +{ + struct cpu_dyn_regs *dyn_regs; + u32 status; + int rc; + + dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs; + + /* Wait for expected status */ + rc = hl_poll_timeout( + hdev, + le32_to_cpu(dyn_regs->cpu_cmd_status_to_host), + status, + FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status, + FW_CPU_STATUS_POLL_INTERVAL_USEC, + timeout); + + if (rc) { + hl_fw_dynamic_report_error_status(hdev, status, + expected_status); + return -EIO; + } + + /* + * skip storing FW response for NOOP to preserve the actual desired + * FW status + */ + if (expected_status == COMMS_STS_NOOP) + return 0; + + rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader, + &fw_loader->dynamic_loader.response, + status); + return rc; +} + +/** + * hl_fw_dynamic_send_clear_cmd - send clear command to FW + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * + * @return 0 on success, otherwise non-zero error code + * + * after command cycle between LKD to FW CPU (i.e. LKD got an expected status + * from FW) we need to clear the CPU status register in order to avoid garbage + * between command cycles. + * This is done by sending clear command and polling the CPU to LKD status + * register to hold the status NOOP + */ +static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev, + struct fw_load_mgr *fw_loader) +{ + hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0); + + return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP, + fw_loader->cpu_timeout); +} + +/** + * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @cmd: LKD to FW cmd code + * @size: size of next FW component to be loaded (0 if not necessary) + * @wait_ok: if true also wait for OK response from FW + * @timeout: timeout for status wait + * + * @return 0 on success, otherwise non-zero error code + * + * brief: + * when sending protocol command we have the following steps: + * - send clear (clear command and verify clear status register) + * - send the actual protocol command + * - wait for ACK on the protocol command + * - send clear + * - send NOOP + * if, in addition, the specific protocol command should wait for OK then: + * - wait for OK + * - send clear + * - send NOOP + * + * NOTES: + * send clear: this is necessary in order to clear the status register to avoid + * leftovers between command + * NOOP command: necessary to avoid loop on the clear command by the FW + */ +int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + enum comms_cmd cmd, unsigned int size, + bool wait_ok, u32 timeout) +{ + int rc; + + /* first send clear command to clean former commands */ + rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader); + + /* send the actual command */ + hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size); + + /* wait for ACK for the command */ + rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK, + timeout); + if (rc) + return rc; + + /* clear command to prepare for NOOP command */ + rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader); + if (rc) + return rc; + + /* send the actual NOOP command */ + hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0); + + if (!wait_ok) + return 0; + + rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK, + timeout); + if (rc) + return rc; + + /* clear command to prepare for NOOP command */ + rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader); + if (rc) + return rc; + + /* send the actual NOOP command */ + hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0); + + return 0; +} + +/** + * hl_fw_compat_crc32 - CRC compatible with FW + * + * @data: pointer to the data + * @size: size of the data + * + * @return the CRC32 result + * + * NOTE: kernel's CRC32 differ's from standard CRC32 calculation. + * in order to be aligned we need to flip the bits of both the input + * initial CRC and kernel's CRC32 result. + * in addition both sides use initial CRC of 0, + */ +static u32 hl_fw_compat_crc32(u8 *data, size_t size) +{ + return ~crc32_le(~((u32)0), data, size); +} + +/** + * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory + * transfer (image or descriptor) between + * host and FW + * + * @hdev: pointer to the habanalabs device structure + * @addr: device address of memory transfer + * @size: memory transter size + * @region: PCI memory region + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev, + u64 addr, size_t size, + struct pci_mem_region *region) +{ + u64 end_addr; + + /* now make sure that the memory transfer is within region's bounds */ + end_addr = addr + size; + if (end_addr >= region->region_base + region->region_size) { + dev_err(hdev->dev, + "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n", + end_addr); + return -EIO; + } + + /* + * now make sure memory transfer is within predefined BAR bounds. + * this is to make sure we do not need to set the bar (e.g. for DRAM + * memory transfers) + */ + if (end_addr >= region->region_base - region->offset_in_bar + + region->bar_size) { + dev_err(hdev->dev, + "FW image beyond PCI BAR bounds\n"); + return -EIO; + } + + return 0; +} + +/** + * hl_fw_dynamic_validate_descriptor - validate FW descriptor + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @fw_desc: the descriptor form FW + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + struct lkd_fw_comms_desc *fw_desc) +{ + struct pci_mem_region *region; + enum pci_region region_id; + size_t data_size; + u32 data_crc32; + u8 *data_ptr; + u64 addr; + int rc; + + if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC) { + dev_err(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n", + fw_desc->header.magic); + return -EIO; + } + + if (fw_desc->header.version != HL_COMMS_DESC_VER) { + dev_err(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n", + fw_desc->header.version); + return -EIO; + } + + /* + * calc CRC32 of data without header. + * note that no alignment/stride address issues here as all structures + * are 64 bit padded + */ + data_size = sizeof(struct lkd_fw_comms_desc) - + sizeof(struct comms_desc_header); + data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header); + + if (le16_to_cpu(fw_desc->header.size) != data_size) { + dev_err(hdev->dev, + "Invalid descriptor size 0x%x, expected size 0x%zx\n", + le16_to_cpu(fw_desc->header.size), data_size); + return -EIO; + } + + data_crc32 = hl_fw_compat_crc32(data_ptr, data_size); + + if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) { + dev_err(hdev->dev, + "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n", + data_crc32, fw_desc->header.crc32); + return -EIO; + } + + /* find memory region to which to copy the image */ + addr = le64_to_cpu(fw_desc->img_addr); + region_id = hl_get_pci_memory_region(hdev, addr); + if ((region_id != PCI_REGION_SRAM) && + ((region_id != PCI_REGION_DRAM))) { + dev_err(hdev->dev, + "Invalid region to copy FW image address=%llx\n", addr); + return -EIO; + } + + region = &hdev->pci_mem_region[region_id]; + + /* store the region for the copy stage */ + fw_loader->dynamic_loader.image_region = region; + + /* + * here we know that the start address is valid, now make sure that the + * image is within region's bounds + */ + rc = hl_fw_dynamic_validate_memory_bound(hdev, addr, + fw_loader->dynamic_loader.fw_image_size, + region); + if (rc) { + dev_err(hdev->dev, + "invalid mem transfer request for FW image\n"); + return rc; + } + + return 0; +} + +static int hl_fw_dynamic_validate_response(struct hl_device *hdev, + struct fw_response *response, + struct pci_mem_region *region) +{ + u64 device_addr; + int rc; + + device_addr = region->region_base + response->ram_offset; + + /* + * validate that the descriptor is within region's bounds + * Note that as the start address was supplied according to the RAM + * type- testing only the end address is enough + */ + rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr, + sizeof(struct lkd_fw_comms_desc), + region); + return rc; +} + +/** + * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev, + struct fw_load_mgr *fw_loader) +{ + struct lkd_fw_comms_desc *fw_desc; + struct pci_mem_region *region; + struct fw_response *response; + enum pci_region region_id; + void __iomem *src; + int rc; + + fw_desc = &fw_loader->dynamic_loader.comm_desc; + response = &fw_loader->dynamic_loader.response; + + region_id = (response->ram_type == COMMS_SRAM) ? + PCI_REGION_SRAM : PCI_REGION_DRAM; + + region = &hdev->pci_mem_region[region_id]; + + rc = hl_fw_dynamic_validate_response(hdev, response, region); + if (rc) { + dev_err(hdev->dev, + "invalid mem transfer request for FW descriptor\n"); + return rc; + } + + /* extract address copy the descriptor from */ + src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar + + response->ram_offset; + memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc)); + + return hl_fw_dynamic_validate_descriptor(hdev, fw_loader, fw_desc); +} + +/** + * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @next_image_size: size to allocate for next FW component + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + size_t next_image_size) +{ + int rc; + + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC, + next_image_size, true, + fw_loader->cpu_timeout); + if (rc) + return rc; + + return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader); +} + +/** + * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties + * + * @hdev: pointer to the habanalabs device structure + * @fwc: the firmware component + * @fw_version: fw component's version string + */ +static void hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev, + enum hl_fw_component fwc, + const char *fw_version) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + char *preboot_ver, *boot_ver; + char btl_ver[32]; + + switch (fwc) { + case FW_COMP_BOOT_FIT: + strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN); + boot_ver = extract_fw_ver_from_str(prop->uboot_ver); + if (boot_ver) { + dev_info(hdev->dev, "boot-fit version %s\n", boot_ver); + kfree(boot_ver); + } + + break; + case FW_COMP_PREBOOT: + strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN); + preboot_ver = strnstr(prop->preboot_ver, "Preboot", + VERSION_MAX_LEN); + if (preboot_ver && preboot_ver != prop->preboot_ver) { + strscpy(btl_ver, prop->preboot_ver, + min((int) (preboot_ver - prop->preboot_ver), + 31)); + dev_info(hdev->dev, "%s\n", btl_ver); + } + + preboot_ver = extract_fw_ver_from_str(prop->preboot_ver); + if (preboot_ver) { + dev_info(hdev->dev, "preboot version %s\n", + preboot_ver); + kfree(preboot_ver); + } + + break; + default: + dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc); + return; + } +} + +/** + * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW + * + * @hdev: pointer to the habanalabs device structure + * @fw: fw descriptor + * @fw_loader: managing structure for loading device's FW + */ +static int hl_fw_dynamic_copy_image(struct hl_device *hdev, + const struct firmware *fw, + struct fw_load_mgr *fw_loader) +{ + struct lkd_fw_comms_desc *fw_desc; + struct pci_mem_region *region; + void __iomem *dest; + u64 addr; + int rc; + + fw_desc = &fw_loader->dynamic_loader.comm_desc; + addr = le64_to_cpu(fw_desc->img_addr); + + /* find memory region to which to copy the image */ + region = fw_loader->dynamic_loader.image_region; + + dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar + + (addr - region->region_base); + + rc = hl_fw_copy_fw_to_device(hdev, fw, dest, + fw_loader->boot_fit_img.src_off, + fw_loader->boot_fit_img.copy_size); + + return rc; +} + +/** + * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW + * + * @hdev: pointer to the habanalabs device structure + * @msg: message + * @fw_loader: managing structure for loading device's FW + */ +static int hl_fw_dynamic_copy_msg(struct hl_device *hdev, + struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader) +{ + struct lkd_fw_comms_desc *fw_desc; + struct pci_mem_region *region; + void __iomem *dest; + u64 addr; + int rc; + + fw_desc = &fw_loader->dynamic_loader.comm_desc; + addr = le64_to_cpu(fw_desc->img_addr); + + /* find memory region to which to copy the image */ + region = fw_loader->dynamic_loader.image_region; + + dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar + + (addr - region->region_base); + + rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0); + + return rc; +} + +/** + * hl_fw_boot_fit_update_state - update internal data structures after boot-fit + * is loaded + * + * @hdev: pointer to the habanalabs device structure + * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0 + * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1 + * + * @return 0 on success, otherwise non-zero error code + */ +static void hl_fw_boot_fit_update_state(struct hl_device *hdev, + u32 cpu_boot_dev_sts0_reg, + u32 cpu_boot_dev_sts1_reg) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + + /* Clear reset status since we need to read it again from boot CPU */ + prop->hard_reset_done_by_fw = false; + + /* Read boot_cpu status bits */ + if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) { + prop->fw_bootfit_cpu_boot_dev_sts0 = + RREG32(cpu_boot_dev_sts0_reg); + + if (prop->fw_bootfit_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) prop->hard_reset_done_by_fw = true; - } else { - prop->fw_security_status_valid = 0; + + dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n", + prop->fw_bootfit_cpu_boot_dev_sts0); } - dev_dbg(hdev->dev, "Firmware preboot security status %#x\n", - security_status); + if (prop->fw_cpu_boot_dev_sts1_valid) { + prop->fw_bootfit_cpu_boot_dev_sts1 = + RREG32(cpu_boot_dev_sts1_reg); - dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n", + dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n", + prop->fw_bootfit_cpu_boot_dev_sts1); + } + + dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n", prop->hard_reset_done_by_fw ? "enabled" : "disabled"); +} + +static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev) +{ + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; + + /* Check whether all 3 interrupt interfaces are set, if not use a + * single interface + */ + if (!hdev->asic_prop.gic_interrupts_enable && + !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) { + dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_irq_ctrl; + dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_irq_ctrl; + + dev_warn(hdev->dev, + "Using a single interrupt interface towards cpucp"); + } +} +/** + * hl_fw_dynamic_load_image - load FW image using dynamic protocol + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @load_fwc: the FW component to be loaded + * @img_ld_timeout: image load timeout + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_load_image(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + enum hl_fw_component load_fwc, + u32 img_ld_timeout) +{ + enum hl_fw_component cur_fwc; + const struct firmware *fw; + char *fw_name; + int rc = 0; + + /* + * when loading image we have one of 2 scenarios: + * 1. current FW component is preboot and we want to load boot-fit + * 2. current FW component is boot-fit and we want to load linux + */ + if (load_fwc == FW_COMP_BOOT_FIT) { + cur_fwc = FW_COMP_PREBOOT; + fw_name = fw_loader->boot_fit_img.image_name; + } else { + cur_fwc = FW_COMP_BOOT_FIT; + fw_name = fw_loader->linux_img.image_name; + } + + /* request FW in order to communicate to FW the size to be allocated */ + rc = hl_request_fw(hdev, &fw, fw_name); + if (rc) + return rc; + + /* store the image size for future validation */ + fw_loader->dynamic_loader.fw_image_size = fw->size; + + rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size); + if (rc) + goto release_fw; + + /* read preboot version */ + hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc, + fw_loader->dynamic_loader.comm_desc.cur_fw_ver); + + + /* update state according to boot stage */ + if (cur_fwc == FW_COMP_BOOT_FIT) { + struct cpu_dyn_regs *dyn_regs; + + dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs; + hl_fw_boot_fit_update_state(hdev, + le32_to_cpu(dyn_regs->cpu_boot_dev_sts0), + le32_to_cpu(dyn_regs->cpu_boot_dev_sts1)); + } + + /* copy boot fit to space allocated by FW */ + rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader); + if (rc) + goto release_fw; - dev_info(hdev->dev, "firmware-level security is %s\n", - prop->fw_security_disabled ? "disabled" : "enabled"); + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY, + 0, true, + fw_loader->cpu_timeout); + if (rc) + goto release_fw; + + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC, + 0, false, + img_ld_timeout); +release_fw: + hl_release_firmware(fw); + return rc; +} + +static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev, + struct fw_load_mgr *fw_loader) +{ + struct dynamic_fw_load_mgr *dyn_loader; + u32 status; + int rc; + + dyn_loader = &fw_loader->dynamic_loader; + + /* Make sure CPU boot-loader is running */ + rc = hl_poll_timeout( + hdev, + le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status), + status, + (status == CPU_BOOT_STATUS_NIC_FW_RDY) || + (status == CPU_BOOT_STATUS_READY_TO_BOOT), + FW_CPU_STATUS_POLL_INTERVAL_USEC, + dyn_loader->wait_for_bl_timeout); + if (rc) { + dev_err(hdev->dev, "failed to wait for boot\n"); + return rc; + } + + dev_dbg(hdev->dev, "uboot status = %d\n", status); return 0; } -int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, - u32 msg_to_cpu_reg, u32 cpu_msg_status_reg, - u32 cpu_security_boot_status_reg, u32 boot_err0_reg, - bool skip_bmc, u32 cpu_timeout, u32 boot_fit_timeout) +static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev, + struct fw_load_mgr *fw_loader) { - struct asic_fixed_properties *prop = &hdev->asic_prop; + struct dynamic_fw_load_mgr *dyn_loader; u32 status; int rc; + dyn_loader = &fw_loader->dynamic_loader; + + /* Make sure CPU boot-loader is running */ + + rc = hl_poll_timeout( + hdev, + le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status), + status, + (status == CPU_BOOT_STATUS_SRAM_AVAIL), + FW_CPU_STATUS_POLL_INTERVAL_USEC, + fw_loader->cpu_timeout); + if (rc) { + dev_err(hdev->dev, "failed to wait for Linux\n"); + return rc; + } + + dev_dbg(hdev->dev, "Boot status = %d\n", status); + return 0; +} + +/** + * hl_fw_linux_update_state - update internal data structures after Linux + * is loaded. + * Note: Linux initialization is comprised mainly + * of two stages - loading kernel (SRAM_AVAIL) + * & loading ARMCP. + * Therefore reading boot device status in any of + * these stages might result in different values. + * + * @hdev: pointer to the habanalabs device structure + * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0 + * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1 + * + * @return 0 on success, otherwise non-zero error code + */ +static void hl_fw_linux_update_state(struct hl_device *hdev, + u32 cpu_boot_dev_sts0_reg, + u32 cpu_boot_dev_sts1_reg) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + + hdev->fw_loader.linux_loaded = true; + + /* Clear reset status since we need to read again from app */ + prop->hard_reset_done_by_fw = false; + + /* Read FW application security bits */ + if (prop->fw_cpu_boot_dev_sts0_valid) { + prop->fw_app_cpu_boot_dev_sts0 = + RREG32(cpu_boot_dev_sts0_reg); + + if (prop->fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) + prop->hard_reset_done_by_fw = true; + + if (prop->fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN) + prop->gic_interrupts_enable = false; + + dev_dbg(hdev->dev, + "Firmware application CPU status0 %#x\n", + prop->fw_app_cpu_boot_dev_sts0); + + dev_dbg(hdev->dev, "GIC controller is %s\n", + prop->gic_interrupts_enable ? + "enabled" : "disabled"); + } + + if (prop->fw_cpu_boot_dev_sts1_valid) { + prop->fw_app_cpu_boot_dev_sts1 = + RREG32(cpu_boot_dev_sts1_reg); + + dev_dbg(hdev->dev, + "Firmware application CPU status1 %#x\n", + prop->fw_app_cpu_boot_dev_sts1); + } + + dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n", + prop->hard_reset_done_by_fw ? "enabled" : "disabled"); + + dev_info(hdev->dev, "Successfully loaded firmware to device\n"); +} + +/** + * hl_fw_dynamic_report_reset_cause - send a COMMS message with the cause + * of the newly triggered hard reset + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @reset_cause: enumerated cause for the recent hard reset + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_report_reset_cause(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + enum comms_reset_cause reset_cause) +{ + struct lkd_msg_comms msg; + int rc; + + memset(&msg, 0, sizeof(msg)); + + /* create message to be sent */ + msg.header.type = HL_COMMS_RESET_CAUSE_TYPE; + msg.header.size = cpu_to_le16(sizeof(struct comms_msg_header)); + msg.header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC); + + msg.reset_cause = reset_cause; + + rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, + sizeof(struct lkd_msg_comms)); + if (rc) + return rc; + + /* copy message to space allocated by FW */ + rc = hl_fw_dynamic_copy_msg(hdev, &msg, fw_loader); + if (rc) + return rc; + + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY, + 0, true, + fw_loader->cpu_timeout); + if (rc) + return rc; + + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC, + 0, true, + fw_loader->cpu_timeout); + if (rc) + return rc; + + return 0; +} + +/** + * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * + * @return 0 on success, otherwise non-zero error code + * + * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol. + * the communication is done using registers: + * - LKD command register + * - FW status register + * the protocol is race free. this goal is achieved by splitting the requests + * and response to known synchronization points between the LKD and the FW. + * each response to LKD request is known and bound to a predefined timeout. + * in case of timeout expiration without the desired status from FW- the + * protocol (and hence the boot) will fail. + */ +static int hl_fw_dynamic_init_cpu(struct hl_device *hdev, + struct fw_load_mgr *fw_loader) +{ + struct cpu_dyn_regs *dyn_regs; + int rc; + + dev_info(hdev->dev, + "Loading firmware to device, may take some time...\n"); + + dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs; + + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE, + 0, true, + fw_loader->cpu_timeout); + if (rc) + goto protocol_err; + + if (hdev->curr_reset_cause) { + rc = hl_fw_dynamic_report_reset_cause(hdev, fw_loader, + hdev->curr_reset_cause); + if (rc) + goto protocol_err; + + /* Clear current reset cause */ + hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; + } + + if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) { + rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0); + if (rc) + goto protocol_err; + + /* read preboot version */ + hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT, + fw_loader->dynamic_loader.comm_desc.cur_fw_ver); + return 0; + } + + /* load boot fit to FW */ + rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT, + fw_loader->boot_fit_timeout); + if (rc) { + dev_err(hdev->dev, "failed to load boot fit\n"); + goto protocol_err; + } + + rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader); + if (rc) + goto protocol_err; + + /* Enable DRAM scrambling before Linux boot and after successful + * UBoot + */ + hdev->asic_funcs->init_cpu_scrambler_dram(hdev); + + if (!(hdev->fw_components & FW_TYPE_LINUX)) { + dev_info(hdev->dev, "Skip loading Linux F/W\n"); + return 0; + } + + if (fw_loader->skip_bmc) { + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, + COMMS_SKIP_BMC, 0, + true, + fw_loader->cpu_timeout); + if (rc) { + dev_err(hdev->dev, "failed to load boot fit\n"); + goto protocol_err; + } + } + + /* load Linux image to FW */ + rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX, + fw_loader->cpu_timeout); + if (rc) { + dev_err(hdev->dev, "failed to load Linux\n"); + goto protocol_err; + } + + rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader); + if (rc) + goto protocol_err; + + hl_fw_linux_update_state(hdev, le32_to_cpu(dyn_regs->cpu_boot_dev_sts0), + le32_to_cpu(dyn_regs->cpu_boot_dev_sts1)); + + hl_fw_dynamic_update_linux_interrupt_if(hdev); + + return 0; + +protocol_err: + fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0), + le32_to_cpu(dyn_regs->cpu_boot_err1), + le32_to_cpu(dyn_regs->cpu_boot_dev_sts0), + le32_to_cpu(dyn_regs->cpu_boot_dev_sts1)); + return rc; +} + +/** + * hl_fw_static_init_cpu - initialize the device CPU using static protocol + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_static_init_cpu(struct hl_device *hdev, + struct fw_load_mgr *fw_loader) +{ + u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status; + u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg; + struct static_fw_load_mgr *static_loader; + u32 cpu_boot_status_reg; + int rc; + if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) return 0; + /* init common loader parameters */ + cpu_timeout = fw_loader->cpu_timeout; + + /* init static loader parameters */ + static_loader = &fw_loader->static_loader; + cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg; + msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg; + cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg; + cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg; + cpu_boot_status_reg = static_loader->cpu_boot_status_reg; + dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n", cpu_timeout / USEC_PER_SEC); @@ -925,8 +2364,8 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, cpu_boot_status_reg, status, status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT, - 10000, - boot_fit_timeout); + FW_CPU_STATUS_POLL_INTERVAL_USEC, + fw_loader->boot_fit_timeout); if (rc) { dev_dbg(hdev->dev, @@ -948,8 +2387,8 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, cpu_msg_status_reg, status, status == CPU_MSG_OK, - 10000, - boot_fit_timeout); + FW_CPU_STATUS_POLL_INTERVAL_USEC, + fw_loader->boot_fit_timeout); if (rc) { dev_err(hdev->dev, @@ -970,33 +2409,17 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, (status == CPU_BOOT_STATUS_NIC_FW_RDY) || (status == CPU_BOOT_STATUS_READY_TO_BOOT) || (status == CPU_BOOT_STATUS_SRAM_AVAIL), - 10000, + FW_CPU_STATUS_POLL_INTERVAL_USEC, cpu_timeout); dev_dbg(hdev->dev, "uboot status = %d\n", status); /* Read U-Boot version now in case we will later fail */ - hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_UBOOT); - - /* Clear reset status since we need to read it again from boot CPU */ - prop->hard_reset_done_by_fw = false; - - /* Read boot_cpu security bits */ - if (prop->fw_security_status_valid) { - prop->fw_boot_cpu_security_map = - RREG32(cpu_security_boot_status_reg); - - if (prop->fw_boot_cpu_security_map & - CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) - prop->hard_reset_done_by_fw = true; - - dev_dbg(hdev->dev, - "Firmware boot CPU security status %#x\n", - prop->fw_boot_cpu_security_map); - } + hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT); - dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n", - prop->hard_reset_done_by_fw ? "enabled" : "disabled"); + /* update state according to boot stage */ + hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg, + cpu_boot_dev_status1_reg); if (rc) { detect_cpu_boot_status(hdev, status); @@ -1004,13 +2427,21 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, goto out; } + /* Enable DRAM scrambling before Linux boot and after successful + * UBoot + */ + hdev->asic_funcs->init_cpu_scrambler_dram(hdev); + if (!(hdev->fw_components & FW_TYPE_LINUX)) { dev_info(hdev->dev, "Skip loading Linux F/W\n"); + rc = 0; goto out; } - if (status == CPU_BOOT_STATUS_SRAM_AVAIL) + if (status == CPU_BOOT_STATUS_SRAM_AVAIL) { + rc = 0; goto out; + } dev_info(hdev->dev, "Loading firmware to device, may take some time...\n"); @@ -1019,7 +2450,7 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, if (rc) goto out; - if (skip_bmc) { + if (fw_loader->skip_bmc) { WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC); rc = hl_poll_timeout( @@ -1027,7 +2458,7 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, cpu_boot_status_reg, status, (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED), - 10000, + FW_CPU_STATUS_POLL_INTERVAL_USEC, cpu_timeout); if (rc) { @@ -1047,7 +2478,7 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, cpu_boot_status_reg, status, (status == CPU_BOOT_STATUS_SRAM_AVAIL), - 10000, + FW_CPU_STATUS_POLL_INTERVAL_USEC, cpu_timeout); /* Clear message */ @@ -1066,36 +2497,43 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, goto out; } - rc = fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg); + rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg, + fw_loader->static_loader.boot_err1_reg, + cpu_boot_dev_status0_reg, + cpu_boot_dev_status1_reg); if (rc) return rc; - /* Clear reset status since we need to read again from app */ - prop->hard_reset_done_by_fw = false; - - /* Read FW application security bits */ - if (prop->fw_security_status_valid) { - prop->fw_app_security_map = - RREG32(cpu_security_boot_status_reg); - - if (prop->fw_app_security_map & - CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) - prop->hard_reset_done_by_fw = true; - - dev_dbg(hdev->dev, - "Firmware application CPU security status %#x\n", - prop->fw_app_security_map); - } - - dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n", - prop->hard_reset_done_by_fw ? "enabled" : "disabled"); - - dev_info(hdev->dev, "Successfully loaded firmware to device\n"); + hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg, + cpu_boot_dev_status1_reg); return 0; out: - fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg); + fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg, + fw_loader->static_loader.boot_err1_reg, + cpu_boot_dev_status0_reg, + cpu_boot_dev_status1_reg); return rc; } + +/** + * hl_fw_init_cpu - initialize the device CPU + * + * @hdev: pointer to the habanalabs device structure + * + * @return 0 on success, otherwise non-zero error code + * + * perform necessary initializations for device's CPU. takes into account if + * init protocol is static or dynamic. + */ +int hl_fw_init_cpu(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct fw_load_mgr *fw_loader = &hdev->fw_loader; + + return prop->dynamic_fw_load ? + hl_fw_dynamic_init_cpu(hdev, fw_loader) : + hl_fw_static_init_cpu(hdev, fw_loader); +} diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 6579f8767abd..6b3cdd7e068a 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -48,6 +48,7 @@ #define HL_PENDING_RESET_LONG_SEC 60 #define HL_HARD_RESET_MAX_TIMEOUT 120 +#define HL_PLDM_HARD_RESET_MAX_TIMEOUT (HL_HARD_RESET_MAX_TIMEOUT * 3) #define HL_DEVICE_TIMEOUT_USEC 1000000 /* 1 s */ @@ -115,10 +116,18 @@ enum hl_mmu_page_table_location { * * - HL_RESET_HEARTBEAT * Set if reset is due to heartbeat + * + * - HL_RESET_TDR + * Set if reset is due to TDR + * + * - HL_RESET_DEVICE_RELEASE + * Set if reset is due to device release */ #define HL_RESET_HARD (1 << 0) #define HL_RESET_FROM_RESET_THREAD (1 << 1) #define HL_RESET_HEARTBEAT (1 << 2) +#define HL_RESET_TDR (1 << 3) +#define HL_RESET_DEVICE_RELEASE (1 << 4) #define HL_MAX_SOBS_PER_MONITOR 8 @@ -178,12 +187,14 @@ enum hl_pci_match_mode { /** * enum hl_fw_component - F/W components to read version through registers. - * @FW_COMP_UBOOT: u-boot. + * @FW_COMP_BOOT_FIT: boot fit. * @FW_COMP_PREBOOT: preboot. + * @FW_COMP_LINUX: linux. */ enum hl_fw_component { - FW_COMP_UBOOT, - FW_COMP_PREBOOT + FW_COMP_BOOT_FIT, + FW_COMP_PREBOOT, + FW_COMP_LINUX, }; /** @@ -420,12 +431,24 @@ struct hl_mmu_properties { * @cb_pool_cb_size: size of each CB in the CB pool. * @max_pending_cs: maximum of concurrent pending command submissions * @max_queues: maximum amount of queues in the system - * @fw_boot_cpu_security_map: bitmap representation of boot cpu security status - * reported by FW, bit description can be found in - * CPU_BOOT_DEV_STS* - * @fw_app_security_map: bitmap representation of application security status - * reported by FW, bit description can be found in - * CPU_BOOT_DEV_STS* + * @fw_preboot_cpu_boot_dev_sts0: bitmap representation of preboot cpu + * capabilities reported by FW, bit description + * can be found in CPU_BOOT_DEV_STS0 + * @fw_preboot_cpu_boot_dev_sts1: bitmap representation of preboot cpu + * capabilities reported by FW, bit description + * can be found in CPU_BOOT_DEV_STS1 + * @fw_bootfit_cpu_boot_dev_sts0: bitmap representation of boot cpu security + * status reported by FW, bit description can be + * found in CPU_BOOT_DEV_STS0 + * @fw_bootfit_cpu_boot_dev_sts1: bitmap representation of boot cpu security + * status reported by FW, bit description can be + * found in CPU_BOOT_DEV_STS1 + * @fw_app_cpu_boot_dev_sts0: bitmap representation of application security + * status reported by FW, bit description can be + * found in CPU_BOOT_DEV_STS0 + * @fw_app_cpu_boot_dev_sts1: bitmap representation of application security + * status reported by FW, bit description can be + * found in CPU_BOOT_DEV_STS1 * @collective_first_sob: first sync object available for collective use * @collective_first_mon: first monitor available for collective use * @sync_stream_first_sob: first sync object available for sync stream use @@ -438,14 +461,19 @@ struct hl_mmu_properties { * @user_interrupt_count: number of user interrupts. * @tpc_enabled_mask: which TPCs are enabled. * @completion_queues_count: number of completion queues. - * @fw_security_disabled: true if security measures are disabled in firmware, - * false otherwise - * @fw_security_status_valid: security status bits are valid and can be fetched - * from BOOT_DEV_STS0 + * @fw_security_enabled: true if security measures are enabled in firmware, + * false otherwise + * @fw_cpu_boot_dev_sts0_valid: status bits are valid and can be fetched from + * BOOT_DEV_STS0 + * @fw_cpu_boot_dev_sts1_valid: status bits are valid and can be fetched from + * BOOT_DEV_STS1 * @dram_supports_virtual_memory: is there an MMU towards the DRAM * @hard_reset_done_by_fw: true if firmware is handling hard reset flow * @num_functional_hbms: number of functional HBMs in each DCORE. * @iatu_done_by_fw: true if iATU configuration is being done by FW. + * @dynamic_fw_load: is dynamic FW load is supported. + * @gic_interrupts_enable: true if FW is not blocking GIC controller, + * false otherwise. */ struct asic_fixed_properties { struct hw_queue_properties *hw_queues_props; @@ -491,8 +519,12 @@ struct asic_fixed_properties { u32 cb_pool_cb_size; u32 max_pending_cs; u32 max_queues; - u32 fw_boot_cpu_security_map; - u32 fw_app_security_map; + u32 fw_preboot_cpu_boot_dev_sts0; + u32 fw_preboot_cpu_boot_dev_sts1; + u32 fw_bootfit_cpu_boot_dev_sts0; + u32 fw_bootfit_cpu_boot_dev_sts1; + u32 fw_app_cpu_boot_dev_sts0; + u32 fw_app_cpu_boot_dev_sts1; u16 collective_first_sob; u16 collective_first_mon; u16 sync_stream_first_sob; @@ -504,12 +536,15 @@ struct asic_fixed_properties { u16 user_interrupt_count; u8 tpc_enabled_mask; u8 completion_queues_count; - u8 fw_security_disabled; - u8 fw_security_status_valid; + u8 fw_security_enabled; + u8 fw_cpu_boot_dev_sts0_valid; + u8 fw_cpu_boot_dev_sts1_valid; u8 dram_supports_virtual_memory; u8 hard_reset_done_by_fw; u8 num_functional_hbms; u8 iatu_done_by_fw; + u8 dynamic_fw_load; + u8 gic_interrupts_enable; }; /** @@ -750,12 +785,19 @@ struct hl_user_pending_interrupt { * @kernel_address: holds the queue's kernel virtual address * @bus_address: holds the queue's DMA address * @ci: ci inside the queue + * @prev_eqe_index: the index of the previous event queue entry. The index of + * the current entry's index must be +1 of the previous one. + * @check_eqe_index: do we need to check the index of the current entry vs. the + * previous one. This is for backward compatibility with older + * firmwares */ struct hl_eq { struct hl_device *hdev; void *kernel_address; dma_addr_t bus_address; u32 ci; + u32 prev_eqe_index; + bool check_eqe_index; }; @@ -812,6 +854,132 @@ enum div_select_defs { DIV_SEL_DIVIDED_PLL = 3, }; +enum pci_region { + PCI_REGION_CFG, + PCI_REGION_SRAM, + PCI_REGION_DRAM, + PCI_REGION_SP_SRAM, + PCI_REGION_NUMBER, +}; + +/** + * struct pci_mem_region - describe memory region in a PCI bar + * @region_base: region base address + * @region_size: region size + * @bar_size: size of the BAR + * @offset_in_bar: region offset into the bar + * @bar_id: bar ID of the region + * @used: if used 1, otherwise 0 + */ +struct pci_mem_region { + u64 region_base; + u64 region_size; + u64 bar_size; + u32 offset_in_bar; + u8 bar_id; + u8 used; +}; + +/** + * struct static_fw_load_mgr - static FW load manager + * @preboot_version_max_off: max offset to preboot version + * @boot_fit_version_max_off: max offset to boot fit version + * @kmd_msg_to_cpu_reg: register address for KDM->CPU messages + * @cpu_cmd_status_to_host_reg: register address for CPU command status response + * @cpu_boot_status_reg: boot status register + * @cpu_boot_dev_status0_reg: boot device status register 0 + * @cpu_boot_dev_status1_reg: boot device status register 1 + * @boot_err0_reg: boot error register 0 + * @boot_err1_reg: boot error register 1 + * @preboot_version_offset_reg: SRAM offset to preboot version register + * @boot_fit_version_offset_reg: SRAM offset to boot fit version register + * @sram_offset_mask: mask for getting offset into the SRAM + * @cpu_reset_wait_msec: used when setting WFE via kmd_msg_to_cpu_reg + */ +struct static_fw_load_mgr { + u64 preboot_version_max_off; + u64 boot_fit_version_max_off; + u32 kmd_msg_to_cpu_reg; + u32 cpu_cmd_status_to_host_reg; + u32 cpu_boot_status_reg; + u32 cpu_boot_dev_status0_reg; + u32 cpu_boot_dev_status1_reg; + u32 boot_err0_reg; + u32 boot_err1_reg; + u32 preboot_version_offset_reg; + u32 boot_fit_version_offset_reg; + u32 sram_offset_mask; + u32 cpu_reset_wait_msec; +}; + +/** + * struct fw_response - FW response to LKD command + * @ram_offset: descriptor offset into the RAM + * @ram_type: RAM type containing the descriptor (SRAM/DRAM) + * @status: command status + */ +struct fw_response { + u32 ram_offset; + u8 ram_type; + u8 status; +}; + +/** + * struct dynamic_fw_load_mgr - dynamic FW load manager + * @response: FW to LKD response + * @comm_desc: the communication descriptor with FW + * @image_region: region to copy the FW image to + * @fw_image_size: size of FW image to load + * @wait_for_bl_timeout: timeout for waiting for boot loader to respond + */ +struct dynamic_fw_load_mgr { + struct fw_response response; + struct lkd_fw_comms_desc comm_desc; + struct pci_mem_region *image_region; + size_t fw_image_size; + u32 wait_for_bl_timeout; +}; + +/** + * struct fw_image_props - properties of FW image + * @image_name: name of the image + * @src_off: offset in src FW to copy from + * @copy_size: amount of bytes to copy (0 to copy the whole binary) + */ +struct fw_image_props { + char *image_name; + u32 src_off; + u32 copy_size; +}; + +/** + * struct fw_load_mgr - manager FW loading process + * @dynamic_loader: specific structure for dynamic load + * @static_loader: specific structure for static load + * @boot_fit_img: boot fit image properties + * @linux_img: linux image properties + * @cpu_timeout: CPU response timeout in usec + * @boot_fit_timeout: Boot fit load timeout in usec + * @skip_bmc: should BMC be skipped + * @sram_bar_id: SRAM bar ID + * @dram_bar_id: DRAM bar ID + * @linux_loaded: true if linux was loaded so far + */ +struct fw_load_mgr { + union { + struct dynamic_fw_load_mgr dynamic_loader; + struct static_fw_load_mgr static_loader; + }; + struct fw_image_props boot_fit_img; + struct fw_image_props linux_img; + u32 cpu_timeout; + u32 boot_fit_timeout; + u8 skip_bmc; + u8 sram_bar_id; + u8 dram_bar_id; + u8 linux_loaded; +}; + /** * struct hl_asic_funcs - ASIC specific functions that are can be called from * common code. @@ -901,8 +1069,6 @@ enum div_select_defs { * @ctx_fini: context dependent cleanup. * @get_clk_rate: Retrieve the ASIC current and maximum clock rate in MHz * @get_queue_id_for_cq: Get the H/W queue id related to the given CQ index. - * @read_device_fw_version: read the device's firmware versions that are - * contained in registers * @load_firmware_to_device: load the firmware to the device's memory * @load_boot_fit_to_device: load boot fit to device's memory * @get_signal_cb_size: Get signal CB size. @@ -933,6 +1099,8 @@ enum div_select_defs { * @get_msi_info: Retrieve asic-specific MSI ID of the f/w async event * @map_pll_idx_to_fw_idx: convert driver specific per asic PLL index to * generic f/w compatible PLL Indexes + * @init_firmware_loader: initialize data for FW loader. + * @init_cpu_scrambler_dram: Enable CPU specific DRAM scrambling */ struct hl_asic_funcs { int (*early_init)(struct hl_device *hdev); @@ -1006,7 +1174,7 @@ struct hl_asic_funcs { int (*mmu_invalidate_cache)(struct hl_device *hdev, bool is_hard, u32 flags); int (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard, - u32 asid, u64 va, u64 size); + u32 flags, u32 asid, u64 va, u64 size); int (*send_heartbeat)(struct hl_device *hdev); void (*set_clock_gating)(struct hl_device *hdev); void (*disable_clock_gating)(struct hl_device *hdev); @@ -1030,8 +1198,6 @@ struct hl_asic_funcs { void (*ctx_fini)(struct hl_ctx *ctx); int (*get_clk_rate)(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk); u32 (*get_queue_id_for_cq)(struct hl_device *hdev, u32 cq_idx); - int (*read_device_fw_version)(struct hl_device *hdev, - enum hl_fw_component fwc); int (*load_firmware_to_device)(struct hl_device *hdev); int (*load_boot_fit_to_device)(struct hl_device *hdev); u32 (*get_signal_cb_size)(struct hl_device *hdev); @@ -1056,8 +1222,10 @@ struct hl_asic_funcs { int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma, u32 block_id, u32 block_size); void (*enable_events_from_fw)(struct hl_device *hdev); - void (*get_msi_info)(u32 *table); + void (*get_msi_info)(__le32 *table); int (*map_pll_idx_to_fw_idx)(u32 pll_idx); + void (*init_firmware_loader)(struct hl_device *hdev); + void (*init_cpu_scrambler_dram)(struct hl_device *hdev); }; @@ -1262,6 +1430,7 @@ struct hl_userptr { * @staged_sequence: the sequence of the staged submission this CS is part of, * relevant only if staged_cs is set. * @timeout_jiffies: cs timeout in jiffies. + * @submission_time_jiffies: submission time of the cs * @type: CS_TYPE_*. * @submitted: true if CS was submitted to H/W. * @completed: true if CS was completed by device. @@ -1274,6 +1443,8 @@ struct hl_userptr { * @staged_first: true if this is the first staged CS and we need to receive * timeout for this CS. * @staged_cs: true if this CS is part of a staged submission. + * @skip_reset_on_timeout: true if we shall not reset the device in case + * timeout occurs (debug scenario). */ struct hl_cs { u16 *jobs_in_queue_cnt; @@ -1291,6 +1462,7 @@ struct hl_cs { u64 sequence; u64 staged_sequence; u64 timeout_jiffies; + u64 submission_time_jiffies; enum hl_cs_type type; u8 submitted; u8 completed; @@ -1301,6 +1473,7 @@ struct hl_cs { u8 staged_last; u8 staged_first; u8 staged_cs; + u8 skip_reset_on_timeout; }; /** @@ -1922,7 +2095,7 @@ struct hl_mmu_funcs { * @kernel_queues: array of hl_hw_queue. * @cs_mirror_list: CS mirror list for TDR. * @cs_mirror_lock: protects cs_mirror_list. - * @kernel_cb_mgr: command buffer manager for creating/destroying/handling CGs. + * @kernel_cb_mgr: command buffer manager for creating/destroying/handling CBs. * @event_queue: event queue for IRQ from CPU-CP. * @dma_pool: DMA pool for small allocations. * @cpu_accessible_dma_mem: Host <-> CPU-CP shared memory CPU address. @@ -1954,6 +2127,8 @@ struct hl_mmu_funcs { * @aggregated_cs_counters: aggregated cs counters among all contexts * @mmu_priv: device-specific MMU data. * @mmu_func: device-related MMU functions. + * @fw_loader: FW loader manager. + * @pci_mem_region: array of memory regions in the PCI * @dram_used_mem: current DRAM memory consumption. * @timeout_jiffies: device CS timeout value. * @max_power: the max power of the device, as configured by the sysadmin. This @@ -1968,6 +2143,11 @@ struct hl_mmu_funcs { * the error will be ignored by the driver during * device initialization. Mainly used to debug and * workaround firmware bugs + * @last_successful_open_jif: timestamp (jiffies) of the last successful + * device open. + * @last_open_session_duration_jif: duration (jiffies) of the last device open + * session. + * @open_counter: number of successful device open operations. * @in_reset: is device in reset flow. * @curr_pll_profile: current PLL profile. * @card_type: Various ASICs have several card types. This indicates the card @@ -2007,6 +2187,8 @@ struct hl_mmu_funcs { * @collective_mon_idx: helper index for collective initialization * @supports_coresight: is CoreSight supported. * @supports_soft_reset: is soft reset supported. + * @allow_external_soft_reset: true if soft reset initiated by user or TDR is + * allowed. * @supports_cb_mapping: is mapping a CB to the device's MMU supported. * @needs_reset: true if reset_on_lockup is false and device should be reset * due to lockup. @@ -2015,6 +2197,14 @@ struct hl_mmu_funcs { * @device_fini_pending: true if device_fini was called and might be * waiting for the reset thread to finish * @supports_staged_submission: true if staged submissions are supported + * @curr_reset_cause: saves an enumerated reset cause when a hard reset is + * triggered, and cleared after it is shared with preboot. + * @skip_reset_on_timeout: Skip device reset if CS has timed out, wait for it to + * complete instead. + * @device_cpu_is_halted: Flag to indicate whether the device CPU was already + * halted. We can't halt it again because the COMMS + * protocol will throw an error. Relevant only for + * cases where Linux was not loaded to device CPU */ struct hl_device { struct pci_dev *pdev; @@ -2079,11 +2269,18 @@ struct hl_device { struct hl_mmu_priv mmu_priv; struct hl_mmu_funcs mmu_func[MMU_NUM_PGT_LOCATIONS]; + struct fw_load_mgr fw_loader; + + struct pci_mem_region pci_mem_region[PCI_REGION_NUMBER]; + atomic64_t dram_used_mem; u64 timeout_jiffies; u64 max_power; u64 clock_gating_mask; u64 boot_error_status_mask; + u64 last_successful_open_jif; + u64 last_open_session_duration_jif; + u64 open_counter; atomic_t in_reset; enum hl_pll_frequency curr_pll_profile; enum cpucp_card_types card_type; @@ -2116,11 +2313,15 @@ struct hl_device { u8 collective_mon_idx; u8 supports_coresight; u8 supports_soft_reset; + u8 allow_external_soft_reset; u8 supports_cb_mapping; u8 needs_reset; u8 process_kill_trial_cnt; u8 device_fini_pending; u8 supports_staged_submission; + u8 curr_reset_cause; + u8 skip_reset_on_timeout; + u8 device_cpu_is_halted; /* Parameters for bring-up */ u64 nic_ports_mask; @@ -2138,6 +2339,7 @@ struct hl_device { u8 rl_enable; u8 reset_on_preboot_fail; u8 reset_upon_device_release; + u8 reset_if_device_not_idle; }; @@ -2384,11 +2586,13 @@ void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr); int hl_fw_send_heartbeat(struct hl_device *hdev); int hl_fw_cpucp_info_get(struct hl_device *hdev, - u32 cpu_security_boot_status_reg, - u32 boot_err0_reg); + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, + u32 boot_err1_reg); int hl_fw_cpucp_handshake(struct hl_device *hdev, - u32 cpu_security_boot_status_reg, - u32 boot_err0_reg); + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, + u32 boot_err1_reg); int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size); int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev, struct hl_info_pci_counters *counters); @@ -2399,14 +2603,17 @@ int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index, int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index, u16 *pll_freq_arr); int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power); -int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, - u32 msg_to_cpu_reg, u32 cpu_msg_status_reg, - u32 cpu_security_boot_status_reg, u32 boot_err0_reg, - bool skip_bmc, u32 cpu_timeout, u32 boot_fit_timeout); +void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev); +void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev); +int hl_fw_init_cpu(struct hl_device *hdev); int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, - u32 cpu_security_boot_status_reg, u32 boot_err0_reg, - u32 timeout); - + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, + u32 boot_err1_reg, u32 timeout); +int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + enum comms_cmd cmd, unsigned int size, + bool wait_ok, u32 timeout); int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3], bool is_wc[3]); int hl_pci_elbi_read(struct hl_device *hdev, u64 addr, u32 *data); @@ -2415,6 +2622,7 @@ int hl_pci_set_inbound_region(struct hl_device *hdev, u8 region, struct hl_inbound_pci_region *pci_region); int hl_pci_set_outbound_region(struct hl_device *hdev, struct hl_outbound_pci_region *pci_region); +enum pci_region hl_get_pci_memory_region(struct hl_device *hdev, u64 addr); int hl_pci_init(struct hl_device *hdev); void hl_pci_fini(struct hl_device *hdev); @@ -2443,6 +2651,8 @@ int hl_set_voltage(struct hl_device *hdev, int hl_set_current(struct hl_device *hdev, int sensor_index, u32 attr, long value); void hl_release_pending_user_interrupts(struct hl_device *hdev); +int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, + struct hl_hw_sob **hw_sob, u32 count); #ifdef CONFIG_DEBUG_FS diff --git a/drivers/misc/habanalabs/common/habanalabs_drv.c b/drivers/misc/habanalabs/common/habanalabs_drv.c index 64d1530db985..4194cda2d04c 100644 --- a/drivers/misc/habanalabs/common/habanalabs_drv.c +++ b/drivers/misc/habanalabs/common/habanalabs_drv.c @@ -29,7 +29,7 @@ static DEFINE_MUTEX(hl_devs_idr_lock); static int timeout_locked = 30; static int reset_on_lockup = 1; -static int memory_scrub = 1; +static int memory_scrub; static ulong boot_error_status_mask = ULONG_MAX; module_param(timeout_locked, int, 0444); @@ -42,7 +42,7 @@ MODULE_PARM_DESC(reset_on_lockup, module_param(memory_scrub, int, 0444); MODULE_PARM_DESC(memory_scrub, - "Scrub device memory in various states (0 = no, 1 = yes, default yes)"); + "Scrub device memory in various states (0 = no, 1 = yes, default no)"); module_param(boot_error_status_mask, ulong, 0444); MODULE_PARM_DESC(boot_error_status_mask, @@ -187,6 +187,9 @@ int hl_device_open(struct inode *inode, struct file *filp) hl_debugfs_add_file(hpriv); + hdev->open_counter++; + hdev->last_successful_open_jif = jiffies; + return 0; out_err: @@ -264,6 +267,7 @@ static void set_driver_behavior_per_device(struct hl_device *hdev) hdev->bmc_enable = 1; hdev->hard_reset_on_fw_events = 1; hdev->reset_on_preboot_fail = 1; + hdev->reset_if_device_not_idle = 1; hdev->reset_pcilink = 0; hdev->axi_drain = 0; @@ -308,10 +312,10 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev, } if (pdev) - hdev->asic_prop.fw_security_disabled = - !is_asic_secured(pdev->device); + hdev->asic_prop.fw_security_enabled = + is_asic_secured(hdev->asic_type); else - hdev->asic_prop.fw_security_disabled = true; + hdev->asic_prop.fw_security_enabled = false; /* Assign status description string */ strncpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION], @@ -325,11 +329,14 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev, hdev->reset_on_lockup = reset_on_lockup; hdev->memory_scrub = memory_scrub; hdev->boot_error_status_mask = boot_error_status_mask; + hdev->stop_on_err = true; hdev->pldm = 0; set_driver_behavior_per_device(hdev); + hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; + if (timeout_locked) hdev->timeout_jiffies = msecs_to_jiffies(timeout_locked * 1000); else @@ -464,6 +471,7 @@ static int hl_pci_probe(struct pci_dev *pdev, return 0; disable_device: + pci_disable_pcie_error_reporting(pdev); pci_set_drvdata(pdev, NULL); destroy_hdev(hdev); @@ -572,7 +580,11 @@ static struct pci_driver hl_pci_driver = { .probe = hl_pci_probe, .remove = hl_pci_remove, .shutdown = hl_pci_remove, - .driver.pm = &hl_pm_ops, + .driver = { + .name = HL_NAME, + .pm = &hl_pm_ops, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, .err_handler = &hl_pci_err_handler, }; diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index 33841c272eb6..f4dda7b4acdd 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -95,7 +95,7 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args) hw_ip.first_available_interrupt_id = prop->first_available_user_msix_interrupt; return copy_to_user(out, &hw_ip, - min((size_t)size, sizeof(hw_ip))) ? -EFAULT : 0; + min((size_t) size, sizeof(hw_ip))) ? -EFAULT : 0; } static int hw_events_info(struct hl_device *hdev, bool aggregate, @@ -460,6 +460,24 @@ static int power_info(struct hl_fpriv *hpriv, struct hl_info_args *args) min((size_t) max_size, sizeof(power_info))) ? -EFAULT : 0; } +static int open_stats_info(struct hl_fpriv *hpriv, struct hl_info_args *args) +{ + struct hl_device *hdev = hpriv->hdev; + u32 max_size = args->return_size; + struct hl_open_stats_info open_stats_info = {0}; + void __user *out = (void __user *) (uintptr_t) args->return_pointer; + + if ((!max_size) || (!out)) + return -EINVAL; + + open_stats_info.last_open_period_ms = jiffies64_to_msecs( + hdev->last_open_session_duration_jif); + open_stats_info.open_counter = hdev->open_counter; + + return copy_to_user(out, &open_stats_info, + min((size_t) max_size, sizeof(open_stats_info))) ? -EFAULT : 0; +} + static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, struct device *dev) { @@ -543,6 +561,9 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, case HL_INFO_POWER: return power_info(hpriv, args); + case HL_INFO_OPEN_STATS: + return open_stats_info(hpriv, args); + default: dev_err(dev, "Invalid request %d\n", args->op); rc = -ENOTTY; diff --git a/drivers/misc/habanalabs/common/hw_queue.c b/drivers/misc/habanalabs/common/hw_queue.c index 173438461835..bcabfdbf1e01 100644 --- a/drivers/misc/habanalabs/common/hw_queue.c +++ b/drivers/misc/habanalabs/common/hw_queue.c @@ -410,19 +410,20 @@ static void hw_queue_schedule_job(struct hl_cs_job *job) ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr); } -static void init_signal_cs(struct hl_device *hdev, +static int init_signal_cs(struct hl_device *hdev, struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl) { struct hl_sync_stream_properties *prop; struct hl_hw_sob *hw_sob; u32 q_idx; + int rc = 0; q_idx = job->hw_queue_id; prop = &hdev->kernel_queues[q_idx].sync_stream_prop; hw_sob = &prop->hw_sob[prop->curr_sob_offset]; cs_cmpl->hw_sob = hw_sob; - cs_cmpl->sob_val = prop->next_sob_val++; + cs_cmpl->sob_val = prop->next_sob_val; dev_dbg(hdev->dev, "generate signal CB, sob_id: %d, sob val: 0x%x, q_idx: %d\n", @@ -434,24 +435,9 @@ static void init_signal_cs(struct hl_device *hdev, hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb, cs_cmpl->hw_sob->sob_id, 0, true); - kref_get(&hw_sob->kref); + rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1); - /* check for wraparound */ - if (prop->next_sob_val == HL_MAX_SOB_VAL) { - /* - * Decrement as we reached the max value. - * The release function won't be called here as we've - * just incremented the refcount. - */ - kref_put(&hw_sob->kref, hl_sob_reset_error); - prop->next_sob_val = 1; - /* only two SOBs are currently in use */ - prop->curr_sob_offset = - (prop->curr_sob_offset + 1) % HL_RSVD_SOBS; - - dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n", - prop->curr_sob_offset, q_idx); - } + return rc; } static void init_wait_cs(struct hl_device *hdev, struct hl_cs *cs, @@ -504,22 +490,25 @@ static void init_wait_cs(struct hl_device *hdev, struct hl_cs *cs, * * H/W queues spinlock should be taken before calling this function */ -static void init_signal_wait_cs(struct hl_cs *cs) +static int init_signal_wait_cs(struct hl_cs *cs) { struct hl_ctx *ctx = cs->ctx; struct hl_device *hdev = ctx->hdev; struct hl_cs_job *job; struct hl_cs_compl *cs_cmpl = container_of(cs->fence, struct hl_cs_compl, base_fence); + int rc = 0; /* There is only one job in a signal/wait CS */ job = list_first_entry(&cs->job_list, struct hl_cs_job, cs_node); if (cs->type & CS_TYPE_SIGNAL) - init_signal_cs(hdev, job, cs_cmpl); + rc = init_signal_cs(hdev, job, cs_cmpl); else if (cs->type & CS_TYPE_WAIT) init_wait_cs(hdev, cs, job, cs_cmpl); + + return rc; } /* @@ -590,11 +579,16 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) } } - if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) - init_signal_wait_cs(cs); - else if (cs->type == CS_TYPE_COLLECTIVE_WAIT) + if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) { + rc = init_signal_wait_cs(cs); + if (rc) { + dev_err(hdev->dev, "Failed to submit signal cs\n"); + goto unroll_cq_resv; + } + } else if (cs->type == CS_TYPE_COLLECTIVE_WAIT) hdev->asic_funcs->collective_wait_init_cs(cs); + spin_lock(&hdev->cs_mirror_lock); /* Verify staged CS exists and add to the staged list */ diff --git a/drivers/misc/habanalabs/common/irq.c b/drivers/misc/habanalabs/common/irq.c index 27129868c711..39b14a933393 100644 --- a/drivers/misc/habanalabs/common/irq.c +++ b/drivers/misc/habanalabs/common/irq.c @@ -207,17 +207,33 @@ irqreturn_t hl_irq_handler_eq(int irq, void *arg) struct hl_eq_entry *eq_entry; struct hl_eq_entry *eq_base; struct hl_eqe_work *handle_eqe_work; + bool entry_ready; + u32 cur_eqe; + u16 cur_eqe_index; eq_base = eq->kernel_address; while (1) { - bool entry_ready = - ((le32_to_cpu(eq_base[eq->ci].hdr.ctl) & - EQ_CTL_READY_MASK) >> EQ_CTL_READY_SHIFT); + cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl); + entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe); if (!entry_ready) break; + cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe); + if ((hdev->event_queue.check_eqe_index) && + (((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK) + != cur_eqe_index)) { + dev_dbg(hdev->dev, + "EQE 0x%x in queue is ready but index does not match %d!=%d", + eq_base[eq->ci].hdr.ctl, + ((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK), + cur_eqe_index); + break; + } + + eq->prev_eqe_index++; + eq_entry = &eq_base[eq->ci]; /* @@ -341,6 +357,7 @@ int hl_eq_init(struct hl_device *hdev, struct hl_eq *q) q->hdev = hdev; q->kernel_address = p; q->ci = 0; + q->prev_eqe_index = 0; return 0; } @@ -365,6 +382,7 @@ void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q) void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q) { q->ci = 0; + q->prev_eqe_index = 0; /* * It's not enough to just reset the PI/CI because the H/W may have diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index 2938cbbafbbc..af339ce1ab4f 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -570,8 +570,10 @@ static u64 get_va_block(struct hl_device *hdev, if ((is_align_pow_2 && (hint_addr & (va_block_align - 1))) || (!is_align_pow_2 && do_div(tmp_hint_addr, va_range->page_size))) { - dev_info(hdev->dev, "Hint address 0x%llx will be ignored\n", - hint_addr); + + dev_dbg(hdev->dev, + "Hint address 0x%llx will be ignored because it is not aligned\n", + hint_addr); hint_addr = 0; } @@ -1117,7 +1119,8 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, goto map_err; } - rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, false, *vm_type); + rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, false, + *vm_type, ctx->asid, ret_vaddr, phys_pg_pack->total_size); mutex_unlock(&ctx->mmu_lock); @@ -1261,8 +1264,9 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, * at the loop end rather than for each iteration */ if (!ctx_free) - rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, true, - *vm_type); + rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, true, + *vm_type, ctx->asid, vaddr, + phys_pg_pack->total_size); mutex_unlock(&ctx->mmu_lock); @@ -1369,12 +1373,7 @@ int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma) /* Driver only allows mapping of a complete HW block */ block_size = vma->vm_end - vma->vm_start; -#ifdef _HAS_TYPE_ARG_IN_ACCESS_OK - if (!access_ok(VERIFY_WRITE, - (void __user *) (uintptr_t) vma->vm_start, block_size)) { -#else if (!access_ok((void __user *) (uintptr_t) vma->vm_start, block_size)) { -#endif dev_err(hdev->dev, "user pointer is invalid - 0x%lx\n", vma->vm_start); @@ -1608,7 +1607,8 @@ static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size, if (rc != npages) { dev_err(hdev->dev, - "Failed to map host memory, user ptr probably wrong\n"); + "Failed (%d) to pin host memory with user ptr 0x%llx, size 0x%llx, npages %d\n", + rc, addr, size, npages); if (rc < 0) goto destroy_pages; npages = rc; diff --git a/drivers/misc/habanalabs/common/mmu/mmu.c b/drivers/misc/habanalabs/common/mmu/mmu.c index b37189956b14..792d25b79ea6 100644 --- a/drivers/misc/habanalabs/common/mmu/mmu.c +++ b/drivers/misc/habanalabs/common/mmu/mmu.c @@ -501,12 +501,20 @@ static void hl_mmu_pa_page_with_offset(struct hl_ctx *ctx, u64 virt_addr, if ((hops->range_type == HL_VA_RANGE_TYPE_DRAM) && !is_power_of_2(prop->dram_page_size)) { - u32 bit; + unsigned long dram_page_size = prop->dram_page_size; u64 page_offset_mask; u64 phys_addr_mask; + u32 bit; - bit = __ffs64((u64)prop->dram_page_size); - page_offset_mask = ((1ull << bit) - 1); + /* + * find last set bit in page_size to cover all bits of page + * offset. note that 1 has to be added to bit index. + * note that the internal ulong variable is used to avoid + * alignment issue. + */ + bit = find_last_bit(&dram_page_size, + sizeof(dram_page_size) * BITS_PER_BYTE) + 1; + page_offset_mask = (BIT_ULL(bit) - 1); phys_addr_mask = ~page_offset_mask; *phys_addr = (tmp_phys_addr & phys_addr_mask) | (virt_addr & page_offset_mask); diff --git a/drivers/misc/habanalabs/common/pci/pci.c b/drivers/misc/habanalabs/common/pci/pci.c index e941b7eef346..d5bedf5ba011 100644 --- a/drivers/misc/habanalabs/common/pci/pci.c +++ b/drivers/misc/habanalabs/common/pci/pci.c @@ -10,7 +10,7 @@ #include <linux/pci.h> -#define HL_PLDM_PCI_ELBI_TIMEOUT_MSEC (HL_PCI_ELBI_TIMEOUT_MSEC * 10) +#define HL_PLDM_PCI_ELBI_TIMEOUT_MSEC (HL_PCI_ELBI_TIMEOUT_MSEC * 100) #define IATU_REGION_CTRL_REGION_EN_MASK BIT(31) #define IATU_REGION_CTRL_MATCH_MODE_MASK BIT(30) @@ -360,6 +360,32 @@ int hl_pci_set_outbound_region(struct hl_device *hdev, } /** + * hl_get_pci_memory_region() - get PCI region for given address + * @hdev: Pointer to hl_device structure. + * @addr: device address + * + * @return region index on success, otherwise PCI_REGION_NUMBER (invalid + * region index) + */ +enum pci_region hl_get_pci_memory_region(struct hl_device *hdev, u64 addr) +{ + int i; + + for (i = 0 ; i < PCI_REGION_NUMBER ; i++) { + struct pci_mem_region *region = &hdev->pci_mem_region[i]; + + if (!region->used) + continue; + + if ((addr >= region->region_base) && + (addr < region->region_base + region->region_size)) + return i; + } + + return PCI_REGION_NUMBER; +} + +/** * hl_pci_init() - PCI initialization code. * @hdev: Pointer to hl_device structure. * @@ -395,6 +421,12 @@ int hl_pci_init(struct hl_device *hdev) goto unmap_pci_bars; } + /* Driver must sleep in order for FW to finish the iATU configuration */ + if (hdev->asic_prop.iatu_done_by_fw) { + usleep_range(2000, 3000); + hdev->asic_funcs->set_dma_mask_from_fw(hdev); + } + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(hdev->dma_mask)); if (rc) { diff --git a/drivers/misc/habanalabs/common/sysfs.c b/drivers/misc/habanalabs/common/sysfs.c index c9f649b31e3a..db72df282ef8 100644 --- a/drivers/misc/habanalabs/common/sysfs.c +++ b/drivers/misc/habanalabs/common/sysfs.c @@ -208,7 +208,7 @@ static ssize_t soft_reset_store(struct device *dev, goto out; } - if (!hdev->supports_soft_reset) { + if (!hdev->allow_external_soft_reset) { dev_err(hdev->dev, "Device does not support soft-reset\n"); goto out; } diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 9e4a6bb3acd1..aa8a0ca5aca2 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -78,6 +78,7 @@ #define GAUDI_PLDM_TPC_KERNEL_WAIT_USEC (HL_DEVICE_TIMEOUT_USEC * 30) #define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC 1000000 /* 1s */ #define GAUDI_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */ +#define GAUDI_WAIT_FOR_BL_TIMEOUT_USEC 15000000 /* 15s */ #define GAUDI_QMAN0_FENCE_VAL 0x72E91AB9 @@ -409,7 +410,7 @@ static inline void set_default_power_values(struct hl_device *hdev) } } -static int gaudi_get_fixed_properties(struct hl_device *hdev) +static int gaudi_set_fixed_properties(struct hl_device *hdev) { struct asic_fixed_properties *prop = &hdev->asic_prop; u32 num_sync_stream_queues = 0; @@ -545,8 +546,10 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev) for (i = 0 ; i < HL_MAX_DCORES ; i++) prop->first_available_cq[i] = USHRT_MAX; - prop->fw_security_status_valid = false; + prop->fw_cpu_boot_dev_sts0_valid = false; + prop->fw_cpu_boot_dev_sts1_valid = false; prop->hard_reset_done_by_fw = false; + prop->gic_interrupts_enable = true; return 0; } @@ -577,6 +580,9 @@ static u64 gaudi_set_hbm_bar_base(struct hl_device *hdev, u64 addr) if ((gaudi) && (gaudi->hbm_bar_cur_addr == addr)) return old_addr; + if (hdev->asic_prop.iatu_done_by_fw) + return U64_MAX; + /* Inbound Region 2 - Bar 4 - Point to HBM */ pci_region.mode = PCI_BAR_MATCH_MODE; pci_region.bar = HBM_BAR_ID; @@ -599,10 +605,8 @@ static int gaudi_init_iatu(struct hl_device *hdev) struct hl_outbound_pci_region outbound_region; int rc; - if (hdev->asic_prop.iatu_done_by_fw) { - hdev->asic_funcs->set_dma_mask_from_fw(hdev); + if (hdev->asic_prop.iatu_done_by_fw) return 0; - } /* Inbound Region 0 - Bar 0 - Point to SRAM + CFG */ inbound_region.mode = PCI_BAR_MATCH_MODE; @@ -651,9 +655,9 @@ static int gaudi_early_init(struct hl_device *hdev) u32 fw_boot_status; int rc; - rc = gaudi_get_fixed_properties(hdev); + rc = gaudi_set_fixed_properties(hdev); if (rc) { - dev_err(hdev->dev, "Failed to get fixed properties\n"); + dev_err(hdev->dev, "Failed setting fixed properties\n"); return rc; } @@ -683,8 +687,14 @@ static int gaudi_early_init(struct hl_device *hdev) prop->dram_pci_bar_size = pci_resource_len(pdev, HBM_BAR_ID); /* If FW security is enabled at this point it means no access to ELBI */ - if (!hdev->asic_prop.fw_security_disabled) { + if (hdev->asic_prop.fw_security_enabled) { hdev->asic_prop.iatu_done_by_fw = true; + + /* + * GIC-security-bit can ONLY be set by CPUCP, so in this stage + * decision can only be taken based on PCI ID security. + */ + hdev->asic_prop.gic_interrupts_enable = false; goto pci_init; } @@ -707,8 +717,10 @@ pci_init: * version to determine whether we run with a security-enabled firmware */ rc = hl_fw_read_preboot_status(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS, - mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0, - GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC); + mmCPU_BOOT_DEV_STS0, + mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, + mmCPU_BOOT_ERR1, + GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC); if (rc) { if (hdev->reset_on_preboot_fail) hdev->asic_funcs->hw_fini(hdev, true); @@ -751,7 +763,14 @@ static int gaudi_fetch_psoc_frequency(struct hl_device *hdev) u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS], freq; int rc; - if (hdev->asic_prop.fw_security_disabled) { + if (hdev->asic_prop.fw_security_enabled) { + rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI_CPU_PLL, pll_freq_arr); + + if (rc) + return rc; + + freq = pll_freq_arr[2]; + } else { /* Backward compatibility */ div_fctr = RREG32(mmPSOC_CPU_PLL_DIV_FACTOR_2); div_sel = RREG32(mmPSOC_CPU_PLL_DIV_SEL_2); @@ -779,13 +798,6 @@ static int gaudi_fetch_psoc_frequency(struct hl_device *hdev) div_sel); freq = 0; } - } else { - rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI_CPU_PLL, pll_freq_arr); - - if (rc) - return rc; - - freq = pll_freq_arr[2]; } prop->psoc_timestamp_frequency = freq; @@ -988,9 +1000,27 @@ static void gaudi_sob_group_reset_error(struct kref *ref) hw_sob_group->base_sob_id); } +static void gaudi_collective_mstr_sob_mask_set(struct gaudi_device *gaudi) +{ + struct gaudi_collective_properties *prop; + int i; + + prop = &gaudi->collective_props; + + memset(prop->mstr_sob_mask, 0, sizeof(prop->mstr_sob_mask)); + + for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) + if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i)) + prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |= + BIT(i % HL_MAX_SOBS_PER_MONITOR); + /* Set collective engine bit */ + prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |= + BIT(i % HL_MAX_SOBS_PER_MONITOR); +} + static int gaudi_collective_init(struct hl_device *hdev) { - u32 i, master_monitor_sobs, sob_id, reserved_sobs_per_group; + u32 i, sob_id, reserved_sobs_per_group; struct gaudi_collective_properties *prop; struct gaudi_device *gaudi; @@ -1016,22 +1046,7 @@ static int gaudi_collective_init(struct hl_device *hdev) gaudi_collective_map_sobs(hdev, i); } - prop->mstr_sob_mask[0] = 0; - master_monitor_sobs = HL_MAX_SOBS_PER_MONITOR; - for (i = 0 ; i < master_monitor_sobs ; i++) - if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i)) - prop->mstr_sob_mask[0] |= BIT(i); - - prop->mstr_sob_mask[1] = 0; - master_monitor_sobs = - NIC_NUMBER_OF_ENGINES - HL_MAX_SOBS_PER_MONITOR; - for (i = 0 ; i < master_monitor_sobs; i++) { - if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i)) - prop->mstr_sob_mask[1] |= BIT(i); - } - - /* Set collective engine bit */ - prop->mstr_sob_mask[1] |= BIT(i); + gaudi_collective_mstr_sob_mask_set(gaudi); return 0; } @@ -1513,7 +1528,7 @@ static int gaudi_alloc_cpu_accessible_dma_mem(struct hl_device *hdev) hdev->cpu_pci_msb_addr = GAUDI_CPU_PCI_MSB_ADDR(hdev->cpu_accessible_dma_address); - if (hdev->asic_prop.fw_security_disabled) + if (!hdev->asic_prop.fw_security_enabled) GAUDI_PCI_TO_CPU_ADDR(hdev->cpu_accessible_dma_address); free_dma_mem_arr: @@ -1590,6 +1605,48 @@ free_internal_qmans_pq_mem: return rc; } +static void gaudi_set_pci_memory_regions(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct pci_mem_region *region; + + /* CFG */ + region = &hdev->pci_mem_region[PCI_REGION_CFG]; + region->region_base = CFG_BASE; + region->region_size = CFG_SIZE; + region->offset_in_bar = CFG_BASE - SPI_FLASH_BASE_ADDR; + region->bar_size = CFG_BAR_SIZE; + region->bar_id = CFG_BAR_ID; + region->used = 1; + + /* SRAM */ + region = &hdev->pci_mem_region[PCI_REGION_SRAM]; + region->region_base = SRAM_BASE_ADDR; + region->region_size = SRAM_SIZE; + region->offset_in_bar = 0; + region->bar_size = SRAM_BAR_SIZE; + region->bar_id = SRAM_BAR_ID; + region->used = 1; + + /* DRAM */ + region = &hdev->pci_mem_region[PCI_REGION_DRAM]; + region->region_base = DRAM_PHYS_BASE; + region->region_size = hdev->asic_prop.dram_size; + region->offset_in_bar = 0; + region->bar_size = prop->dram_pci_bar_size; + region->bar_id = HBM_BAR_ID; + region->used = 1; + + /* SP SRAM */ + region = &hdev->pci_mem_region[PCI_REGION_SP_SRAM]; + region->region_base = PSOC_SCRATCHPAD_ADDR; + region->region_size = PSOC_SCRATCHPAD_SIZE; + region->offset_in_bar = PSOC_SCRATCHPAD_ADDR - SPI_FLASH_BASE_ADDR; + region->bar_size = CFG_BAR_SIZE; + region->bar_id = CFG_BAR_ID; + region->used = 1; +} + static int gaudi_sw_init(struct hl_device *hdev) { struct gaudi_device *gaudi; @@ -1664,12 +1721,14 @@ static int gaudi_sw_init(struct hl_device *hdev) hdev->supports_coresight = true; hdev->supports_staged_submission = true; + gaudi_set_pci_memory_regions(hdev); + return 0; free_cpu_accessible_dma_pool: gen_pool_destroy(hdev->cpu_accessible_dma_pool); free_cpu_dma_mem: - if (hdev->asic_prop.fw_security_disabled) + if (!hdev->asic_prop.fw_security_enabled) GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address, hdev->cpu_pci_msb_addr); hdev->asic_funcs->asic_dma_free_coherent(hdev, @@ -1691,7 +1750,7 @@ static int gaudi_sw_fini(struct hl_device *hdev) gen_pool_destroy(hdev->cpu_accessible_dma_pool); - if (hdev->asic_prop.fw_security_disabled) + if (!hdev->asic_prop.fw_security_enabled) GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address, hdev->cpu_pci_msb_addr); @@ -1879,12 +1938,11 @@ static void gaudi_init_scrambler_sram(struct hl_device *hdev) { struct gaudi_device *gaudi = hdev->asic_specific; - if (!hdev->asic_prop.fw_security_disabled) + if (hdev->asic_prop.fw_security_enabled) return; - if (hdev->asic_prop.fw_security_status_valid && - (hdev->asic_prop.fw_app_security_map & - CPU_BOOT_DEV_STS0_SRAM_SCR_EN)) + if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_SRAM_SCR_EN) return; if (gaudi->hw_cap_initialized & HW_CAP_SRAM_SCRAMBLER) @@ -1951,12 +2009,11 @@ static void gaudi_init_scrambler_hbm(struct hl_device *hdev) { struct gaudi_device *gaudi = hdev->asic_specific; - if (!hdev->asic_prop.fw_security_disabled) + if (hdev->asic_prop.fw_security_enabled) return; - if (hdev->asic_prop.fw_security_status_valid && - (hdev->asic_prop.fw_boot_cpu_security_map & - CPU_BOOT_DEV_STS0_DRAM_SCR_EN)) + if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_DRAM_SCR_EN) return; if (gaudi->hw_cap_initialized & HW_CAP_HBM_SCRAMBLER) @@ -2021,12 +2078,11 @@ static void gaudi_init_scrambler_hbm(struct hl_device *hdev) static void gaudi_init_e2e(struct hl_device *hdev) { - if (!hdev->asic_prop.fw_security_disabled) + if (hdev->asic_prop.fw_security_enabled) return; - if (hdev->asic_prop.fw_security_status_valid && - (hdev->asic_prop.fw_boot_cpu_security_map & - CPU_BOOT_DEV_STS0_E2E_CRED_EN)) + if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_E2E_CRED_EN) return; WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 247 >> 3); @@ -2396,12 +2452,11 @@ static void gaudi_init_hbm_cred(struct hl_device *hdev) { uint32_t hbm0_wr, hbm1_wr, hbm0_rd, hbm1_rd; - if (!hdev->asic_prop.fw_security_disabled) + if (hdev->asic_prop.fw_security_enabled) return; - if (hdev->asic_prop.fw_security_status_valid && - (hdev->asic_prop.fw_boot_cpu_security_map & - CPU_BOOT_DEV_STS0_HBM_CRED_EN)) + if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_HBM_CRED_EN) return; hbm0_wr = 0x33333333; @@ -2487,10 +2542,12 @@ static void gaudi_init_golden_registers(struct hl_device *hdev) static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id, int qman_id, dma_addr_t qman_pq_addr) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; u32 q_off, dma_qm_offset; - u32 dma_qm_err_cfg; + u32 dma_qm_err_cfg, irq_handler_offset; dma_qm_offset = dma_id * DMA_QMAN_OFFSET; @@ -2539,20 +2596,23 @@ static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id, /* The following configuration is needed only once per QMAN */ if (qman_id == 0) { + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); + /* Configure RAZWI IRQ */ dma_qm_err_cfg = PCI_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; - if (hdev->stop_on_err) { + if (hdev->stop_on_err) dma_qm_err_cfg |= PCI_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; - } WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg); + WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset, - lower_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + lower_32_bits(CFG_BASE + irq_handler_offset)); WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset, - upper_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + upper_32_bits(CFG_BASE + irq_handler_offset)); + WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset, gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id + dma_id); @@ -2573,8 +2633,11 @@ static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id, static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id) { - u32 dma_offset = dma_id * DMA_CORE_OFFSET; + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; u32 dma_err_cfg = 1 << DMA0_CORE_ERR_CFG_ERR_MSG_EN_SHIFT; + u32 dma_offset = dma_id * DMA_CORE_OFFSET; + u32 irq_handler_offset; /* Set to maximum possible according to physical size */ WREG32(mmDMA0_CORE_RD_MAX_OUTSTAND + dma_offset, 0); @@ -2588,10 +2651,16 @@ static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id) dma_err_cfg |= 1 << DMA0_CORE_ERR_CFG_STOP_ON_ERR_SHIFT; WREG32(mmDMA0_CORE_ERR_CFG + dma_offset, dma_err_cfg); + + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); + WREG32(mmDMA0_CORE_ERRMSG_ADDR_LO + dma_offset, - lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + lower_32_bits(CFG_BASE + irq_handler_offset)); WREG32(mmDMA0_CORE_ERRMSG_ADDR_HI + dma_offset, - upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + upper_32_bits(CFG_BASE + irq_handler_offset)); + WREG32(mmDMA0_CORE_ERRMSG_WDATA + dma_offset, gaudi_irq_map_table[GAUDI_EVENT_DMA0_CORE].cpu_id + dma_id); WREG32(mmDMA0_CORE_PROT + dma_offset, @@ -2654,10 +2723,12 @@ static void gaudi_init_pci_dma_qmans(struct hl_device *hdev) static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id, int qman_id, u64 qman_base_addr) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; + u32 dma_qm_err_cfg, irq_handler_offset; u32 q_off, dma_qm_offset; - u32 dma_qm_err_cfg; dma_qm_offset = dma_id * DMA_QMAN_OFFSET; @@ -2697,6 +2768,10 @@ static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id, WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, QMAN_CPDMA_DST_OFFSET); } else { + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); + WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, QMAN_LDMA_SIZE_OFFSET); WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, @@ -2706,18 +2781,17 @@ static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id, /* Configure RAZWI IRQ */ dma_qm_err_cfg = HBM_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; - if (hdev->stop_on_err) { + if (hdev->stop_on_err) dma_qm_err_cfg |= HBM_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; - } + WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg); WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset, - lower_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + lower_32_bits(CFG_BASE + irq_handler_offset)); WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset, - upper_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + upper_32_bits(CFG_BASE + irq_handler_offset)); + WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset, gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id + dma_id); @@ -2792,8 +2866,11 @@ static void gaudi_init_hbm_dma_qmans(struct hl_device *hdev) static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset, int qman_id, u64 qman_base_addr) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; u32 mtr_base_lo, mtr_base_hi; u32 so_base_lo, so_base_hi; + u32 irq_handler_offset; u32 q_off, mme_id; u32 mme_qm_err_cfg; @@ -2825,6 +2902,10 @@ static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset, WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, QMAN_CPDMA_DST_OFFSET); } else { + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); + WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, QMAN_LDMA_SIZE_OFFSET); WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, @@ -2834,20 +2915,20 @@ static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset, /* Configure RAZWI IRQ */ mme_id = mme_offset / - (mmMME1_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0); + (mmMME1_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0) / 2; mme_qm_err_cfg = MME_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; - if (hdev->stop_on_err) { + if (hdev->stop_on_err) mme_qm_err_cfg |= MME_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; - } + WREG32(mmMME0_QM_GLBL_ERR_CFG + mme_offset, mme_qm_err_cfg); + WREG32(mmMME0_QM_GLBL_ERR_ADDR_LO + mme_offset, - lower_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + lower_32_bits(CFG_BASE + irq_handler_offset)); WREG32(mmMME0_QM_GLBL_ERR_ADDR_HI + mme_offset, - upper_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + upper_32_bits(CFG_BASE + irq_handler_offset)); + WREG32(mmMME0_QM_GLBL_ERR_WDATA + mme_offset, gaudi_irq_map_table[GAUDI_EVENT_MME0_QM].cpu_id + mme_id); @@ -2912,10 +2993,12 @@ static void gaudi_init_mme_qmans(struct hl_device *hdev) static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset, int qman_id, u64 qman_base_addr) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; + u32 tpc_qm_err_cfg, irq_handler_offset; u32 q_off, tpc_id; - u32 tpc_qm_err_cfg; mtr_base_en_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); @@ -2956,6 +3039,10 @@ static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset, WREG32(mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, QMAN_CPDMA_DST_OFFSET); } else { + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); + WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, QMAN_LDMA_SIZE_OFFSET); WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, @@ -2965,18 +3052,17 @@ static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset, /* Configure RAZWI IRQ */ tpc_qm_err_cfg = TPC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; - if (hdev->stop_on_err) { + if (hdev->stop_on_err) tpc_qm_err_cfg |= TPC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; - } WREG32(mmTPC0_QM_GLBL_ERR_CFG + tpc_offset, tpc_qm_err_cfg); + WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + tpc_offset, - lower_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + lower_32_bits(CFG_BASE + irq_handler_offset)); WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + tpc_offset, - upper_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + upper_32_bits(CFG_BASE + irq_handler_offset)); + WREG32(mmTPC0_QM_GLBL_ERR_WDATA + tpc_offset, gaudi_irq_map_table[GAUDI_EVENT_TPC0_QM].cpu_id + tpc_id); @@ -3059,10 +3145,12 @@ static void gaudi_init_tpc_qmans(struct hl_device *hdev) static void gaudi_init_nic_qman(struct hl_device *hdev, u32 nic_offset, int qman_id, u64 qman_base_addr, int nic_id) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; + u32 nic_qm_err_cfg, irq_handler_offset; u32 q_off; - u32 nic_qm_err_cfg; mtr_base_en_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); @@ -3109,20 +3197,23 @@ static void gaudi_init_nic_qman(struct hl_device *hdev, u32 nic_offset, WREG32(mmNIC0_QM0_CP_MSG_BASE3_ADDR_HI_0 + q_off, so_base_ws_hi); if (qman_id == 0) { + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); + /* Configure RAZWI IRQ */ nic_qm_err_cfg = NIC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; - if (hdev->stop_on_err) { + if (hdev->stop_on_err) nic_qm_err_cfg |= NIC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; - } WREG32(mmNIC0_QM0_GLBL_ERR_CFG + nic_offset, nic_qm_err_cfg); + WREG32(mmNIC0_QM0_GLBL_ERR_ADDR_LO + nic_offset, - lower_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + lower_32_bits(CFG_BASE + irq_handler_offset)); WREG32(mmNIC0_QM0_GLBL_ERR_ADDR_HI + nic_offset, - upper_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + upper_32_bits(CFG_BASE + irq_handler_offset)); + WREG32(mmNIC0_QM0_GLBL_ERR_WDATA + nic_offset, gaudi_irq_map_table[GAUDI_EVENT_NIC0_QM0].cpu_id + nic_id); @@ -3475,7 +3566,7 @@ static void gaudi_set_clock_gating(struct hl_device *hdev) if (hdev->in_debug) return; - if (!hdev->asic_prop.fw_security_disabled) + if (hdev->asic_prop.fw_security_enabled) return; for (i = GAUDI_PCI_DMA_1, qman_offset = 0 ; i < GAUDI_HBM_DMA_1 ; i++) { @@ -3535,7 +3626,7 @@ static void gaudi_disable_clock_gating(struct hl_device *hdev) u32 qman_offset; int i; - if (!hdev->asic_prop.fw_security_disabled) + if (hdev->asic_prop.fw_security_enabled) return; for (i = 0, qman_offset = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) { @@ -3674,9 +3765,6 @@ static int gaudi_load_firmware_to_device(struct hl_device *hdev) { void __iomem *dst; - /* HBM scrambler must be initialized before pushing F/W to HBM */ - gaudi_init_scrambler_hbm(hdev); - dst = hdev->pcie_bar[HBM_BAR_ID] + LINUX_FW_OFFSET; return hl_fw_load_fw_to_device(hdev, GAUDI_LINUX_FW_FILE, dst, 0, 0); @@ -3691,42 +3779,71 @@ static int gaudi_load_boot_fit_to_device(struct hl_device *hdev) return hl_fw_load_fw_to_device(hdev, GAUDI_BOOT_FIT_FILE, dst, 0, 0); } -static int gaudi_read_device_fw_version(struct hl_device *hdev, - enum hl_fw_component fwc) +static void gaudi_init_dynamic_firmware_loader(struct hl_device *hdev) { - const char *name; - u32 ver_off; - char *dest; + struct dynamic_fw_load_mgr *dynamic_loader; + struct cpu_dyn_regs *dyn_regs; - switch (fwc) { - case FW_COMP_UBOOT: - ver_off = RREG32(mmUBOOT_VER_OFFSET); - dest = hdev->asic_prop.uboot_ver; - name = "U-Boot"; - break; - case FW_COMP_PREBOOT: - ver_off = RREG32(mmPREBOOT_VER_OFFSET); - dest = hdev->asic_prop.preboot_ver; - name = "Preboot"; - break; - default: - dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc); - return -EIO; - } + dynamic_loader = &hdev->fw_loader.dynamic_loader; - ver_off &= ~((u32)SRAM_BASE_ADDR); + /* + * here we update initial values for few specific dynamic regs (as + * before reading the first descriptor from FW those value has to be + * hard-coded) in later stages of the protocol those values will be + * updated automatically by reading the FW descriptor so data there + * will always be up-to-date + */ + dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; + dyn_regs->kmd_msg_to_cpu = + cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); + dyn_regs->cpu_cmd_status_to_host = + cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); - if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) { - memcpy_fromio(dest, hdev->pcie_bar[SRAM_BAR_ID] + ver_off, - VERSION_MAX_LEN); - } else { - dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n", - name, ver_off); - strcpy(dest, "unavailable"); - return -EIO; - } + dynamic_loader->wait_for_bl_timeout = GAUDI_WAIT_FOR_BL_TIMEOUT_USEC; +} - return 0; +static void gaudi_init_static_firmware_loader(struct hl_device *hdev) +{ + struct static_fw_load_mgr *static_loader; + + static_loader = &hdev->fw_loader.static_loader; + + static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; + static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; + static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU; + static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST; + static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; + static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0; + static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1; + static_loader->boot_err0_reg = mmCPU_BOOT_ERR0; + static_loader->boot_err1_reg = mmCPU_BOOT_ERR1; + static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET; + static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET; + static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR)); + static_loader->cpu_reset_wait_msec = hdev->pldm ? + GAUDI_PLDM_RESET_WAIT_MSEC : + GAUDI_CPU_RESET_WAIT_MSEC; +} + +static void gaudi_init_firmware_loader(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct fw_load_mgr *fw_loader = &hdev->fw_loader; + + /* fill common fields */ + fw_loader->linux_loaded = false; + fw_loader->boot_fit_img.image_name = GAUDI_BOOT_FIT_FILE; + fw_loader->linux_img.image_name = GAUDI_LINUX_FW_FILE; + fw_loader->cpu_timeout = GAUDI_CPU_TIMEOUT_USEC; + fw_loader->boot_fit_timeout = GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC; + fw_loader->skip_bmc = !hdev->bmc_enable; + fw_loader->sram_bar_id = SRAM_BAR_ID; + fw_loader->dram_bar_id = HBM_BAR_ID; + + if (prop->dynamic_fw_load) + gaudi_init_dynamic_firmware_loader(hdev); + else + gaudi_init_static_firmware_loader(hdev); } static int gaudi_init_cpu(struct hl_device *hdev) @@ -3744,15 +3861,10 @@ static int gaudi_init_cpu(struct hl_device *hdev) * The device CPU works with 40 bits addresses. * This register sets the extension to 50 bits. */ - if (hdev->asic_prop.fw_security_disabled) + if (!hdev->asic_prop.fw_security_enabled) WREG32(mmCPU_IF_CPU_MSB_ADDR, hdev->cpu_pci_msb_addr); - rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS, - mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU, - mmCPU_CMD_STATUS_TO_HOST, - mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0, - !hdev->bmc_enable, GAUDI_CPU_TIMEOUT_USEC, - GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC); + rc = hl_fw_init_cpu(hdev); if (rc) return rc; @@ -3764,10 +3876,12 @@ static int gaudi_init_cpu(struct hl_device *hdev) static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) { - struct gaudi_device *gaudi = hdev->asic_specific; + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; struct asic_fixed_properties *prop = &hdev->asic_prop; + struct gaudi_device *gaudi = hdev->asic_specific; + u32 status, irq_handler_offset; struct hl_eq *eq; - u32 status; struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ]; int err; @@ -3806,7 +3920,12 @@ static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI); - WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_PI_UPDATE); + irq_handler_offset = prop->gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_host_pi_upd_irq); + + WREG32(irq_handler_offset, + gaudi_irq_map_table[GAUDI_EVENT_PI_UPDATE].cpu_id); err = hl_poll_timeout( hdev, @@ -3823,8 +3942,10 @@ static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) } /* update FW application security bits */ - if (prop->fw_security_status_valid) - prop->fw_app_security_map = RREG32(mmCPU_BOOT_DEV_STS0); + if (prop->fw_cpu_boot_dev_sts0_valid) + prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); + if (prop->fw_cpu_boot_dev_sts1_valid) + prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); gaudi->hw_cap_initialized |= HW_CAP_CPU_Q; return 0; @@ -3835,7 +3956,7 @@ static void gaudi_pre_hw_init(struct hl_device *hdev) /* Perform read from the device to make sure device is up */ RREG32(mmHW_STATE); - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { /* Set the access through PCI bars (Linux driver only) as * secured */ @@ -3860,13 +3981,27 @@ static void gaudi_pre_hw_init(struct hl_device *hdev) static int gaudi_hw_init(struct hl_device *hdev) { + struct gaudi_device *gaudi = hdev->asic_specific; int rc; gaudi_pre_hw_init(hdev); - gaudi_init_pci_dma_qmans(hdev); + /* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE. + * So we set it here and if anyone tries to move it later to + * a different address, there will be an error + */ + if (hdev->asic_prop.iatu_done_by_fw) + gaudi->hbm_bar_cur_addr = DRAM_PHYS_BASE; - gaudi_init_hbm_dma_qmans(hdev); + /* + * Before pushing u-boot/linux to device, need to set the hbm bar to + * base address of dram + */ + if (gaudi_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) { + dev_err(hdev->dev, + "failed to map HBM bar to DRAM base address\n"); + return -EIO; + } rc = gaudi_init_cpu(hdev); if (rc) { @@ -3895,6 +4030,10 @@ static int gaudi_hw_init(struct hl_device *hdev) gaudi_init_security(hdev); + gaudi_init_pci_dma_qmans(hdev); + + gaudi_init_hbm_dma_qmans(hdev); + gaudi_init_mme_qmans(hdev); gaudi_init_tpc_qmans(hdev); @@ -3934,8 +4073,11 @@ disable_queues: static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; + u32 status, reset_timeout_ms, cpu_timeout_ms, irq_handler_offset; struct gaudi_device *gaudi = hdev->asic_specific; - u32 status, reset_timeout_ms, cpu_timeout_ms; + bool driver_performs_reset; if (!hard_reset) { dev_err(hdev->dev, "GAUDI doesn't support soft-reset\n"); @@ -3950,26 +4092,35 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset) cpu_timeout_ms = GAUDI_CPU_RESET_WAIT_MSEC; } + driver_performs_reset = !!(!hdev->asic_prop.fw_security_enabled && + !hdev->asic_prop.hard_reset_done_by_fw); + /* Set device to handle FLR by H/W as we will put the device CPU to * halt mode */ - if (hdev->asic_prop.fw_security_disabled && - !hdev->asic_prop.hard_reset_done_by_fw) + if (driver_performs_reset) WREG32(mmPCIE_AUX_FLR_CTRL, (PCIE_AUX_FLR_CTRL_HW_CTRL_MASK | PCIE_AUX_FLR_CTRL_INT_MASK_MASK)); - /* I don't know what is the state of the CPU so make sure it is - * stopped in any means necessary + /* If linux is loaded in the device CPU we need to communicate with it + * via the GIC. Otherwise, we need to use COMMS or the MSG_TO_CPU + * registers in case of old F/Ws */ - if (hdev->asic_prop.hard_reset_done_by_fw) - WREG32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU, KMD_MSG_RST_DEV); - else - WREG32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU, KMD_MSG_GOTO_WFE); + if (hdev->fw_loader.linux_loaded) { + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_host_halt_irq); - WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_HALT_MACHINE); + WREG32(irq_handler_offset, + gaudi_irq_map_table[GAUDI_EVENT_HALT_MACHINE].cpu_id); + } else { + if (hdev->asic_prop.hard_reset_done_by_fw) + hl_fw_ask_hard_reset_without_linux(hdev); + else + hl_fw_ask_halt_machine_without_linux(hdev); + } - if (hdev->asic_prop.fw_security_disabled && - !hdev->asic_prop.hard_reset_done_by_fw) { + if (driver_performs_reset) { /* Configure the reset registers. Must be done as early as * possible in case we fail during H/W initialization @@ -4003,8 +4154,7 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset) WREG32(mmPREBOOT_PCIE_EN, LKD_HARD_RESET_MAGIC); /* Restart BTL/BLR upon hard-reset */ - if (hdev->asic_prop.fw_security_disabled) - WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 1); + WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 1); WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST, 1 << PSOC_GLOBAL_CONF_SW_ALL_RST_IND_SHIFT); @@ -4041,6 +4191,8 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset) HW_CAP_CLK_GATE); memset(gaudi->events_stat, 0, sizeof(gaudi->events_stat)); + + hdev->device_cpu_is_halted = false; } } @@ -4078,10 +4230,12 @@ static int gaudi_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma, static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; + u32 db_reg_offset, db_value, dma_qm_offset, q_off, irq_handler_offset; struct gaudi_device *gaudi = hdev->asic_specific; - u32 db_reg_offset, db_value, dma_qm_offset, q_off; - int dma_id; bool invalid_queue = false; + int dma_id; switch (hw_queue_id) { case GAUDI_QUEUE_ID_DMA_0_0...GAUDI_QUEUE_ID_DMA_0_3: @@ -4307,164 +4461,84 @@ static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) db_reg_offset = mmTPC7_QM_PQ_PI_3; break; - case GAUDI_QUEUE_ID_NIC_0_0: - db_reg_offset = mmNIC0_QM0_PQ_PI_0; - break; - - case GAUDI_QUEUE_ID_NIC_0_1: - db_reg_offset = mmNIC0_QM0_PQ_PI_1; - break; - - case GAUDI_QUEUE_ID_NIC_0_2: - db_reg_offset = mmNIC0_QM0_PQ_PI_2; - break; - - case GAUDI_QUEUE_ID_NIC_0_3: - db_reg_offset = mmNIC0_QM0_PQ_PI_3; - break; - - case GAUDI_QUEUE_ID_NIC_1_0: - db_reg_offset = mmNIC0_QM1_PQ_PI_0; - break; - - case GAUDI_QUEUE_ID_NIC_1_1: - db_reg_offset = mmNIC0_QM1_PQ_PI_1; - break; - - case GAUDI_QUEUE_ID_NIC_1_2: - db_reg_offset = mmNIC0_QM1_PQ_PI_2; - break; - - case GAUDI_QUEUE_ID_NIC_1_3: - db_reg_offset = mmNIC0_QM1_PQ_PI_3; - break; - - case GAUDI_QUEUE_ID_NIC_2_0: - db_reg_offset = mmNIC1_QM0_PQ_PI_0; - break; - - case GAUDI_QUEUE_ID_NIC_2_1: - db_reg_offset = mmNIC1_QM0_PQ_PI_1; - break; - - case GAUDI_QUEUE_ID_NIC_2_2: - db_reg_offset = mmNIC1_QM0_PQ_PI_2; - break; - - case GAUDI_QUEUE_ID_NIC_2_3: - db_reg_offset = mmNIC1_QM0_PQ_PI_3; - break; - - case GAUDI_QUEUE_ID_NIC_3_0: - db_reg_offset = mmNIC1_QM1_PQ_PI_0; - break; - - case GAUDI_QUEUE_ID_NIC_3_1: - db_reg_offset = mmNIC1_QM1_PQ_PI_1; - break; - - case GAUDI_QUEUE_ID_NIC_3_2: - db_reg_offset = mmNIC1_QM1_PQ_PI_2; - break; - - case GAUDI_QUEUE_ID_NIC_3_3: - db_reg_offset = mmNIC1_QM1_PQ_PI_3; - break; - - case GAUDI_QUEUE_ID_NIC_4_0: - db_reg_offset = mmNIC2_QM0_PQ_PI_0; - break; - - case GAUDI_QUEUE_ID_NIC_4_1: - db_reg_offset = mmNIC2_QM0_PQ_PI_1; - break; - - case GAUDI_QUEUE_ID_NIC_4_2: - db_reg_offset = mmNIC2_QM0_PQ_PI_2; - break; - - case GAUDI_QUEUE_ID_NIC_4_3: - db_reg_offset = mmNIC2_QM0_PQ_PI_3; - break; - - case GAUDI_QUEUE_ID_NIC_5_0: - db_reg_offset = mmNIC2_QM1_PQ_PI_0; - break; + case GAUDI_QUEUE_ID_NIC_0_0...GAUDI_QUEUE_ID_NIC_0_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC0)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_5_1: - db_reg_offset = mmNIC2_QM1_PQ_PI_1; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC0_QM0_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_5_2: - db_reg_offset = mmNIC2_QM1_PQ_PI_2; - break; + case GAUDI_QUEUE_ID_NIC_1_0...GAUDI_QUEUE_ID_NIC_1_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC1)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_5_3: - db_reg_offset = mmNIC2_QM1_PQ_PI_3; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC0_QM1_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_6_0: - db_reg_offset = mmNIC3_QM0_PQ_PI_0; - break; + case GAUDI_QUEUE_ID_NIC_2_0...GAUDI_QUEUE_ID_NIC_2_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC2)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_6_1: - db_reg_offset = mmNIC3_QM0_PQ_PI_1; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC1_QM0_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_6_2: - db_reg_offset = mmNIC3_QM0_PQ_PI_2; - break; + case GAUDI_QUEUE_ID_NIC_3_0...GAUDI_QUEUE_ID_NIC_3_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC3)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_6_3: - db_reg_offset = mmNIC3_QM0_PQ_PI_3; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC1_QM1_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_7_0: - db_reg_offset = mmNIC3_QM1_PQ_PI_0; - break; + case GAUDI_QUEUE_ID_NIC_4_0...GAUDI_QUEUE_ID_NIC_4_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC4)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_7_1: - db_reg_offset = mmNIC3_QM1_PQ_PI_1; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC2_QM0_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_7_2: - db_reg_offset = mmNIC3_QM1_PQ_PI_2; - break; + case GAUDI_QUEUE_ID_NIC_5_0...GAUDI_QUEUE_ID_NIC_5_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC5)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_7_3: - db_reg_offset = mmNIC3_QM1_PQ_PI_3; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC2_QM1_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_8_0: - db_reg_offset = mmNIC4_QM0_PQ_PI_0; - break; + case GAUDI_QUEUE_ID_NIC_6_0...GAUDI_QUEUE_ID_NIC_6_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC6)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_8_1: - db_reg_offset = mmNIC4_QM0_PQ_PI_1; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC3_QM0_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_8_2: - db_reg_offset = mmNIC4_QM0_PQ_PI_2; - break; + case GAUDI_QUEUE_ID_NIC_7_0...GAUDI_QUEUE_ID_NIC_7_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC7)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_8_3: - db_reg_offset = mmNIC4_QM0_PQ_PI_3; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC3_QM1_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_9_0: - db_reg_offset = mmNIC4_QM1_PQ_PI_0; - break; + case GAUDI_QUEUE_ID_NIC_8_0...GAUDI_QUEUE_ID_NIC_8_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC8)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_9_1: - db_reg_offset = mmNIC4_QM1_PQ_PI_1; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC4_QM0_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_9_2: - db_reg_offset = mmNIC4_QM1_PQ_PI_2; - break; + case GAUDI_QUEUE_ID_NIC_9_0...GAUDI_QUEUE_ID_NIC_9_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC9)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_9_3: - db_reg_offset = mmNIC4_QM1_PQ_PI_3; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC4_QM1_PQ_PI_0 + q_off; break; default: @@ -4486,8 +4560,13 @@ static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) if (hw_queue_id == GAUDI_QUEUE_ID_CPU_PQ) { /* make sure device CPU will read latest data from host */ mb(); - WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, - GAUDI_EVENT_PI_UPDATE); + + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_host_pi_upd_irq); + + WREG32(irq_handler_offset, + gaudi_irq_map_table[GAUDI_EVENT_PI_UPDATE].cpu_id); } } @@ -4934,6 +5013,7 @@ already_pinned: return 0; unpin_memory: + list_del(&userptr->job_node); hl_unpin_host_memory(hdev, userptr); free_userptr: kfree(userptr); @@ -6513,7 +6593,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) gaudi_mmu_prepare_reg(hdev, mmMME2_ACC_WBC, asid); gaudi_mmu_prepare_reg(hdev, mmMME3_ACC_WBC, asid); - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC0) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC0) { gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_1, @@ -6526,7 +6606,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC1) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC1) { gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_1, @@ -6539,7 +6619,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC2) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC2) { gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_1, @@ -6552,7 +6632,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC3) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC3) { gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_1, @@ -6565,7 +6645,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC4) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC4) { gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_1, @@ -6578,7 +6658,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC5) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC5) { gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_1, @@ -6591,7 +6671,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC6) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC6) { gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_1, @@ -6604,7 +6684,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC7) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC7) { gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_1, @@ -6617,7 +6697,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC8) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC8) { gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_1, @@ -6630,7 +6710,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC9) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC9) { gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_1, @@ -7044,14 +7124,158 @@ enable_clk_gate: return rc; } +/* + * gaudi_queue_idx_dec - decrement queue index (pi/ci) and handle wrap + * + * @idx: the current pi/ci value + * @q_len: the queue length (power of 2) + * + * @return the cyclically decremented index + */ +static inline u32 gaudi_queue_idx_dec(u32 idx, u32 q_len) +{ + u32 mask = q_len - 1; + + /* + * modular decrement is equivalent to adding (queue_size -1) + * later we take LSBs to make sure the value is in the + * range [0, queue_len - 1] + */ + return (idx + q_len - 1) & mask; +} + +/** + * gaudi_print_sw_config_stream_data - print SW config stream data + * + * @hdev: pointer to the habanalabs device structure + * @stream: the QMAN's stream + * @qman_base: base address of QMAN registers block + */ +static void gaudi_print_sw_config_stream_data(struct hl_device *hdev, u32 stream, + u64 qman_base) +{ + u64 cq_ptr_lo, cq_ptr_hi, cq_tsize, cq_ptr; + u32 cq_ptr_lo_off, size; + + cq_ptr_lo_off = mmTPC0_QM_CQ_PTR_LO_1 - mmTPC0_QM_CQ_PTR_LO_0; + + cq_ptr_lo = qman_base + (mmTPC0_QM_CQ_PTR_LO_0 - mmTPC0_QM_BASE) + + stream * cq_ptr_lo_off; + cq_ptr_hi = cq_ptr_lo + + (mmTPC0_QM_CQ_PTR_HI_0 - mmTPC0_QM_CQ_PTR_LO_0); + cq_tsize = cq_ptr_lo + + (mmTPC0_QM_CQ_TSIZE_0 - mmTPC0_QM_CQ_PTR_LO_0); + + cq_ptr = (((u64) RREG32(cq_ptr_hi)) << 32) | RREG32(cq_ptr_lo); + size = RREG32(cq_tsize); + dev_info(hdev->dev, "stop on err: stream: %u, addr: %#llx, size: %x\n", + stream, cq_ptr, size); +} + +/** + * gaudi_print_last_pqes_on_err - print last PQEs on error + * + * @hdev: pointer to the habanalabs device structure + * @qid_base: first QID of the QMAN (out of 4 streams) + * @stream: the QMAN's stream + * @qman_base: base address of QMAN registers block + * @pr_sw_conf: if true print the SW config stream data (CQ PTR and SIZE) + */ +static void gaudi_print_last_pqes_on_err(struct hl_device *hdev, u32 qid_base, + u32 stream, u64 qman_base, + bool pr_sw_conf) +{ + u32 ci, qm_ci_stream_off, queue_len; + struct hl_hw_queue *q; + u64 pq_ci; + int i; + + q = &hdev->kernel_queues[qid_base + stream]; + + qm_ci_stream_off = mmTPC0_QM_PQ_CI_1 - mmTPC0_QM_PQ_CI_0; + pq_ci = qman_base + (mmTPC0_QM_PQ_CI_0 - mmTPC0_QM_BASE) + + stream * qm_ci_stream_off; + + queue_len = (q->queue_type == QUEUE_TYPE_INT) ? + q->int_queue_len : HL_QUEUE_LENGTH; + + hdev->asic_funcs->hw_queues_lock(hdev); + + if (pr_sw_conf) + gaudi_print_sw_config_stream_data(hdev, stream, qman_base); + + ci = RREG32(pq_ci); + + /* we should start printing form ci -1 */ + ci = gaudi_queue_idx_dec(ci, queue_len); + + for (i = 0; i < PQ_FETCHER_CACHE_SIZE; i++) { + struct hl_bd *bd; + u64 addr; + u32 len; + + bd = q->kernel_address; + bd += ci; + + len = le32_to_cpu(bd->len); + /* len 0 means uninitialized entry- break */ + if (!len) + break; + + addr = le64_to_cpu(bd->ptr); + + dev_info(hdev->dev, "stop on err PQE(stream %u): ci: %u, addr: %#llx, size: %x\n", + stream, ci, addr, len); + + /* get previous ci, wrap if needed */ + ci = gaudi_queue_idx_dec(ci, queue_len); + } + + hdev->asic_funcs->hw_queues_unlock(hdev); +} + +/** + * print_qman_data_on_err - extract QMAN data on error + * + * @hdev: pointer to the habanalabs device structure + * @qid_base: first QID of the QMAN (out of 4 streams) + * @stream: the QMAN's stream + * @qman_base: base address of QMAN registers block + * + * This function attempt to exatract as much data as possible on QMAN error. + * On upper CP print the SW config stream data and last 8 PQEs. + * On lower CP print SW config data and last PQEs of ALL 4 upper CPs + */ +static void print_qman_data_on_err(struct hl_device *hdev, u32 qid_base, + u32 stream, u64 qman_base) +{ + u32 i; + + if (stream != QMAN_STREAMS) { + gaudi_print_last_pqes_on_err(hdev, qid_base, stream, qman_base, + true); + return; + } + + gaudi_print_sw_config_stream_data(hdev, stream, qman_base); + + for (i = 0; i < QMAN_STREAMS; i++) + gaudi_print_last_pqes_on_err(hdev, qid_base, i, qman_base, + false); +} + static void gaudi_handle_qman_err_generic(struct hl_device *hdev, const char *qm_name, - u64 glbl_sts_addr, - u64 arb_err_addr) + u64 qman_base, + u32 qid_base) { u32 i, j, glbl_sts_val, arb_err_val, glbl_sts_clr_val; + u64 glbl_sts_addr, arb_err_addr; char reg_desc[32]; + glbl_sts_addr = qman_base + (mmTPC0_QM_GLBL_STS1_0 - mmTPC0_QM_BASE); + arb_err_addr = qman_base + (mmTPC0_QM_ARB_ERR_CAUSE - mmTPC0_QM_BASE); + /* Iterate through all stream GLBL_STS1 registers + Lower CP */ for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) { glbl_sts_clr_val = 0; @@ -7078,6 +7302,8 @@ static void gaudi_handle_qman_err_generic(struct hl_device *hdev, /* Write 1 clear errors */ if (!hdev->stop_on_err) WREG32(glbl_sts_addr + 4 * i, glbl_sts_clr_val); + else + print_qman_data_on_err(hdev, qid_base, i, qman_base); } arb_err_val = RREG32(arb_err_addr); @@ -7222,90 +7448,88 @@ static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type, static void gaudi_handle_qman_err(struct hl_device *hdev, u16 event_type) { - u64 glbl_sts_addr, arb_err_addr; - u8 index; + u64 qman_base; char desc[32]; + u32 qid_base; + u8 index; switch (event_type) { case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM: index = event_type - GAUDI_EVENT_TPC0_QM; - glbl_sts_addr = - mmTPC0_QM_GLBL_STS1_0 + index * TPC_QMAN_OFFSET; - arb_err_addr = - mmTPC0_QM_ARB_ERR_CAUSE + index * TPC_QMAN_OFFSET; + qid_base = GAUDI_QUEUE_ID_TPC_0_0 + index * QMAN_STREAMS; + qman_base = mmTPC0_QM_BASE + index * TPC_QMAN_OFFSET; snprintf(desc, ARRAY_SIZE(desc), "%s%d", "TPC_QM", index); break; case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM: index = event_type - GAUDI_EVENT_MME0_QM; - glbl_sts_addr = - mmMME0_QM_GLBL_STS1_0 + index * MME_QMAN_OFFSET; - arb_err_addr = - mmMME0_QM_ARB_ERR_CAUSE + index * MME_QMAN_OFFSET; + qid_base = GAUDI_QUEUE_ID_MME_0_0 + index * QMAN_STREAMS; + qman_base = mmMME0_QM_BASE + index * MME_QMAN_OFFSET; snprintf(desc, ARRAY_SIZE(desc), "%s%d", "MME_QM", index); break; case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM: index = event_type - GAUDI_EVENT_DMA0_QM; - glbl_sts_addr = - mmDMA0_QM_GLBL_STS1_0 + index * DMA_QMAN_OFFSET; - arb_err_addr = - mmDMA0_QM_ARB_ERR_CAUSE + index * DMA_QMAN_OFFSET; + qid_base = GAUDI_QUEUE_ID_DMA_0_0 + index * QMAN_STREAMS; + /* skip GAUDI_QUEUE_ID_CPU_PQ if necessary */ + if (index > 1) + qid_base++; + qman_base = mmDMA0_QM_BASE + index * DMA_QMAN_OFFSET; snprintf(desc, ARRAY_SIZE(desc), "%s%d", "DMA_QM", index); break; case GAUDI_EVENT_NIC0_QM0: - glbl_sts_addr = mmNIC0_QM0_GLBL_STS1_0; - arb_err_addr = mmNIC0_QM0_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_0_0; + qman_base = mmNIC0_QM0_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC0_QM0"); break; case GAUDI_EVENT_NIC0_QM1: - glbl_sts_addr = mmNIC0_QM1_GLBL_STS1_0; - arb_err_addr = mmNIC0_QM1_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_1_0; + qman_base = mmNIC0_QM1_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC0_QM1"); break; case GAUDI_EVENT_NIC1_QM0: - glbl_sts_addr = mmNIC1_QM0_GLBL_STS1_0; - arb_err_addr = mmNIC1_QM0_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_2_0; + qman_base = mmNIC1_QM0_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC1_QM0"); break; case GAUDI_EVENT_NIC1_QM1: - glbl_sts_addr = mmNIC1_QM1_GLBL_STS1_0; - arb_err_addr = mmNIC1_QM1_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_3_0; + qman_base = mmNIC1_QM1_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC1_QM1"); break; case GAUDI_EVENT_NIC2_QM0: - glbl_sts_addr = mmNIC2_QM0_GLBL_STS1_0; - arb_err_addr = mmNIC2_QM0_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_4_0; + qman_base = mmNIC2_QM0_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC2_QM0"); break; case GAUDI_EVENT_NIC2_QM1: - glbl_sts_addr = mmNIC2_QM1_GLBL_STS1_0; - arb_err_addr = mmNIC2_QM1_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_5_0; + qman_base = mmNIC2_QM1_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC2_QM1"); break; case GAUDI_EVENT_NIC3_QM0: - glbl_sts_addr = mmNIC3_QM0_GLBL_STS1_0; - arb_err_addr = mmNIC3_QM0_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_6_0; + qman_base = mmNIC3_QM0_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC3_QM0"); break; case GAUDI_EVENT_NIC3_QM1: - glbl_sts_addr = mmNIC3_QM1_GLBL_STS1_0; - arb_err_addr = mmNIC3_QM1_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_7_0; + qman_base = mmNIC3_QM1_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC3_QM1"); break; case GAUDI_EVENT_NIC4_QM0: - glbl_sts_addr = mmNIC4_QM0_GLBL_STS1_0; - arb_err_addr = mmNIC4_QM0_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_8_0; + qman_base = mmNIC4_QM0_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC4_QM0"); break; case GAUDI_EVENT_NIC4_QM1: - glbl_sts_addr = mmNIC4_QM1_GLBL_STS1_0; - arb_err_addr = mmNIC4_QM1_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_9_0; + qman_base = mmNIC4_QM1_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC4_QM1"); break; default: return; } - gaudi_handle_qman_err_generic(hdev, desc, glbl_sts_addr, arb_err_addr); + gaudi_handle_qman_err_generic(hdev, desc, qman_base, qid_base); } static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type, @@ -7332,6 +7556,16 @@ static void gaudi_print_out_of_sync_info(struct hl_device *hdev, sync_err->pi, sync_err->ci, q->pi, atomic_read(&q->ci)); } +static void gaudi_print_fw_alive_info(struct hl_device *hdev, + struct hl_eq_fw_alive *fw_alive) +{ + dev_err(hdev->dev, + "FW alive report: severity=%s, process_id=%u, thread_id=%u, uptime=%llu seconds\n", + (fw_alive->severity == FW_ALIVE_SEVERITY_MINOR) ? + "Minor" : "Critical", fw_alive->process_id, + fw_alive->thread_id, fw_alive->uptime_seconds); +} + static int gaudi_soft_reset_late_init(struct hl_device *hdev) { struct gaudi_device *gaudi = hdev->asic_specific; @@ -7346,11 +7580,10 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, struct hl_eq_hbm_ecc_data *hbm_ecc_data) { u32 base, val, val2, wr_par, rd_par, ca_par, derr, serr, type, ch; - int err = 0; + int rc = 0; - if (hdev->asic_prop.fw_security_status_valid && - (hdev->asic_prop.fw_app_security_map & - CPU_BOOT_DEV_STS0_HBM_ECC_EN)) { + if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_HBM_ECC_EN) { if (!hbm_ecc_data) { dev_err(hdev->dev, "No FW ECC data"); return 0; @@ -7379,13 +7612,10 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, device, ch, hbm_ecc_data->first_addr, type, hbm_ecc_data->sec_cont_cnt, hbm_ecc_data->sec_cnt, hbm_ecc_data->dec_cnt); - - err = 1; - return 0; } - if (!hdev->asic_prop.fw_security_disabled) { + if (hdev->asic_prop.fw_security_enabled) { dev_info(hdev->dev, "Cannot access MC regs for ECC data while security is enabled\n"); return 0; } @@ -7395,7 +7625,7 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, val = RREG32_MASK(base + ch * 0x1000 + 0x06C, 0x0000FFFF); val = (val & 0xFF) | ((val >> 8) & 0xFF); if (val) { - err = 1; + rc = -EIO; dev_err(hdev->dev, "HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n", device, ch * 2, val & 0x1, (val >> 1) & 0x1, @@ -7415,7 +7645,7 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, val = RREG32_MASK(base + ch * 0x1000 + 0x07C, 0x0000FFFF); val = (val & 0xFF) | ((val >> 8) & 0xFF); if (val) { - err = 1; + rc = -EIO; dev_err(hdev->dev, "HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n", device, ch * 2 + 1, val & 0x1, (val >> 1) & 0x1, @@ -7444,7 +7674,7 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, val = RREG32(base + 0x8F30); val2 = RREG32(base + 0x8F34); if (val | val2) { - err = 1; + rc = -EIO; dev_err(hdev->dev, "HBM %d MC SRAM SERR info: Reg 0x8F30=0x%x, Reg 0x8F34=0x%x\n", device, val, val2); @@ -7452,13 +7682,13 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, val = RREG32(base + 0x8F40); val2 = RREG32(base + 0x8F44); if (val | val2) { - err = 1; + rc = -EIO; dev_err(hdev->dev, "HBM %d MC SRAM DERR info: Reg 0x8F40=0x%x, Reg 0x8F44=0x%x\n", device, val, val2); } - return err; + return rc; } static int gaudi_hbm_event_to_dev(u16 hbm_event_type) @@ -7604,6 +7834,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, case GAUDI_EVENT_DMA_IF0_DERR ... GAUDI_EVENT_DMA_IF3_DERR: case GAUDI_EVENT_HBM_0_DERR ... GAUDI_EVENT_HBM_3_DERR: case GAUDI_EVENT_MMU_DERR: + case GAUDI_EVENT_NIC0_CS_DBG_DERR ... GAUDI_EVENT_NIC4_CS_DBG_DERR: gaudi_print_irq_info(hdev, event_type, true); gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); goto reset_device; @@ -7786,6 +8017,11 @@ static void gaudi_handle_eqe(struct hl_device *hdev, gaudi_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err); goto reset_device; + case GAUDI_EVENT_FW_ALIVE_S: + gaudi_print_irq_info(hdev, event_type, false); + gaudi_print_fw_alive_info(hdev, &eq_entry->fw_alive); + goto reset_device; + default: dev_err(hdev->dev, "Received invalid H/W interrupt %d\n", event_type); @@ -7856,52 +8092,13 @@ static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, } static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev, - bool is_hard, u32 asid, u64 va, u64 size) + bool is_hard, u32 flags, + u32 asid, u64 va, u64 size) { - struct gaudi_device *gaudi = hdev->asic_specific; - u32 status, timeout_usec; - u32 inv_data; - u32 pi; - int rc; - - if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) || - hdev->hard_reset_pending) - return 0; - - if (hdev->pldm) - timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC; - else - timeout_usec = MMU_CONFIG_TIMEOUT_USEC; - - /* - * TODO: currently invalidate entire L0 & L1 as in regular hard - * invalidation. Need to apply invalidation of specific cache - * lines with mask of ASID & VA & size. - * Note that L1 with be flushed entirely in any case. + /* Treat as invalidate all because there is no range invalidation + * in Gaudi */ - - /* L0 & L1 invalidation */ - inv_data = RREG32(mmSTLB_CACHE_INV); - /* PI is 8 bit */ - pi = ((inv_data & STLB_CACHE_INV_PRODUCER_INDEX_MASK) + 1) & 0xFF; - WREG32(mmSTLB_CACHE_INV, - (inv_data & STLB_CACHE_INV_INDEX_MASK_MASK) | pi); - - rc = hl_poll_timeout( - hdev, - mmSTLB_INV_CONSUMER_INDEX, - status, - status == pi, - 1000, - timeout_usec); - - if (rc) { - dev_err_ratelimited(hdev->dev, - "MMU cache invalidation timeout\n"); - hl_device_reset(hdev, HL_RESET_HARD); - } - - return rc; + return hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags); } static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, @@ -7956,7 +8153,9 @@ static int gaudi_cpucp_info_get(struct hl_device *hdev) if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) return 0; - rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0); + rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, + mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, + mmCPU_BOOT_ERR1); if (rc) return rc; @@ -8077,7 +8276,7 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, for (i = 0 ; i < (NIC_NUMBER_OF_ENGINES / 2) ; i++) { offset = i * NIC_MACRO_QMAN_OFFSET; port = 2 * i; - if (hdev->nic_ports_mask & BIT(port)) { + if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + port)) { qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset); qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset); is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts); @@ -8092,7 +8291,7 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, } port = 2 * i + 1; - if (hdev->nic_ports_mask & BIT(port)) { + if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + port)) { qm_glbl_sts0 = RREG32(mmNIC0_QM1_GLBL_STS0 + offset); qm_cgm_sts = RREG32(mmNIC0_QM1_CGM_STS + offset); is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts); @@ -8306,8 +8505,10 @@ static int gaudi_internal_cb_pool_init(struct hl_device *hdev, HL_VA_RANGE_TYPE_HOST, HOST_SPACE_INTERNAL_CB_SZ, HL_MMU_VA_ALIGNMENT_NOT_NEEDED); - if (!hdev->internal_cb_va_base) + if (!hdev->internal_cb_va_base) { + rc = -ENOMEM; goto destroy_internal_cb_pool; + } mutex_lock(&ctx->mmu_lock); rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, @@ -8749,7 +8950,14 @@ static int gaudi_block_mmap(struct hl_device *hdev, static void gaudi_enable_events_from_fw(struct hl_device *hdev) { - WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_INTS_REGISTER); + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; + u32 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_host_ints_irq); + + WREG32(irq_handler_offset, + gaudi_irq_map_table[GAUDI_EVENT_INTS_REGISTER].cpu_id); } static int gaudi_map_pll_idx_to_fw_idx(u32 pll_idx) @@ -8834,7 +9042,6 @@ static const struct hl_asic_funcs gaudi_funcs = { .ctx_fini = gaudi_ctx_fini, .get_clk_rate = gaudi_get_clk_rate, .get_queue_id_for_cq = gaudi_get_queue_id_for_cq, - .read_device_fw_version = gaudi_read_device_fw_version, .load_firmware_to_device = gaudi_load_firmware_to_device, .load_boot_fit_to_device = gaudi_load_boot_fit_to_device, .get_signal_cb_size = gaudi_get_signal_cb_size, @@ -8853,7 +9060,9 @@ static const struct hl_asic_funcs gaudi_funcs = { .get_hw_block_id = gaudi_get_hw_block_id, .hw_block_mmap = gaudi_block_mmap, .enable_events_from_fw = gaudi_enable_events_from_fw, - .map_pll_idx_to_fw_idx = gaudi_map_pll_idx_to_fw_idx + .map_pll_idx_to_fw_idx = gaudi_map_pll_idx_to_fw_idx, + .init_firmware_loader = gaudi_init_firmware_loader, + .init_cpu_scrambler_dram = gaudi_init_scrambler_hbm }; /** diff --git a/drivers/misc/habanalabs/gaudi/gaudiP.h b/drivers/misc/habanalabs/gaudi/gaudiP.h index 5929be81ec23..957bf3720f70 100644 --- a/drivers/misc/habanalabs/gaudi/gaudiP.h +++ b/drivers/misc/habanalabs/gaudi/gaudiP.h @@ -82,6 +82,7 @@ QMAN_STREAMS) #define QMAN_STREAMS 4 +#define PQ_FETCHER_CACHE_SIZE 8 #define DMA_QMAN_OFFSET (mmDMA1_QM_BASE - mmDMA0_QM_BASE) #define TPC_QMAN_OFFSET (mmTPC1_QM_BASE - mmTPC0_QM_BASE) diff --git a/drivers/misc/habanalabs/gaudi/gaudi_coresight.c b/drivers/misc/habanalabs/gaudi/gaudi_coresight.c index 6e56fa1c6c69..c2a27ed1c4d1 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi_coresight.c +++ b/drivers/misc/habanalabs/gaudi/gaudi_coresight.c @@ -424,7 +424,7 @@ static int gaudi_config_stm(struct hl_device *hdev, if (frequency == 0) frequency = input->frequency; WREG32(base_reg + 0xE8C, frequency); - WREG32(base_reg + 0xE90, 0x7FF); + WREG32(base_reg + 0xE90, 0x1F00); /* SW-2176 - SW WA for HW bug */ if ((CFG_BASE + base_reg) >= mmDMA_CH_0_CS_STM_BASE && @@ -434,7 +434,7 @@ static int gaudi_config_stm(struct hl_device *hdev, WREG32(base_reg + 0xE6C, 0x0); } - WREG32(base_reg + 0xE80, 0x27 | (input->id << 16)); + WREG32(base_reg + 0xE80, 0x23 | (input->id << 16)); } else { WREG32(base_reg + 0xE80, 4); WREG32(base_reg + 0xD64, 0); @@ -634,7 +634,7 @@ static int gaudi_config_etr(struct hl_device *hdev, WREG32(mmPSOC_ETR_BUFWM, 0x3FFC); WREG32(mmPSOC_ETR_RSZ, input->buffer_size); WREG32(mmPSOC_ETR_MODE, input->sink_mode); - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { /* make ETR not privileged */ val = FIELD_PREP( PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK, 0); diff --git a/drivers/misc/habanalabs/gaudi/gaudi_security.c b/drivers/misc/habanalabs/gaudi/gaudi_security.c index 9a706c5980ef..0d3240f1f7d7 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi_security.c +++ b/drivers/misc/habanalabs/gaudi/gaudi_security.c @@ -1448,7 +1448,7 @@ static void gaudi_init_dma_protection_bits(struct hl_device *hdev) u32 pb_addr, mask; u8 word_offset; - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { gaudi_pb_set_block(hdev, mmDMA_IF_E_S_BASE); gaudi_pb_set_block(hdev, mmDMA_IF_E_S_DOWN_CH0_BASE); gaudi_pb_set_block(hdev, mmDMA_IF_E_S_DOWN_CH1_BASE); @@ -9135,7 +9135,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) u32 pb_addr, mask; u8 word_offset; - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { gaudi_pb_set_block(hdev, mmTPC0_E2E_CRED_BASE); gaudi_pb_set_block(hdev, mmTPC1_E2E_CRED_BASE); gaudi_pb_set_block(hdev, mmTPC2_E2E_CRED_BASE); @@ -12818,7 +12818,7 @@ static void gaudi_init_protection_bits(struct hl_device *hdev) * secured */ - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { gaudi_pb_set_block(hdev, mmIF_E_PLL_BASE); gaudi_pb_set_block(hdev, mmMESH_W_PLL_BASE); gaudi_pb_set_block(hdev, mmSRAM_W_PLL_BASE); @@ -13023,7 +13023,7 @@ void gaudi_init_security(struct hl_device *hdev) * property configuration of MME SBAB and ACC to be non-privileged and * non-secured */ - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { WREG32(mmMME0_SBAB_PROT, 0x2); WREG32(mmMME0_ACC_PROT, 0x2); WREG32(mmMME1_SBAB_PROT, 0x2); @@ -13032,11 +13032,12 @@ void gaudi_init_security(struct hl_device *hdev) WREG32(mmMME2_ACC_PROT, 0x2); WREG32(mmMME3_SBAB_PROT, 0x2); WREG32(mmMME3_ACC_PROT, 0x2); - } - /* On RAZWI, 0 will be returned from RR and 0xBABA0BAD from PB */ - if (hdev->asic_prop.fw_security_disabled) + /* + * On RAZWI, 0 will be returned from RR and 0xBABA0BAD from PB + */ WREG32(0xC01B28, 0x1); + } gaudi_init_range_registers_lbw(hdev); diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index e0ad2a269779..755e08cf2ecc 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -87,6 +87,7 @@ #define GOYA_PLDM_QMAN0_TIMEOUT_USEC (HL_DEVICE_TIMEOUT_USEC * 30) #define GOYA_BOOT_FIT_REQ_TIMEOUT_USEC 1000000 /* 1s */ #define GOYA_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */ +#define GOYA_WAIT_FOR_BL_TIMEOUT_USEC 15000000 /* 15s */ #define GOYA_QMAN0_FENCE_VAL 0xD169B243 @@ -354,7 +355,7 @@ static int goya_mmu_set_dram_default_page(struct hl_device *hdev); static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev); static void goya_mmu_prepare(struct hl_device *hdev, u32 asid); -int goya_get_fixed_properties(struct hl_device *hdev) +int goya_set_fixed_properties(struct hl_device *hdev) { struct asic_fixed_properties *prop = &hdev->asic_prop; int i; @@ -460,8 +461,10 @@ int goya_get_fixed_properties(struct hl_device *hdev) for (i = 0 ; i < HL_MAX_DCORES ; i++) prop->first_available_cq[i] = USHRT_MAX; - prop->fw_security_status_valid = false; + prop->fw_cpu_boot_dev_sts0_valid = false; + prop->fw_cpu_boot_dev_sts1_valid = false; prop->hard_reset_done_by_fw = false; + prop->gic_interrupts_enable = true; return 0; } @@ -531,10 +534,8 @@ static int goya_init_iatu(struct hl_device *hdev) struct hl_outbound_pci_region outbound_region; int rc; - if (hdev->asic_prop.iatu_done_by_fw) { - hdev->asic_funcs->set_dma_mask_from_fw(hdev); + if (hdev->asic_prop.iatu_done_by_fw) return 0; - } /* Inbound Region 0 - Bar 0 - Point to SRAM and CFG */ inbound_region.mode = PCI_BAR_MATCH_MODE; @@ -586,7 +587,7 @@ static int goya_early_init(struct hl_device *hdev) u32 fw_boot_status, val; int rc; - rc = goya_get_fixed_properties(hdev); + rc = goya_set_fixed_properties(hdev); if (rc) { dev_err(hdev->dev, "Failed to get fixed properties\n"); return rc; @@ -618,7 +619,7 @@ static int goya_early_init(struct hl_device *hdev) prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID); /* If FW security is enabled at this point it means no access to ELBI */ - if (!hdev->asic_prop.fw_security_disabled) { + if (hdev->asic_prop.fw_security_enabled) { hdev->asic_prop.iatu_done_by_fw = true; goto pci_init; } @@ -642,8 +643,10 @@ pci_init: * version to determine whether we run with a security-enabled firmware */ rc = hl_fw_read_preboot_status(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS, - mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0, - GOYA_BOOT_FIT_REQ_TIMEOUT_USEC); + mmCPU_BOOT_DEV_STS0, + mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, + mmCPU_BOOT_ERR1, + GOYA_BOOT_FIT_REQ_TIMEOUT_USEC); if (rc) { if (hdev->reset_on_preboot_fail) hdev->asic_funcs->hw_fini(hdev, true); @@ -723,7 +726,15 @@ static void goya_fetch_psoc_frequency(struct hl_device *hdev) u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS], freq; int rc; - if (hdev->asic_prop.fw_security_disabled) { + if (hdev->asic_prop.fw_security_enabled) { + rc = hl_fw_cpucp_pll_info_get(hdev, HL_GOYA_PCI_PLL, + pll_freq_arr); + + if (rc) + return; + + freq = pll_freq_arr[1]; + } else { div_fctr = RREG32(mmPSOC_PCI_PLL_DIV_FACTOR_1); div_sel = RREG32(mmPSOC_PCI_PLL_DIV_SEL_1); nr = RREG32(mmPSOC_PCI_PLL_NR); @@ -750,14 +761,6 @@ static void goya_fetch_psoc_frequency(struct hl_device *hdev) div_sel); freq = 0; } - } else { - rc = hl_fw_cpucp_pll_info_get(hdev, HL_GOYA_PCI_PLL, - pll_freq_arr); - - if (rc) - return; - - freq = pll_freq_arr[1]; } prop->psoc_timestamp_frequency = freq; @@ -849,6 +852,39 @@ void goya_late_fini(struct hl_device *hdev) hdev->hl_chip_info->info = NULL; } +static void goya_set_pci_memory_regions(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct pci_mem_region *region; + + /* CFG */ + region = &hdev->pci_mem_region[PCI_REGION_CFG]; + region->region_base = CFG_BASE; + region->region_size = CFG_SIZE; + region->offset_in_bar = CFG_BASE - SRAM_BASE_ADDR; + region->bar_size = CFG_BAR_SIZE; + region->bar_id = SRAM_CFG_BAR_ID; + region->used = 1; + + /* SRAM */ + region = &hdev->pci_mem_region[PCI_REGION_SRAM]; + region->region_base = SRAM_BASE_ADDR; + region->region_size = SRAM_SIZE; + region->offset_in_bar = 0; + region->bar_size = CFG_BAR_SIZE; + region->bar_id = SRAM_CFG_BAR_ID; + region->used = 1; + + /* DRAM */ + region = &hdev->pci_mem_region[PCI_REGION_DRAM]; + region->region_base = DRAM_PHYS_BASE; + region->region_size = hdev->asic_prop.dram_size; + region->offset_in_bar = 0; + region->bar_size = prop->dram_pci_bar_size; + region->bar_id = DDR_BAR_ID; + region->used = 1; +} + /* * goya_sw_init - Goya software initialization code * @@ -918,6 +954,9 @@ static int goya_sw_init(struct hl_device *hdev) spin_lock_init(&goya->hw_queues_lock); hdev->supports_coresight = true; hdev->supports_soft_reset = true; + hdev->allow_external_soft_reset = true; + + goya_set_pci_memory_regions(hdev); return 0; @@ -1263,8 +1302,11 @@ int goya_init_cpu_queues(struct hl_device *hdev) } /* update FW application security bits */ - if (prop->fw_security_status_valid) - prop->fw_app_security_map = RREG32(mmCPU_BOOT_DEV_STS0); + if (prop->fw_cpu_boot_dev_sts0_valid) + prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); + + if (prop->fw_cpu_boot_dev_sts1_valid) + prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); goya->hw_cap_initialized |= HW_CAP_CPU_Q; return 0; @@ -2402,47 +2444,67 @@ static int goya_load_boot_fit_to_device(struct hl_device *hdev) return hl_fw_load_fw_to_device(hdev, GOYA_BOOT_FIT_FILE, dst, 0, 0); } -/* - * FW component passes an offset from SRAM_BASE_ADDR in SCRATCHPAD_xx. - * The version string should be located by that offset. - */ -static int goya_read_device_fw_version(struct hl_device *hdev, - enum hl_fw_component fwc) -{ - const char *name; - u32 ver_off; - char *dest; - - switch (fwc) { - case FW_COMP_UBOOT: - ver_off = RREG32(mmUBOOT_VER_OFFSET); - dest = hdev->asic_prop.uboot_ver; - name = "U-Boot"; - break; - case FW_COMP_PREBOOT: - ver_off = RREG32(mmPREBOOT_VER_OFFSET); - dest = hdev->asic_prop.preboot_ver; - name = "Preboot"; - break; - default: - dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc); - return -EIO; - } +static void goya_init_dynamic_firmware_loader(struct hl_device *hdev) +{ + struct dynamic_fw_load_mgr *dynamic_loader; + struct cpu_dyn_regs *dyn_regs; - ver_off &= ~((u32)SRAM_BASE_ADDR); + dynamic_loader = &hdev->fw_loader.dynamic_loader; - if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) { - memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off, - VERSION_MAX_LEN); - } else { - dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n", - name, ver_off); - strcpy(dest, "unavailable"); + /* + * here we update initial values for few specific dynamic regs (as + * before reading the first descriptor from FW those value has to be + * hard-coded) in later stages of the protocol those values will be + * updated automatically by reading the FW descriptor so data there + * will always be up-to-date + */ + dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; + dyn_regs->kmd_msg_to_cpu = + cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); + dyn_regs->cpu_cmd_status_to_host = + cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); - return -EIO; - } + dynamic_loader->wait_for_bl_timeout = GOYA_WAIT_FOR_BL_TIMEOUT_USEC; +} - return 0; +static void goya_init_static_firmware_loader(struct hl_device *hdev) +{ + struct static_fw_load_mgr *static_loader; + + static_loader = &hdev->fw_loader.static_loader; + + static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; + static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; + static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU; + static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST; + static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; + static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0; + static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1; + static_loader->boot_err0_reg = mmCPU_BOOT_ERR0; + static_loader->boot_err1_reg = mmCPU_BOOT_ERR1; + static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET; + static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET; + static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR)); +} + +static void goya_init_firmware_loader(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct fw_load_mgr *fw_loader = &hdev->fw_loader; + + /* fill common fields */ + fw_loader->boot_fit_img.image_name = GOYA_BOOT_FIT_FILE; + fw_loader->linux_img.image_name = GOYA_LINUX_FW_FILE; + fw_loader->cpu_timeout = GOYA_CPU_TIMEOUT_USEC; + fw_loader->boot_fit_timeout = GOYA_BOOT_FIT_REQ_TIMEOUT_USEC; + fw_loader->skip_bmc = false; + fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; + fw_loader->dram_bar_id = DDR_BAR_ID; + + if (prop->dynamic_fw_load) + goya_init_dynamic_firmware_loader(hdev); + else + goya_init_static_firmware_loader(hdev); } static int goya_init_cpu(struct hl_device *hdev) @@ -2466,12 +2528,7 @@ static int goya_init_cpu(struct hl_device *hdev) return -EIO; } - rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS, - mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, - mmCPU_CMD_STATUS_TO_HOST, - mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0, - false, GOYA_CPU_TIMEOUT_USEC, - GOYA_BOOT_FIT_REQ_TIMEOUT_USEC); + rc = hl_fw_init_cpu(hdev); if (rc) return rc; @@ -2881,7 +2938,7 @@ void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id, *dma_handle = hdev->asic_prop.sram_base_address; - base = (void *) hdev->pcie_bar[SRAM_CFG_BAR_ID]; + base = (__force void *) hdev->pcie_bar[SRAM_CFG_BAR_ID]; switch (queue_id) { case GOYA_QUEUE_ID_MME: @@ -3270,6 +3327,7 @@ already_pinned: return 0; unpin_memory: + list_del(&userptr->job_node); hl_unpin_host_memory(hdev, userptr); free_userptr: kfree(userptr); @@ -5169,54 +5227,13 @@ static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, } static int goya_mmu_invalidate_cache_range(struct hl_device *hdev, - bool is_hard, u32 asid, u64 va, u64 size) + bool is_hard, u32 flags, + u32 asid, u64 va, u64 size) { - struct goya_device *goya = hdev->asic_specific; - u32 status, timeout_usec, inv_data, pi; - int rc; - - if (!(goya->hw_cap_initialized & HW_CAP_MMU) || - hdev->hard_reset_pending) - return 0; - - /* no need in L1 only invalidation in Goya */ - if (!is_hard) - return 0; - - if (hdev->pldm) - timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC; - else - timeout_usec = MMU_CONFIG_TIMEOUT_USEC; - - /* - * TODO: currently invalidate entire L0 & L1 as in regular hard - * invalidation. Need to apply invalidation of specific cache lines with - * mask of ASID & VA & size. - * Note that L1 with be flushed entirely in any case. + /* Treat as invalidate all because there is no range invalidation + * in Goya */ - - /* L0 & L1 invalidation */ - inv_data = RREG32(mmSTLB_CACHE_INV); - /* PI is 8 bit */ - pi = ((inv_data & STLB_CACHE_INV_PRODUCER_INDEX_MASK) + 1) & 0xFF; - WREG32(mmSTLB_CACHE_INV, - (inv_data & STLB_CACHE_INV_INDEX_MASK_MASK) | pi); - - rc = hl_poll_timeout( - hdev, - mmSTLB_INV_CONSUMER_INDEX, - status, - status == pi, - 1000, - timeout_usec); - - if (rc) { - dev_err_ratelimited(hdev->dev, - "MMU cache invalidation timeout\n"); - hl_device_reset(hdev, HL_RESET_HARD); - } - - return rc; + return hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags); } int goya_send_heartbeat(struct hl_device *hdev) @@ -5239,7 +5256,9 @@ int goya_cpucp_info_get(struct hl_device *hdev) if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) return 0; - rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0); + rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, + mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, + mmCPU_BOOT_ERR1); if (rc) return rc; @@ -5385,6 +5404,11 @@ static int goya_get_eeprom_data(struct hl_device *hdev, void *data, return hl_fw_get_eeprom_data(hdev, data, max_size); } +static void goya_cpu_init_scrambler_dram(struct hl_device *hdev) +{ + +} + static int goya_ctx_init(struct hl_ctx *ctx) { if (ctx->asid != HL_KERNEL_ASID_ID) @@ -5565,7 +5589,6 @@ static const struct hl_asic_funcs goya_funcs = { .ctx_fini = goya_ctx_fini, .get_clk_rate = goya_get_clk_rate, .get_queue_id_for_cq = goya_get_queue_id_for_cq, - .read_device_fw_version = goya_read_device_fw_version, .load_firmware_to_device = goya_load_firmware_to_device, .load_boot_fit_to_device = goya_load_boot_fit_to_device, .get_signal_cb_size = goya_get_signal_cb_size, @@ -5584,7 +5607,9 @@ static const struct hl_asic_funcs goya_funcs = { .get_hw_block_id = goya_get_hw_block_id, .hw_block_mmap = goya_block_mmap, .enable_events_from_fw = goya_enable_events_from_fw, - .map_pll_idx_to_fw_idx = goya_map_pll_idx_to_fw_idx + .map_pll_idx_to_fw_idx = goya_map_pll_idx_to_fw_idx, + .init_firmware_loader = goya_init_firmware_loader, + .init_cpu_scrambler_dram = goya_cpu_init_scrambler_dram }; /* diff --git a/drivers/misc/habanalabs/goya/goyaP.h b/drivers/misc/habanalabs/goya/goyaP.h index ef8c6c8b5c8d..0b05da614729 100644 --- a/drivers/misc/habanalabs/goya/goyaP.h +++ b/drivers/misc/habanalabs/goya/goyaP.h @@ -168,7 +168,7 @@ struct goya_device { u8 device_cpu_mmu_mappings_done; }; -int goya_get_fixed_properties(struct hl_device *hdev); +int goya_set_fixed_properties(struct hl_device *hdev); int goya_mmu_init(struct hl_device *hdev); void goya_init_dma_qmans(struct hl_device *hdev); void goya_init_mme_qmans(struct hl_device *hdev); diff --git a/drivers/misc/habanalabs/goya/goya_coresight.c b/drivers/misc/habanalabs/goya/goya_coresight.c index 6b7445cca580..c55c100fdd24 100644 --- a/drivers/misc/habanalabs/goya/goya_coresight.c +++ b/drivers/misc/habanalabs/goya/goya_coresight.c @@ -434,7 +434,7 @@ static int goya_config_etr(struct hl_device *hdev, WREG32(mmPSOC_ETR_BUFWM, 0x3FFC); WREG32(mmPSOC_ETR_RSZ, input->buffer_size); WREG32(mmPSOC_ETR_MODE, input->sink_mode); - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { /* make ETR not privileged */ val = FIELD_PREP(PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK, 0); /* make ETR non-secured (inverted logic) */ diff --git a/drivers/misc/habanalabs/include/common/cpucp_if.h b/drivers/misc/habanalabs/include/common/cpucp_if.h index 27cd0ba99aa3..80b1d5a9d9f1 100644 --- a/drivers/misc/habanalabs/include/common/cpucp_if.h +++ b/drivers/misc/habanalabs/include/common/cpucp_if.h @@ -84,6 +84,20 @@ struct hl_eq_sm_sei_data { __u8 pad[3]; }; +enum hl_fw_alive_severity { + FW_ALIVE_SEVERITY_MINOR, + FW_ALIVE_SEVERITY_CRITICAL +}; + +struct hl_eq_fw_alive { + __le64 uptime_seconds; + __le32 process_id; + __le32 thread_id; + /* enum hl_fw_alive_severity */ + __u8 severity; + __u8 pad[7]; +}; + struct hl_eq_entry { struct hl_eq_header hdr; union { @@ -91,6 +105,7 @@ struct hl_eq_entry { struct hl_eq_hbm_ecc_data hbm_ecc_data; struct hl_eq_sm_sei_data sm_sei_data; struct cpucp_pkt_sync_err pkt_sync_err; + struct hl_eq_fw_alive fw_alive; __le64 data[7]; }; }; @@ -103,11 +118,16 @@ struct hl_eq_entry { #define EQ_CTL_EVENT_TYPE_SHIFT 16 #define EQ_CTL_EVENT_TYPE_MASK 0x03FF0000 +#define EQ_CTL_INDEX_SHIFT 0 +#define EQ_CTL_INDEX_MASK 0x0000FFFF + enum pq_init_status { PQ_INIT_STATUS_NA = 0, PQ_INIT_STATUS_READY_FOR_CP, PQ_INIT_STATUS_READY_FOR_HOST, - PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI + PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI, + PQ_INIT_STATUS_LEN_NOT_POWER_OF_TWO_ERR, + PQ_INIT_STATUS_ILLEGAL_Q_ADDR_ERR }; /* @@ -384,6 +404,20 @@ enum cpucp_packet_id { #define CPUCP_PKT_RES_PLL_OUT3_SHIFT 48 #define CPUCP_PKT_RES_PLL_OUT3_MASK 0xFFFF000000000000ull +#define CPUCP_PKT_VAL_PFC_IN1_SHIFT 0 +#define CPUCP_PKT_VAL_PFC_IN1_MASK 0x0000000000000001ull +#define CPUCP_PKT_VAL_PFC_IN2_SHIFT 1 +#define CPUCP_PKT_VAL_PFC_IN2_MASK 0x000000000000001Eull + +#define CPUCP_PKT_VAL_LPBK_IN1_SHIFT 0 +#define CPUCP_PKT_VAL_LPBK_IN1_MASK 0x0000000000000001ull +#define CPUCP_PKT_VAL_LPBK_IN2_SHIFT 1 +#define CPUCP_PKT_VAL_LPBK_IN2_MASK 0x000000000000001Eull + +/* heartbeat status bits */ +#define CPUCP_PKT_HB_STATUS_EQ_FAULT_SHIFT 0 +#define CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK 0x00000001 + struct cpucp_packet { union { __le64 value; /* For SET packets */ @@ -425,6 +459,12 @@ struct cpucp_packet { /* For get CpuCP info/EEPROM data/NIC info */ __le32 data_max_size; + + /* + * For any general status bitmask. Shall be used whenever the + * result cannot be used to hold general purpose data. + */ + __le32 status_mask; }; __le32 reserved; @@ -629,6 +669,8 @@ struct cpucp_security_info { * @card_name: card name that will be displayed in HWMON subsystem on the host * @sec_info: security information * @pll_map: Bit map of supported PLLs for current ASIC version. + * @mme_binning_mask: MME binning mask, + * (0 = functional, 1 = binned) */ struct cpucp_info { struct cpucp_sensor sensors[CPUCP_MAX_SENSORS]; @@ -651,6 +693,7 @@ struct cpucp_info { struct cpucp_security_info sec_info; __le32 reserved6; __u8 pll_map[PLL_MAP_LEN]; + __le64 mme_binning_mask; }; struct cpucp_mac_addr { diff --git a/drivers/misc/habanalabs/include/common/hl_boot_if.h b/drivers/misc/habanalabs/include/common/hl_boot_if.h index e0a259e0495c..fa8a5ad2d438 100644 --- a/drivers/misc/habanalabs/include/common/hl_boot_if.h +++ b/drivers/misc/habanalabs/include/common/hl_boot_if.h @@ -8,7 +8,7 @@ #ifndef HL_BOOT_IF_H #define HL_BOOT_IF_H -#define LKD_HARD_RESET_MAGIC 0xED7BD694 +#define LKD_HARD_RESET_MAGIC 0xED7BD694 /* deprecated - do not use */ #define HL_POWER9_HOST_MAGIC 0x1DA30009 #define BOOT_FIT_SRAM_OFFSET 0x200000 @@ -99,6 +99,7 @@ #define CPU_BOOT_ERR0_PLL_FAIL (1 << 12) #define CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL (1 << 13) #define CPU_BOOT_ERR0_ENABLED (1 << 31) +#define CPU_BOOT_ERR1_ENABLED (1 << 31) /* * BOOT DEVICE STATUS bits in BOOT_DEVICE_STS registers @@ -190,6 +191,24 @@ * PLLs. * Initialized in: linux * + * CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN GIC access permission only from + * previleged entity. FW sets this status + * bit for host. If this bit is set then + * GIC can not be accessed from host. + * Initialized in: linux + * + * CPU_BOOT_DEV_STS0_EQ_INDEX_EN Event Queue (EQ) index is a running + * index for each new event sent to host. + * This is used as a method in host to + * identify that the waiting event in + * queue is actually a new event which + * was not served before. + * Initialized in: linux + * + * CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN Use multiple scratchpad interfaces to + * prevent IRQs overriding each other. + * Initialized in: linux + * * CPU_BOOT_DEV_STS0_ENABLED Device status register enabled. * This is a main indication that the * running FW populates the device status @@ -218,7 +237,11 @@ #define CPU_BOOT_DEV_STS0_FW_LD_COM_EN (1 << 16) #define CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN (1 << 17) #define CPU_BOOT_DEV_STS0_DYN_PLL_EN (1 << 19) +#define CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN (1 << 20) +#define CPU_BOOT_DEV_STS0_EQ_INDEX_EN (1 << 21) +#define CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN (1 << 22) #define CPU_BOOT_DEV_STS0_ENABLED (1 << 31) +#define CPU_BOOT_DEV_STS1_ENABLED (1 << 31) enum cpu_boot_status { CPU_BOOT_STATUS_NA = 0, /* Default value after reset of chip */ @@ -264,46 +287,98 @@ enum cpu_msg_status { /* communication registers mapping - consider ABI when changing */ struct cpu_dyn_regs { - uint32_t cpu_pq_base_addr_low; - uint32_t cpu_pq_base_addr_high; - uint32_t cpu_pq_length; - uint32_t cpu_pq_init_status; - uint32_t cpu_eq_base_addr_low; - uint32_t cpu_eq_base_addr_high; - uint32_t cpu_eq_length; - uint32_t cpu_eq_ci; - uint32_t cpu_cq_base_addr_low; - uint32_t cpu_cq_base_addr_high; - uint32_t cpu_cq_length; - uint32_t cpu_pf_pq_pi; - uint32_t cpu_boot_dev_sts0; - uint32_t cpu_boot_dev_sts1; - uint32_t cpu_boot_err0; - uint32_t cpu_boot_err1; - uint32_t cpu_boot_status; - uint32_t fw_upd_sts; - uint32_t fw_upd_cmd; - uint32_t fw_upd_pending_sts; - uint32_t fuse_ver_offset; - uint32_t preboot_ver_offset; - uint32_t uboot_ver_offset; - uint32_t hw_state; - uint32_t kmd_msg_to_cpu; - uint32_t cpu_cmd_status_to_host; - uint32_t reserved1[32]; /* reserve for future use */ + __le32 cpu_pq_base_addr_low; + __le32 cpu_pq_base_addr_high; + __le32 cpu_pq_length; + __le32 cpu_pq_init_status; + __le32 cpu_eq_base_addr_low; + __le32 cpu_eq_base_addr_high; + __le32 cpu_eq_length; + __le32 cpu_eq_ci; + __le32 cpu_cq_base_addr_low; + __le32 cpu_cq_base_addr_high; + __le32 cpu_cq_length; + __le32 cpu_pf_pq_pi; + __le32 cpu_boot_dev_sts0; + __le32 cpu_boot_dev_sts1; + __le32 cpu_boot_err0; + __le32 cpu_boot_err1; + __le32 cpu_boot_status; + __le32 fw_upd_sts; + __le32 fw_upd_cmd; + __le32 fw_upd_pending_sts; + __le32 fuse_ver_offset; + __le32 preboot_ver_offset; + __le32 uboot_ver_offset; + __le32 hw_state; + __le32 kmd_msg_to_cpu; + __le32 cpu_cmd_status_to_host; + union { + __le32 gic_host_irq_ctrl; + __le32 gic_host_pi_upd_irq; + }; + __le32 gic_tpc_qm_irq_ctrl; + __le32 gic_mme_qm_irq_ctrl; + __le32 gic_dma_qm_irq_ctrl; + __le32 gic_nic_qm_irq_ctrl; + __le32 gic_dma_core_irq_ctrl; + __le32 gic_host_halt_irq; + __le32 gic_host_ints_irq; + __le32 reserved1[24]; /* reserve for future use */ }; +/* TODO: remove the desc magic after the code is updated to use message */ /* HCDM - Habana Communications Descriptor Magic */ #define HL_COMMS_DESC_MAGIC 0x4843444D #define HL_COMMS_DESC_VER 1 +/* HCMv - Habana Communications Message + header version */ +#define HL_COMMS_MSG_MAGIC_VALUE 0x48434D00 +#define HL_COMMS_MSG_MAGIC_MASK 0xFFFFFF00 +#define HL_COMMS_MSG_MAGIC_VER_MASK 0xFF + +#define HL_COMMS_MSG_MAGIC_VER(ver) (HL_COMMS_MSG_MAGIC_VALUE | \ + ((ver) & HL_COMMS_MSG_MAGIC_VER_MASK)) +#define HL_COMMS_MSG_MAGIC_V0 HL_COMMS_DESC_MAGIC +#define HL_COMMS_MSG_MAGIC_V1 HL_COMMS_MSG_MAGIC_VER(1) + +#define HL_COMMS_MSG_MAGIC HL_COMMS_MSG_MAGIC_V1 + +#define HL_COMMS_MSG_MAGIC_VALIDATE_MAGIC(magic) \ + (((magic) & HL_COMMS_MSG_MAGIC_MASK) == \ + HL_COMMS_MSG_MAGIC_VALUE) + +#define HL_COMMS_MSG_MAGIC_VALIDATE_VERSION(magic, ver) \ + (((magic) & HL_COMMS_MSG_MAGIC_VER_MASK) >= \ + ((ver) & HL_COMMS_MSG_MAGIC_VER_MASK)) + +#define HL_COMMS_MSG_MAGIC_VALIDATE(magic, ver) \ + (HL_COMMS_MSG_MAGIC_VALIDATE_MAGIC((magic)) && \ + HL_COMMS_MSG_MAGIC_VALIDATE_VERSION((magic), (ver))) + +enum comms_msg_type { + HL_COMMS_DESC_TYPE = 0, + HL_COMMS_RESET_CAUSE_TYPE = 1, +}; + +/* TODO: remove this struct after the code is updated to use message */ /* this is the comms descriptor header - meta data */ struct comms_desc_header { - uint32_t magic; /* magic for validation */ - uint32_t crc32; /* CRC32 of the descriptor w/o header */ - uint16_t size; /* size of the descriptor w/o header */ - uint8_t version; /* descriptor version */ - uint8_t reserved[5]; /* pad to 64 bit */ + __le32 magic; /* magic for validation */ + __le32 crc32; /* CRC32 of the descriptor w/o header */ + __le16 size; /* size of the descriptor w/o header */ + __u8 version; /* descriptor version */ + __u8 reserved[5]; /* pad to 64 bit */ +}; + +/* this is the comms message header - meta data */ +struct comms_msg_header { + __le32 magic; /* magic for validation */ + __le32 crc32; /* CRC32 of the message w/o header */ + __le16 size; /* size of the message w/o header */ + __u8 version; /* message payload version */ + __u8 type; /* message type */ + __u8 reserved[4]; /* pad to 64 bit */ }; /* this is the main FW descriptor - consider ABI when changing */ @@ -314,7 +389,36 @@ struct lkd_fw_comms_desc { char cur_fw_ver[VERSION_MAX_LEN]; /* can be used for 1 more version w/o ABI change */ char reserved0[VERSION_MAX_LEN]; - uint64_t img_addr; /* address for next FW component load */ + __le64 img_addr; /* address for next FW component load */ +}; + +enum comms_reset_cause { + HL_RESET_CAUSE_UNKNOWN = 0, + HL_RESET_CAUSE_HEARTBEAT = 1, + HL_RESET_CAUSE_TDR = 2, +}; + +/* TODO: remove define after struct name is aligned on all projects */ +#define lkd_msg_comms lkd_fw_comms_msg + +/* this is the comms message descriptor */ +struct lkd_fw_comms_msg { + struct comms_msg_header header; + /* union for future expantions of new messages */ + union { + struct { + struct cpu_dyn_regs cpu_dyn_regs; + char fuse_ver[VERSION_MAX_LEN]; + char cur_fw_ver[VERSION_MAX_LEN]; + /* can be used for 1 more version w/o ABI change */ + char reserved0[VERSION_MAX_LEN]; + /* address for next FW component load */ + __le64 img_addr; + }; + struct { + __u8 reset_cause; + }; + }; }; /* @@ -386,11 +490,11 @@ enum comms_cmd { struct comms_command { union { /* bit fields are only for FW use */ struct { - unsigned int size :25; /* 32MB max. */ - unsigned int reserved :2; + u32 size :25; /* 32MB max. */ + u32 reserved :2; enum comms_cmd cmd :5; /* 32 commands */ }; - unsigned int val; + __le32 val; }; }; @@ -449,11 +553,11 @@ enum comms_ram_types { struct comms_status { union { /* bit fields are only for FW use */ struct { - unsigned int offset :26; - unsigned int ram_type :2; + u32 offset :26; + enum comms_ram_types ram_type :2; enum comms_sts status :4; /* 16 statuses */ }; - unsigned int val; + __le32 val; }; }; diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h b/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h index e8651abf84f2..d966bd4dfea6 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h @@ -252,10 +252,11 @@ enum gaudi_async_event_id { GAUDI_EVENT_HBM3_SPI_0 = 407, GAUDI_EVENT_HBM3_SPI_1 = 408, GAUDI_EVENT_PSOC_GPIO_U16_0 = 421, - GAUDI_EVENT_PI_UPDATE = 484, - GAUDI_EVENT_HALT_MACHINE = 485, - GAUDI_EVENT_INTS_REGISTER = 486, - GAUDI_EVENT_SOFT_RESET = 487, + GAUDI_EVENT_NIC0_CS_DBG_DERR = 483, + GAUDI_EVENT_NIC1_CS_DBG_DERR = 487, + GAUDI_EVENT_NIC2_CS_DBG_DERR = 491, + GAUDI_EVENT_NIC3_CS_DBG_DERR = 495, + GAUDI_EVENT_NIC4_CS_DBG_DERR = 499, GAUDI_EVENT_RAZWI_OR_ADC = 548, GAUDI_EVENT_TPC0_QM = 572, GAUDI_EVENT_TPC1_QM = 573, @@ -303,6 +304,11 @@ enum gaudi_async_event_id { GAUDI_EVENT_NIC3_QP1 = 619, GAUDI_EVENT_NIC4_QP0 = 620, GAUDI_EVENT_NIC4_QP1 = 621, + GAUDI_EVENT_PI_UPDATE = 635, + GAUDI_EVENT_HALT_MACHINE = 636, + GAUDI_EVENT_INTS_REGISTER = 637, + GAUDI_EVENT_SOFT_RESET = 638, + GAUDI_EVENT_FW_ALIVE_S = 645, GAUDI_EVENT_DEV_RESET_REQ = 646, GAUDI_EVENT_PKT_QUEUE_OUT_SYNC = 647, GAUDI_EVENT_FIX_POWER_ENV_S = 658, diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h b/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h index 3dc79c131805..479b6b038254 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h @@ -507,23 +507,28 @@ static struct gaudi_async_events_ids_map gaudi_irq_map_table[] = { { .fc_id = 480, .cpu_id = 329, .valid = 0, .name = "" }, { .fc_id = 481, .cpu_id = 330, .valid = 0, .name = "" }, { .fc_id = 482, .cpu_id = 331, .valid = 0, .name = "" }, - { .fc_id = 483, .cpu_id = 332, .valid = 0, .name = "" }, - { .fc_id = 484, .cpu_id = 333, .valid = 1, .name = "PI_UPDATE" }, - { .fc_id = 485, .cpu_id = 334, .valid = 1, .name = "HALT_MACHINE" }, - { .fc_id = 486, .cpu_id = 335, .valid = 1, .name = "INTS_REGISTER" }, - { .fc_id = 487, .cpu_id = 336, .valid = 1, .name = "SOFT_RESET" }, + { .fc_id = 483, .cpu_id = 332, .valid = 1, + .name = "NIC0_CS_DBG_DERR" }, + { .fc_id = 484, .cpu_id = 333, .valid = 0, .name = "" }, + { .fc_id = 485, .cpu_id = 334, .valid = 0, .name = "" }, + { .fc_id = 486, .cpu_id = 335, .valid = 0, .name = "" }, + { .fc_id = 487, .cpu_id = 336, .valid = 1, + .name = "NIC1_CS_DBG_DERR" }, { .fc_id = 488, .cpu_id = 337, .valid = 0, .name = "" }, { .fc_id = 489, .cpu_id = 338, .valid = 0, .name = "" }, { .fc_id = 490, .cpu_id = 339, .valid = 0, .name = "" }, - { .fc_id = 491, .cpu_id = 340, .valid = 0, .name = "" }, + { .fc_id = 491, .cpu_id = 340, .valid = 1, + .name = "NIC2_CS_DBG_DERR" }, { .fc_id = 492, .cpu_id = 341, .valid = 0, .name = "" }, { .fc_id = 493, .cpu_id = 342, .valid = 0, .name = "" }, { .fc_id = 494, .cpu_id = 343, .valid = 0, .name = "" }, - { .fc_id = 495, .cpu_id = 344, .valid = 0, .name = "" }, + { .fc_id = 495, .cpu_id = 344, .valid = 1, + .name = "NIC3_CS_DBG_DERR" }, { .fc_id = 496, .cpu_id = 345, .valid = 0, .name = "" }, { .fc_id = 497, .cpu_id = 346, .valid = 0, .name = "" }, { .fc_id = 498, .cpu_id = 347, .valid = 0, .name = "" }, - { .fc_id = 499, .cpu_id = 348, .valid = 0, .name = "" }, + { .fc_id = 499, .cpu_id = 348, .valid = 1, + .name = "NIC4_CS_DBG_DERR" }, { .fc_id = 500, .cpu_id = 349, .valid = 0, .name = "" }, { .fc_id = 501, .cpu_id = 350, .valid = 0, .name = "" }, { .fc_id = 502, .cpu_id = 351, .valid = 0, .name = "" }, @@ -659,17 +664,17 @@ static struct gaudi_async_events_ids_map gaudi_irq_map_table[] = { { .fc_id = 632, .cpu_id = 481, .valid = 0, .name = "" }, { .fc_id = 633, .cpu_id = 482, .valid = 0, .name = "" }, { .fc_id = 634, .cpu_id = 483, .valid = 0, .name = "" }, - { .fc_id = 635, .cpu_id = 484, .valid = 0, .name = "" }, - { .fc_id = 636, .cpu_id = 485, .valid = 0, .name = "" }, - { .fc_id = 637, .cpu_id = 486, .valid = 0, .name = "" }, - { .fc_id = 638, .cpu_id = 487, .valid = 0, .name = "" }, + { .fc_id = 635, .cpu_id = 484, .valid = 1, .name = "PI_UPDATE" }, + { .fc_id = 636, .cpu_id = 485, .valid = 1, .name = "HALT_MACHINE" }, + { .fc_id = 637, .cpu_id = 486, .valid = 1, .name = "INTS_REGISTER" }, + { .fc_id = 638, .cpu_id = 487, .valid = 1, .name = "SOFT_RESET" }, { .fc_id = 639, .cpu_id = 488, .valid = 0, .name = "" }, { .fc_id = 640, .cpu_id = 489, .valid = 0, .name = "" }, { .fc_id = 641, .cpu_id = 490, .valid = 0, .name = "" }, { .fc_id = 642, .cpu_id = 491, .valid = 0, .name = "" }, { .fc_id = 643, .cpu_id = 492, .valid = 0, .name = "" }, { .fc_id = 644, .cpu_id = 493, .valid = 0, .name = "" }, - { .fc_id = 645, .cpu_id = 494, .valid = 0, .name = "" }, + { .fc_id = 645, .cpu_id = 494, .valid = 1, .name = "FW_ALIVE_S" }, { .fc_id = 646, .cpu_id = 495, .valid = 1, .name = "DEV_RESET_REQ" }, { .fc_id = 647, .cpu_id = 496, .valid = 1, .name = "PKT_QUEUE_OUT_SYNC" }, diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h b/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h index a9f51f9f9e92..34ca4fe50d91 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h @@ -20,6 +20,9 @@ #define UBOOT_FW_OFFSET 0x100000 /* 1MB in SRAM */ #define LINUX_FW_OFFSET 0x800000 /* 8MB in HBM */ +/* HBM thermal delta in [Deg] added to composite (CTemp) */ +#define HBM_TEMP_ADJUST_COEFF 6 + enum gaudi_nic_axi_error { RXB, RXE, @@ -27,6 +30,7 @@ enum gaudi_nic_axi_error { TXE, QPC_RESP, NON_AXI_ERR, + TMR, }; /* @@ -42,6 +46,48 @@ struct eq_nic_sei_event { __u8 pad[6]; }; +/* + * struct gaudi_nic_status - describes the status of a NIC port. + * @port: NIC port index. + * @bad_format_cnt: e.g. CRC. + * @responder_out_of_sequence_psn_cnt: e.g NAK. + * @high_ber_reinit_cnt: link reinit due to high BER. + * @correctable_err_cnt: e.g. bit-flip. + * @uncorrectable_err_cnt: e.g. MAC errors. + * @retraining_cnt: re-training counter. + * @up: is port up. + * @pcs_link: has PCS link. + * @phy_ready: is PHY ready. + * @auto_neg: is Autoneg enabled. + * @timeout_retransmission_cnt: timeout retransmission events + * @high_ber_cnt: high ber events + */ +struct gaudi_nic_status { + __u32 port; + __u32 bad_format_cnt; + __u32 responder_out_of_sequence_psn_cnt; + __u32 high_ber_reinit; + __u32 correctable_err_cnt; + __u32 uncorrectable_err_cnt; + __u32 retraining_cnt; + __u8 up; + __u8 pcs_link; + __u8 phy_ready; + __u8 auto_neg; + __u32 timeout_retransmission_cnt; + __u32 high_ber_cnt; +}; + +struct gaudi_flops_2_data { + union { + struct { + __u32 spsram_init_done : 1; + __u32 reserved : 31; + }; + __u32 data; + }; +}; + #define GAUDI_PLL_FREQ_LOW 200000000 /* 200 MHz */ #endif /* GAUDI_FW_IF_H */ diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h b/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h index b53aeda9a982..9aea7e996654 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h @@ -66,7 +66,8 @@ #define PCI_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0xF)) | \ - (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0xF))) + (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0xF)) | \ + (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1))) #define HBM_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK (\ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK, 0xF)) | \ @@ -76,7 +77,8 @@ #define HBM_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0x1F)) | \ - (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F))) + (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F)) | \ + (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1))) #define TPC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK (\ (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK, 0xF)) | \ @@ -86,7 +88,8 @@ #define TPC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\ (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \ (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0x1F)) | \ - (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F))) + (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F)) | \ + (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1))) #define MME_QMAN_GLBL_ERR_CFG_MSG_EN_MASK (\ (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK, 0xF)) | \ @@ -96,7 +99,8 @@ #define MME_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\ (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \ (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0x1F)) | \ - (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F))) + (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F)) | \ + (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1))) #define NIC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK (\ (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK, 0xF)) | \ @@ -106,7 +110,8 @@ #define NIC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\ (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \ (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0xF)) | \ - (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0xF))) + (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0xF)) | \ + (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1))) #define QMAN_CGM1_PWR_GATE_EN (FIELD_PREP(DMA0_QM_CGM_CFG1_MASK_TH_MASK, 0xA)) diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h b/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h index 137afedf5f15..d95d4162ae2c 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h @@ -12,6 +12,16 @@ * PSOC scratch-pad registers */ #define mmHW_STATE mmPSOC_GLOBAL_CONF_SCRATCHPAD_0 +/* TODO: remove mmGIC_HOST_IRQ_CTRL_POLL_REG */ +#define mmGIC_HOST_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_1 +#define mmGIC_HOST_PI_UPD_IRQ_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_1 +#define mmGIC_TPC_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_2 +#define mmGIC_MME_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_3 +#define mmGIC_DMA_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_4 +#define mmGIC_NIC_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_5 +#define mmGIC_DMA_CR_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_6 +#define mmGIC_HOST_HALT_IRQ_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_7 +#define mmGIC_HOST_INTS_IRQ_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_8 #define mmCPU_BOOT_DEV_STS0 mmPSOC_GLOBAL_CONF_SCRATCHPAD_20 #define mmCPU_BOOT_DEV_STS1 mmPSOC_GLOBAL_CONF_SCRATCHPAD_21 #define mmFUSE_VER_OFFSET mmPSOC_GLOBAL_CONF_SCRATCHPAD_22 diff --git a/drivers/misc/hpilo.c b/drivers/misc/hpilo.c index fea3ae9d8686..8d00df9243c4 100644 --- a/drivers/misc/hpilo.c +++ b/drivers/misc/hpilo.c @@ -693,6 +693,8 @@ static int ilo_map_device(struct pci_dev *pdev, struct ilo_hwinfo *hw) { int bar; unsigned long off; + u8 pci_rev_id; + int rc; /* map the memory mapped i/o registers */ hw->mmio_vaddr = pci_iomap(pdev, 1, 0); @@ -702,7 +704,13 @@ static int ilo_map_device(struct pci_dev *pdev, struct ilo_hwinfo *hw) } /* map the adapter shared memory region */ - if (pdev->subsystem_device == 0x00E4) { + rc = pci_read_config_byte(pdev, PCI_REVISION_ID, &pci_rev_id); + if (rc != 0) { + dev_err(&pdev->dev, "Error reading PCI rev id: %d\n", rc); + goto out; + } + + if (pci_rev_id >= PCI_REV_ID_NECHES) { bar = 5; /* Last 8k is reserved for CCBs */ off = pci_resource_len(pdev, bar) - 0x2000; diff --git a/drivers/misc/hpilo.h b/drivers/misc/hpilo.h index f69ff645cac9..d57c34680b09 100644 --- a/drivers/misc/hpilo.h +++ b/drivers/misc/hpilo.h @@ -10,6 +10,9 @@ #define ILO_NAME "hpilo" +/* iLO ASIC PCI revision id */ +#define PCI_REV_ID_NECHES 7 + /* max number of open channel control blocks per device, hw limited to 32 */ #define MAX_CCB 24 /* min number of open channel control blocks per device, hw limited to 32 */ diff --git a/drivers/misc/ibmasm/module.c b/drivers/misc/ibmasm/module.c index 4edad6c445d3..dc8a06c06c63 100644 --- a/drivers/misc/ibmasm/module.c +++ b/drivers/misc/ibmasm/module.c @@ -111,7 +111,7 @@ static int ibmasm_init_one(struct pci_dev *pdev, const struct pci_device_id *id) result = ibmasm_init_remote_input_dev(sp); if (result) { dev_err(sp->dev, "Failed to initialize remote queue\n"); - goto error_send_message; + goto error_init_remote; } result = ibmasm_send_driver_vpd(sp); @@ -131,8 +131,9 @@ static int ibmasm_init_one(struct pci_dev *pdev, const struct pci_device_id *id) return 0; error_send_message: - disable_sp_interrupts(sp->base_address); ibmasm_free_remote_input_dev(sp); +error_init_remote: + disable_sp_interrupts(sp->base_address); free_irq(sp->irq, (void *)sp); error_request_irq: iounmap(sp->base_address); diff --git a/drivers/misc/ibmasm/remote.h b/drivers/misc/ibmasm/remote.h index 8d364462aeea..ec4e78ec5a67 100644 --- a/drivers/misc/ibmasm/remote.h +++ b/drivers/misc/ibmasm/remote.h @@ -43,7 +43,7 @@ #define REMOTE_BUTTON_MIDDLE 0x02 #define REMOTE_BUTTON_RIGHT 0x04 -/* size of keysym/keycode translation matricies */ +/* size of keysym/keycode translation matrices */ #define XLATE_SIZE 256 struct mouse_input { diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c index a164896dc6d4..88c218a9f8b3 100644 --- a/drivers/misc/lkdtm/bugs.c +++ b/drivers/misc/lkdtm/bugs.c @@ -161,6 +161,9 @@ void lkdtm_UNALIGNED_LOAD_STORE_WRITE(void) if (*p == 0) val = 0x87654321; *p = val; + + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) + pr_err("XFAIL: arch has CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS\n"); } void lkdtm_SOFTLOCKUP(void) @@ -300,8 +303,10 @@ void lkdtm_CORRUPT_LIST_ADD(void) if (target[0] == NULL && target[1] == NULL) pr_err("Overwrite did not happen, but no BUG?!\n"); - else + else { pr_err("list_add() corruption not detected!\n"); + pr_expected_config(CONFIG_DEBUG_LIST); + } } void lkdtm_CORRUPT_LIST_DEL(void) @@ -325,8 +330,10 @@ void lkdtm_CORRUPT_LIST_DEL(void) if (target[0] == NULL && target[1] == NULL) pr_err("Overwrite did not happen, but no BUG?!\n"); - else + else { pr_err("list_del() corruption not detected!\n"); + pr_expected_config(CONFIG_DEBUG_LIST); + } } /* Test that VMAP_STACK is actually allocating with a leading guard page */ diff --git a/drivers/misc/lkdtm/cfi.c b/drivers/misc/lkdtm/cfi.c index e73ebdbfa806..c9aeddef1044 100644 --- a/drivers/misc/lkdtm/cfi.c +++ b/drivers/misc/lkdtm/cfi.c @@ -38,5 +38,6 @@ void lkdtm_CFI_FORWARD_PROTO(void) func = (void *)lkdtm_increment_int; func(&called_count); - pr_info("Fail: survived mismatched prototype function call!\n"); + pr_err("FAIL: survived mismatched prototype function call!\n"); + pr_expected_config(CONFIG_CFI_CLANG); } diff --git a/drivers/misc/lkdtm/core.c b/drivers/misc/lkdtm/core.c index 8024b6a5cc7f..9dda87c6b54a 100644 --- a/drivers/misc/lkdtm/core.c +++ b/drivers/misc/lkdtm/core.c @@ -26,6 +26,7 @@ #include <linux/init.h> #include <linux/slab.h> #include <linux/debugfs.h> +#include <linux/init.h> #define DEFAULT_COUNT 10 @@ -120,11 +121,14 @@ static const struct crashtype crashtypes[] = { CRASHTYPE(UNALIGNED_LOAD_STORE_WRITE), CRASHTYPE(FORTIFY_OBJECT), CRASHTYPE(FORTIFY_SUBOBJECT), - CRASHTYPE(OVERWRITE_ALLOCATION), + CRASHTYPE(SLAB_LINEAR_OVERFLOW), + CRASHTYPE(VMALLOC_LINEAR_OVERFLOW), CRASHTYPE(WRITE_AFTER_FREE), CRASHTYPE(READ_AFTER_FREE), CRASHTYPE(WRITE_BUDDY_AFTER_FREE), CRASHTYPE(READ_BUDDY_AFTER_FREE), + CRASHTYPE(SLAB_INIT_ON_ALLOC), + CRASHTYPE(BUDDY_INIT_ON_ALLOC), CRASHTYPE(SLAB_FREE_DOUBLE), CRASHTYPE(SLAB_FREE_CROSS), CRASHTYPE(SLAB_FREE_PAGE), @@ -177,9 +181,7 @@ static const struct crashtype crashtypes[] = { CRASHTYPE(STACKLEAK_ERASING), CRASHTYPE(CFI_FORWARD_PROTO), CRASHTYPE(FORTIFIED_STRSCPY), -#ifdef CONFIG_X86_32 CRASHTYPE(DOUBLE_FAULT), -#endif #ifdef CONFIG_PPC_BOOK3S_64 CRASHTYPE(PPC_SLB_MULTIHIT), #endif @@ -399,6 +401,56 @@ static ssize_t direct_entry(struct file *f, const char __user *user_buf, return count; } +#ifndef MODULE +/* + * To avoid needing to export parse_args(), just don't use this code + * when LKDTM is built as a module. + */ +struct check_cmdline_args { + const char *param; + int value; +}; + +static int lkdtm_parse_one(char *param, char *val, + const char *unused, void *arg) +{ + struct check_cmdline_args *args = arg; + + /* short circuit if we already found a value. */ + if (args->value != -ESRCH) + return 0; + if (strncmp(param, args->param, strlen(args->param)) == 0) { + bool bool_result; + int ret; + + ret = kstrtobool(val, &bool_result); + if (ret == 0) + args->value = bool_result; + } + return 0; +} + +int lkdtm_check_bool_cmdline(const char *param) +{ + char *command_line; + struct check_cmdline_args args = { + .param = param, + .value = -ESRCH, + }; + + command_line = kstrdup(saved_command_line, GFP_KERNEL); + if (!command_line) + return -ENOMEM; + + parse_args("Setting sysctl args", command_line, + NULL, 0, -1, -1, &args, lkdtm_parse_one); + + kfree(command_line); + + return args.value; +} +#endif + static struct dentry *lkdtm_debugfs_root; static int __init lkdtm_module_init(void) diff --git a/drivers/misc/lkdtm/fortify.c b/drivers/misc/lkdtm/fortify.c index faf29cf04baa..0f51d31b57ca 100644 --- a/drivers/misc/lkdtm/fortify.c +++ b/drivers/misc/lkdtm/fortify.c @@ -76,7 +76,8 @@ void lkdtm_FORTIFIED_STRSCPY(void) */ strscpy(dst, src, strlen(src)); - pr_warn("FAIL: No overflow in above strscpy()\n"); + pr_err("FAIL: strscpy() overflow not detected!\n"); + pr_expected_config(CONFIG_FORTIFY_SOURCE); kfree(src); } diff --git a/drivers/misc/lkdtm/heap.c b/drivers/misc/lkdtm/heap.c index 1323bc16f113..3d9aae5821a0 100644 --- a/drivers/misc/lkdtm/heap.c +++ b/drivers/misc/lkdtm/heap.c @@ -5,6 +5,7 @@ */ #include "lkdtm.h" #include <linux/slab.h> +#include <linux/vmalloc.h> #include <linux/sched.h> static struct kmem_cache *double_free_cache; @@ -12,17 +13,36 @@ static struct kmem_cache *a_cache; static struct kmem_cache *b_cache; /* + * If there aren't guard pages, it's likely that a consecutive allocation will + * let us overflow into the second allocation without overwriting something real. + */ +void lkdtm_VMALLOC_LINEAR_OVERFLOW(void) +{ + char *one, *two; + + one = vzalloc(PAGE_SIZE); + two = vzalloc(PAGE_SIZE); + + pr_info("Attempting vmalloc linear overflow ...\n"); + memset(one, 0xAA, PAGE_SIZE + 1); + + vfree(two); + vfree(one); +} + +/* * This tries to stay within the next largest power-of-2 kmalloc cache * to avoid actually overwriting anything important if it's not detected * correctly. */ -void lkdtm_OVERWRITE_ALLOCATION(void) +void lkdtm_SLAB_LINEAR_OVERFLOW(void) { size_t len = 1020; u32 *data = kmalloc(len, GFP_KERNEL); if (!data) return; + pr_info("Attempting slab linear overflow ...\n"); data[1024 / sizeof(u32)] = 0x12345678; kfree(data); } @@ -89,9 +109,10 @@ void lkdtm_READ_AFTER_FREE(void) if (saw != *val) { /* Good! Poisoning happened, so declare a win. */ pr_info("Memory correctly poisoned (%x)\n", saw); - BUG(); + } else { + pr_err("FAIL: Memory was not poisoned!\n"); + pr_expected_config_param(CONFIG_INIT_ON_FREE_DEFAULT_ON, "init_on_free"); } - pr_info("Memory was not poisoned\n"); kfree(val); } @@ -145,13 +166,79 @@ void lkdtm_READ_BUDDY_AFTER_FREE(void) if (saw != *val) { /* Good! Poisoning happened, so declare a win. */ pr_info("Memory correctly poisoned (%x)\n", saw); - BUG(); + } else { + pr_err("FAIL: Buddy page was not poisoned!\n"); + pr_expected_config_param(CONFIG_INIT_ON_FREE_DEFAULT_ON, "init_on_free"); + } + + kfree(val); +} + +void lkdtm_SLAB_INIT_ON_ALLOC(void) +{ + u8 *first; + u8 *val; + + first = kmalloc(512, GFP_KERNEL); + if (!first) { + pr_info("Unable to allocate 512 bytes the first time.\n"); + return; + } + + memset(first, 0xAB, 512); + kfree(first); + + val = kmalloc(512, GFP_KERNEL); + if (!val) { + pr_info("Unable to allocate 512 bytes the second time.\n"); + return; + } + if (val != first) { + pr_warn("Reallocation missed clobbered memory.\n"); } - pr_info("Buddy page was not poisoned\n"); + if (memchr(val, 0xAB, 512) == NULL) { + pr_info("Memory appears initialized (%x, no earlier values)\n", *val); + } else { + pr_err("FAIL: Slab was not initialized\n"); + pr_expected_config_param(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, "init_on_alloc"); + } kfree(val); } +void lkdtm_BUDDY_INIT_ON_ALLOC(void) +{ + u8 *first; + u8 *val; + + first = (u8 *)__get_free_page(GFP_KERNEL); + if (!first) { + pr_info("Unable to allocate first free page\n"); + return; + } + + memset(first, 0xAB, PAGE_SIZE); + free_page((unsigned long)first); + + val = (u8 *)__get_free_page(GFP_KERNEL); + if (!val) { + pr_info("Unable to allocate second free page\n"); + return; + } + + if (val != first) { + pr_warn("Reallocation missed clobbered memory.\n"); + } + + if (memchr(val, 0xAB, PAGE_SIZE) == NULL) { + pr_info("Memory appears initialized (%x, no earlier values)\n", *val); + } else { + pr_err("FAIL: Slab was not initialized\n"); + pr_expected_config_param(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, "init_on_alloc"); + } + free_page((unsigned long)val); +} + void lkdtm_SLAB_FREE_DOUBLE(void) { int *val; diff --git a/drivers/misc/lkdtm/lkdtm.h b/drivers/misc/lkdtm/lkdtm.h index 99f90d3e5e9c..6a30b60519f3 100644 --- a/drivers/misc/lkdtm/lkdtm.h +++ b/drivers/misc/lkdtm/lkdtm.h @@ -6,6 +6,47 @@ #include <linux/kernel.h> +#define pr_expected_config(kconfig) \ +{ \ + if (IS_ENABLED(kconfig)) \ + pr_err("Unexpected! This kernel was built with " #kconfig "=y\n"); \ + else \ + pr_warn("This is probably expected, since this kernel was built *without* " #kconfig "=y\n"); \ +} + +#ifndef MODULE +int lkdtm_check_bool_cmdline(const char *param); +#define pr_expected_config_param(kconfig, param) \ +{ \ + if (IS_ENABLED(kconfig)) { \ + switch (lkdtm_check_bool_cmdline(param)) { \ + case 0: \ + pr_warn("This is probably expected, since this kernel was built with " #kconfig "=y but booted with '" param "=N'\n"); \ + break; \ + case 1: \ + pr_err("Unexpected! This kernel was built with " #kconfig "=y and booted with '" param "=Y'\n"); \ + break; \ + default: \ + pr_err("Unexpected! This kernel was built with " #kconfig "=y (and booted without '" param "' specified)\n"); \ + } \ + } else { \ + switch (lkdtm_check_bool_cmdline(param)) { \ + case 0: \ + pr_warn("This is probably expected, as kernel was built *without* " #kconfig "=y and booted with '" param "=N'\n"); \ + break; \ + case 1: \ + pr_err("Unexpected! This kernel was built *without* " #kconfig "=y but booted with '" param "=Y'\n"); \ + break; \ + default: \ + pr_err("This is probably expected, since this kernel was built *without* " #kconfig "=y (and booted without '" param "' specified)\n"); \ + break; \ + } \ + } \ +} +#else +#define pr_expected_config_param(kconfig, param) pr_expected_config(kconfig) +#endif + /* bugs.c */ void __init lkdtm_bugs_init(int *recur_param); void lkdtm_PANIC(void); @@ -39,11 +80,14 @@ void lkdtm_FORTIFY_SUBOBJECT(void); /* heap.c */ void __init lkdtm_heap_init(void); void __exit lkdtm_heap_exit(void); -void lkdtm_OVERWRITE_ALLOCATION(void); +void lkdtm_VMALLOC_LINEAR_OVERFLOW(void); +void lkdtm_SLAB_LINEAR_OVERFLOW(void); void lkdtm_WRITE_AFTER_FREE(void); void lkdtm_READ_AFTER_FREE(void); void lkdtm_WRITE_BUDDY_AFTER_FREE(void); void lkdtm_READ_BUDDY_AFTER_FREE(void); +void lkdtm_SLAB_INIT_ON_ALLOC(void); +void lkdtm_BUDDY_INIT_ON_ALLOC(void); void lkdtm_SLAB_FREE_DOUBLE(void); void lkdtm_SLAB_FREE_CROSS(void); void lkdtm_SLAB_FREE_PAGE(void); diff --git a/drivers/misc/lkdtm/stackleak.c b/drivers/misc/lkdtm/stackleak.c index d1a5c0705be3..00db21ff115e 100644 --- a/drivers/misc/lkdtm/stackleak.c +++ b/drivers/misc/lkdtm/stackleak.c @@ -74,8 +74,8 @@ void lkdtm_STACKLEAK_ERASING(void) end: if (test_failed) { - pr_err("FAIL: the thread stack is NOT properly erased\n"); - dump_stack(); + pr_err("FAIL: the thread stack is NOT properly erased!\n"); + pr_expected_config(CONFIG_GCC_PLUGIN_STACKLEAK); } else { pr_info("OK: the rest of the thread stack is properly erased\n"); } diff --git a/drivers/misc/lkdtm/usercopy.c b/drivers/misc/lkdtm/usercopy.c index 15d220ef35a5..9161ce7ed47a 100644 --- a/drivers/misc/lkdtm/usercopy.c +++ b/drivers/misc/lkdtm/usercopy.c @@ -173,6 +173,8 @@ static void do_usercopy_heap_size(bool to_user) goto free_user; } } + pr_err("FAIL: bad usercopy not detected!\n"); + pr_expected_config_param(CONFIG_HARDENED_USERCOPY, "hardened_usercopy"); free_user: vm_munmap(user_addr, PAGE_SIZE); @@ -248,6 +250,8 @@ static void do_usercopy_heap_whitelist(bool to_user) goto free_user; } } + pr_err("FAIL: bad usercopy not detected!\n"); + pr_expected_config_param(CONFIG_HARDENED_USERCOPY, "hardened_usercopy"); free_user: vm_munmap(user_alloc, PAGE_SIZE); @@ -319,7 +323,8 @@ void lkdtm_USERCOPY_KERNEL(void) pr_warn("copy_to_user failed, but lacked Oops\n"); goto free_user; } - pr_err("FAIL: survived bad copy_to_user()\n"); + pr_err("FAIL: bad copy_to_user() not detected!\n"); + pr_expected_config_param(CONFIG_HARDENED_USERCOPY, "hardened_usercopy"); free_user: vm_munmap(user_addr, PAGE_SIZE); diff --git a/drivers/misc/mei/bus-fixup.c b/drivers/misc/mei/bus-fixup.c index d8e760b11ae3..67844089db21 100644 --- a/drivers/misc/mei/bus-fixup.c +++ b/drivers/misc/mei/bus-fixup.c @@ -498,7 +498,7 @@ static struct mei_fixup { }; /** - * mei_cldev_fixup - run fixup handlers + * mei_cl_bus_dev_fixup - run fixup handlers * * @cldev: me client device */ diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c index 2cc370adb238..96f4e59c32a5 100644 --- a/drivers/misc/mei/client.c +++ b/drivers/misc/mei/client.c @@ -326,7 +326,7 @@ void mei_io_cb_free(struct mei_cl_cb *cb) } /** - * mei_tx_cb_queue - queue tx callback + * mei_tx_cb_enqueue - queue tx callback * * Locking: called under "dev->device_lock" lock * @@ -1726,12 +1726,15 @@ nortpm: return rets; } -static inline u8 mei_ext_hdr_set_vtag(struct mei_ext_hdr *ext, u8 vtag) +static inline u8 mei_ext_hdr_set_vtag(void *ext, u8 vtag) { - ext->type = MEI_EXT_HDR_VTAG; - ext->ext_payload[0] = vtag; - ext->length = mei_data2slots(sizeof(*ext)); - return ext->length; + struct mei_ext_hdr_vtag *vtag_hdr = ext; + + vtag_hdr->hdr.type = MEI_EXT_HDR_VTAG; + vtag_hdr->hdr.length = mei_data2slots(sizeof(*vtag_hdr)); + vtag_hdr->vtag = vtag; + vtag_hdr->reserved = 0; + return vtag_hdr->hdr.length; } /** @@ -1745,7 +1748,6 @@ static struct mei_msg_hdr *mei_msg_hdr_init(const struct mei_cl_cb *cb) { size_t hdr_len; struct mei_ext_meta_hdr *meta; - struct mei_ext_hdr *ext; struct mei_msg_hdr *mei_hdr; bool is_ext, is_vtag; @@ -1764,7 +1766,7 @@ static struct mei_msg_hdr *mei_msg_hdr_init(const struct mei_cl_cb *cb) hdr_len += sizeof(*meta); if (is_vtag) - hdr_len += sizeof(*ext); + hdr_len += sizeof(struct mei_ext_hdr_vtag); setup_hdr: mei_hdr = kzalloc(hdr_len, GFP_KERNEL); @@ -2250,7 +2252,7 @@ static void mei_cl_dma_free(struct mei_cl *cl) } /** - * mei_cl_alloc_and_map - send client dma map request + * mei_cl_dma_alloc_and_map - send client dma map request * * @cl: host client * @fp: pointer to file structure @@ -2349,7 +2351,7 @@ out: } /** - * mei_cl_unmap_and_free - send client dma unmap request + * mei_cl_dma_unmap - send client dma unmap request * * @cl: host client * @fp: pointer to file structure diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c index d0277c7fed10..99b5c1ecc444 100644 --- a/drivers/misc/mei/hbm.c +++ b/drivers/misc/mei/hbm.c @@ -853,7 +853,7 @@ out: } /** - * mei_hbm_cl_flow_control_res - flow control response from me + * mei_hbm_cl_tx_flow_ctrl_creds_res - flow control response from me * * @dev: the device structure * @fctrl: flow control response bus message diff --git a/drivers/misc/mei/hdcp/Kconfig b/drivers/misc/mei/hdcp/Kconfig index 95b2d6d37f10..54e1c9526909 100644 --- a/drivers/misc/mei/hdcp/Kconfig +++ b/drivers/misc/mei/hdcp/Kconfig @@ -1,4 +1,3 @@ - # SPDX-License-Identifier: GPL-2.0 # Copyright (c) 2019, Intel Corporation. All rights reserved. # diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c index cda0829ac589..d3a6c0728645 100644 --- a/drivers/misc/mei/hw-me.c +++ b/drivers/misc/mei/hw-me.c @@ -1380,7 +1380,7 @@ static bool mei_me_fw_type_nm(const struct pci_dev *pdev) .quirk_probe = mei_me_fw_type_nm /** - * mei_me_fw_sku_sps_4() - check for sps 4.0 sku + * mei_me_fw_type_sps_4() - check for sps 4.0 sku * * Read ME FW Status register to check for SPS Firmware. * The SPS FW is only signaled in the PCI function 0. @@ -1405,7 +1405,7 @@ static bool mei_me_fw_type_sps_4(const struct pci_dev *pdev) .quirk_probe = mei_me_fw_type_sps_4 /** - * mei_me_fw_sku_sps() - check for sps sku + * mei_me_fw_type_sps() - check for sps sku * * Read ME FW Status register to check for SPS Firmware. * The SPS FW is only signaled in pci function 0 diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h index b10606550613..dfd60c916da0 100644 --- a/drivers/misc/mei/hw.h +++ b/drivers/misc/mei/hw.h @@ -235,9 +235,8 @@ enum mei_ext_hdr_type { struct mei_ext_hdr { u8 type; u8 length; - u8 ext_payload[2]; - u8 hdr[]; -}; + u8 data[]; +} __packed; /** * struct mei_ext_meta_hdr - extend header meta data @@ -250,8 +249,21 @@ struct mei_ext_meta_hdr { u8 count; u8 size; u8 reserved[2]; - struct mei_ext_hdr hdrs[]; -}; + u8 hdrs[]; +} __packed; + +/** + * struct mei_ext_hdr_vtag - extend header for vtag + * + * @hdr: standard extend header + * @vtag: virtual tag + * @reserved: reserved + */ +struct mei_ext_hdr_vtag { + struct mei_ext_hdr hdr; + u8 vtag; + u8 reserved; +} __packed; /* * Extended header iterator functions @@ -266,7 +278,7 @@ struct mei_ext_meta_hdr { */ static inline struct mei_ext_hdr *mei_ext_begin(struct mei_ext_meta_hdr *meta) { - return meta->hdrs; + return (struct mei_ext_hdr *)meta->hdrs; } /** @@ -284,7 +296,7 @@ static inline bool mei_ext_last(struct mei_ext_meta_hdr *meta, } /** - *mei_ext_next - following extended header on the TLV list + * mei_ext_next - following extended header on the TLV list * * @ext: current extend header * @@ -295,7 +307,7 @@ static inline bool mei_ext_last(struct mei_ext_meta_hdr *meta, */ static inline struct mei_ext_hdr *mei_ext_next(struct mei_ext_hdr *ext) { - return (struct mei_ext_hdr *)(ext->hdr + (ext->length * 4)); + return (struct mei_ext_hdr *)((u8 *)ext + (ext->length * 4)); } /** diff --git a/drivers/misc/mei/interrupt.c b/drivers/misc/mei/interrupt.c index aab3ebfa9fc4..a67f4f2d33a9 100644 --- a/drivers/misc/mei/interrupt.c +++ b/drivers/misc/mei/interrupt.c @@ -123,13 +123,13 @@ static int mei_cl_irq_read_msg(struct mei_cl *cl, if (mei_hdr->extended) { struct mei_ext_hdr *ext; - struct mei_ext_hdr *vtag = NULL; + struct mei_ext_hdr_vtag *vtag_hdr = NULL; ext = mei_ext_begin(meta); do { switch (ext->type) { case MEI_EXT_HDR_VTAG: - vtag = ext; + vtag_hdr = (struct mei_ext_hdr_vtag *)ext; break; case MEI_EXT_HDR_NONE: fallthrough; @@ -141,20 +141,20 @@ static int mei_cl_irq_read_msg(struct mei_cl *cl, ext = mei_ext_next(ext); } while (!mei_ext_last(meta, ext)); - if (!vtag) { + if (!vtag_hdr) { cl_dbg(dev, cl, "vtag not found in extended header.\n"); cb->status = -EPROTO; goto discard; } - cl_dbg(dev, cl, "vtag: %d\n", vtag->ext_payload[0]); - if (cb->vtag && cb->vtag != vtag->ext_payload[0]) { + cl_dbg(dev, cl, "vtag: %d\n", vtag_hdr->vtag); + if (cb->vtag && cb->vtag != vtag_hdr->vtag) { cl_err(dev, cl, "mismatched tag: %d != %d\n", - cb->vtag, vtag->ext_payload[0]); + cb->vtag, vtag_hdr->vtag); cb->status = -EPROTO; goto discard; } - cb->vtag = vtag->ext_payload[0]; + cb->vtag = vtag_hdr->vtag; } if (!mei_cl_is_connected(cl)) { @@ -331,7 +331,6 @@ int mei_irq_read_handler(struct mei_device *dev, struct mei_ext_meta_hdr *meta_hdr = NULL; struct mei_cl *cl; int ret; - u32 ext_meta_hdr_u32; u32 hdr_size_left; u32 hdr_size_ext; int i; @@ -367,14 +366,12 @@ int mei_irq_read_handler(struct mei_device *dev, if (mei_hdr->extended) { if (!dev->rd_msg_hdr[1]) { - ext_meta_hdr_u32 = mei_read_hdr(dev); - dev->rd_msg_hdr[1] = ext_meta_hdr_u32; + dev->rd_msg_hdr[1] = mei_read_hdr(dev); dev->rd_msg_hdr_count++; (*slots)--; - dev_dbg(dev->dev, "extended header is %08x\n", - ext_meta_hdr_u32); + dev_dbg(dev->dev, "extended header is %08x\n", dev->rd_msg_hdr[1]); } - meta_hdr = ((struct mei_ext_meta_hdr *)dev->rd_msg_hdr + 1); + meta_hdr = ((struct mei_ext_meta_hdr *)&dev->rd_msg_hdr[1]); if (check_add_overflow((u32)sizeof(*meta_hdr), mei_slots2data(meta_hdr->size), &hdr_size_ext)) { diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c index 28937b6e7e0c..786f7c8f7f61 100644 --- a/drivers/misc/mei/main.c +++ b/drivers/misc/mei/main.c @@ -50,8 +50,6 @@ static int mei_open(struct inode *inode, struct file *file) int err; dev = container_of(inode->i_cdev, struct mei_device, cdev); - if (!dev) - return -ENODEV; mutex_lock(&dev->device_lock); @@ -1104,7 +1102,7 @@ static ssize_t dev_state_show(struct device *device, static DEVICE_ATTR_RO(dev_state); /** - * dev_set_devstate: set to new device state and notify sysfs file. + * mei_set_devstate: set to new device state and notify sysfs file. * * @dev: mei_device * @state: new device state diff --git a/drivers/misc/mei/pci-txe.c b/drivers/misc/mei/pci-txe.c index 4bf26ce61044..aec0483b8e72 100644 --- a/drivers/misc/mei/pci-txe.c +++ b/drivers/misc/mei/pci-txe.c @@ -156,7 +156,7 @@ end: } /** - * mei_txe_remove - Device Shutdown Routine + * mei_txe_shutdown- Device Shutdown Routine * * @pdev: PCI device structure * diff --git a/drivers/misc/pvpanic/pvpanic-mmio.c b/drivers/misc/pvpanic/pvpanic-mmio.c index 4c0841776087..be4016084979 100644 --- a/drivers/misc/pvpanic/pvpanic-mmio.c +++ b/drivers/misc/pvpanic/pvpanic-mmio.c @@ -93,7 +93,7 @@ static int pvpanic_mmio_probe(struct platform_device *pdev) return -EINVAL; } - pi = kmalloc(sizeof(*pi), GFP_ATOMIC); + pi = devm_kmalloc(dev, sizeof(*pi), GFP_KERNEL); if (!pi) return -ENOMEM; @@ -104,19 +104,7 @@ static int pvpanic_mmio_probe(struct platform_device *pdev) pi->capability &= ioread8(base); pi->events = pi->capability; - dev_set_drvdata(dev, pi); - - return pvpanic_probe(pi); -} - -static int pvpanic_mmio_remove(struct platform_device *pdev) -{ - struct pvpanic_instance *pi = dev_get_drvdata(&pdev->dev); - - pvpanic_remove(pi); - kfree(pi); - - return 0; + return devm_pvpanic_probe(dev, pi); } static const struct of_device_id pvpanic_mmio_match[] = { @@ -139,6 +127,5 @@ static struct platform_driver pvpanic_mmio_driver = { .dev_groups = pvpanic_mmio_dev_groups, }, .probe = pvpanic_mmio_probe, - .remove = pvpanic_mmio_remove, }; module_platform_driver(pvpanic_mmio_driver); diff --git a/drivers/misc/pvpanic/pvpanic-pci.c b/drivers/misc/pvpanic/pvpanic-pci.c index 9ecc4e8559d5..a43c401017ae 100644 --- a/drivers/misc/pvpanic/pvpanic-pci.c +++ b/drivers/misc/pvpanic/pvpanic-pci.c @@ -73,20 +73,19 @@ ATTRIBUTE_GROUPS(pvpanic_pci_dev); static int pvpanic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { - struct device *dev = &pdev->dev; struct pvpanic_instance *pi; void __iomem *base; int ret; - ret = pci_enable_device(pdev); + ret = pcim_enable_device(pdev); if (ret < 0) return ret; - base = pci_iomap(pdev, 0, 0); + base = pcim_iomap(pdev, 0, 0); if (!base) return -ENOMEM; - pi = kmalloc(sizeof(*pi), GFP_ATOMIC); + pi = devm_kmalloc(&pdev->dev, sizeof(*pi), GFP_KERNEL); if (!pi) return -ENOMEM; @@ -97,26 +96,13 @@ static int pvpanic_pci_probe(struct pci_dev *pdev, pi->capability &= ioread8(base); pi->events = pi->capability; - dev_set_drvdata(dev, pi); - - return pvpanic_probe(pi); -} - -static void pvpanic_pci_remove(struct pci_dev *pdev) -{ - struct pvpanic_instance *pi = dev_get_drvdata(&pdev->dev); - - pvpanic_remove(pi); - iounmap(pi->base); - kfree(pi); - pci_disable_device(pdev); + return devm_pvpanic_probe(&pdev->dev, pi); } static struct pci_driver pvpanic_pci_driver = { .name = "pvpanic-pci", .id_table = pvpanic_pci_id_tbl, .probe = pvpanic_pci_probe, - .remove = pvpanic_pci_remove, .driver = { .dev_groups = pvpanic_pci_dev_groups, }, diff --git a/drivers/misc/pvpanic/pvpanic.c b/drivers/misc/pvpanic/pvpanic.c index 793ea0c01193..02b807c788c9 100644 --- a/drivers/misc/pvpanic/pvpanic.c +++ b/drivers/misc/pvpanic/pvpanic.c @@ -61,25 +61,10 @@ static struct notifier_block pvpanic_panic_nb = { .priority = 1, /* let this called before broken drm_fb_helper */ }; -int pvpanic_probe(struct pvpanic_instance *pi) -{ - if (!pi || !pi->base) - return -EINVAL; - - spin_lock(&pvpanic_lock); - list_add(&pi->list, &pvpanic_list); - spin_unlock(&pvpanic_lock); - - return 0; -} -EXPORT_SYMBOL_GPL(pvpanic_probe); - -void pvpanic_remove(struct pvpanic_instance *pi) +static void pvpanic_remove(void *param) { struct pvpanic_instance *pi_cur, *pi_next; - - if (!pi) - return; + struct pvpanic_instance *pi = param; spin_lock(&pvpanic_lock); list_for_each_entry_safe(pi_cur, pi_next, &pvpanic_list, list) { @@ -90,7 +75,19 @@ void pvpanic_remove(struct pvpanic_instance *pi) } spin_unlock(&pvpanic_lock); } -EXPORT_SYMBOL_GPL(pvpanic_remove); + +int devm_pvpanic_probe(struct device *dev, struct pvpanic_instance *pi) +{ + if (!pi || !pi->base) + return -EINVAL; + + spin_lock(&pvpanic_lock); + list_add(&pi->list, &pvpanic_list); + spin_unlock(&pvpanic_lock); + + return devm_add_action_or_reset(dev, pvpanic_remove, pi); +} +EXPORT_SYMBOL_GPL(devm_pvpanic_probe); static int pvpanic_init(void) { diff --git a/drivers/misc/pvpanic/pvpanic.h b/drivers/misc/pvpanic/pvpanic.h index 1afccc2e9fec..493545951754 100644 --- a/drivers/misc/pvpanic/pvpanic.h +++ b/drivers/misc/pvpanic/pvpanic.h @@ -15,7 +15,6 @@ struct pvpanic_instance { struct list_head list; }; -int pvpanic_probe(struct pvpanic_instance *pi); -void pvpanic_remove(struct pvpanic_instance *pi); +int devm_pvpanic_probe(struct device *dev, struct pvpanic_instance *pi); #endif /* PVPANIC_H_ */ diff --git a/drivers/misc/uacce/uacce.c b/drivers/misc/uacce/uacce.c index bae18ef03dcb..488eeb2811ae 100644 --- a/drivers/misc/uacce/uacce.c +++ b/drivers/misc/uacce/uacce.c @@ -387,15 +387,22 @@ static void uacce_release(struct device *dev) static unsigned int uacce_enable_sva(struct device *parent, unsigned int flags) { + int ret; + if (!(flags & UACCE_DEV_SVA)) return flags; flags &= ~UACCE_DEV_SVA; - if (iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF)) + ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF); + if (ret) { + dev_err(parent, "failed to enable IOPF feature! ret = %pe\n", ERR_PTR(ret)); return flags; + } - if (iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA)) { + ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA); + if (ret) { + dev_err(parent, "failed to enable SVA feature! ret = %pe\n", ERR_PTR(ret)); iommu_dev_disable_feature(parent, IOMMU_DEV_FEAT_IOPF); return flags; } diff --git a/drivers/misc/vmw_vmci/vmci_context.c b/drivers/misc/vmw_vmci/vmci_context.c index 26ff49fdf0f7..c0b5e339d5a1 100644 --- a/drivers/misc/vmw_vmci/vmci_context.c +++ b/drivers/misc/vmw_vmci/vmci_context.c @@ -107,7 +107,7 @@ struct vmci_ctx *vmci_ctx_create(u32 cid, u32 priv_flags, context = kzalloc(sizeof(*context), GFP_KERNEL); if (!context) { pr_warn("Failed to allocate memory for VMCI context\n"); - error = -EINVAL; + error = -ENOMEM; goto err_out; } diff --git a/drivers/misc/xilinx_sdfec.c b/drivers/misc/xilinx_sdfec.c index 23c8448a9c3b..d6e3c650bd11 100644 --- a/drivers/misc/xilinx_sdfec.c +++ b/drivers/misc/xilinx_sdfec.c @@ -1013,9 +1013,6 @@ static __poll_t xsdfec_poll(struct file *file, poll_table *wait) xsdfec = container_of(file->private_data, struct xsdfec_dev, miscdev); - if (!xsdfec) - return EPOLLNVAL | EPOLLHUP; - poll_wait(file, &xsdfec->waitq, wait); /* XSDFEC ISR detected an error */ |